(in-package #:topic-maps.xtm.tests)
(defun datatype-native-preflight ()
;; Run before changing the Reader, and again in the regular ASDF test-op.
(let* ((map (make-topic-map))
(owner (intern-topic map :item-identifier "urn:datatype:owner"))
(base (intern-topic map :item-identifier "urn:datatype:base"))
(a (intern-topic map :item-identifier "urn:datatype:a"))
(b (intern-topic map :item-identifier "urn:datatype:b"))
(r1 (intern-topic map :item-identifier "urn:datatype:r1"))
(r2 (intern-topic map :item-identifier "urn:datatype:r2"))
(name (add-default-name owner "N" :scope (list base)))
(count (length (topic-map-topics map)))
(datatype "https://types.example.invalid/custom")
(value " 42 & 馬🦊 ")
(v1 (add-variant name value :datatype datatype :scope (list base a)))
(v2 (add-variant name value :datatype datatype :scope (list base b)))
(other (add-variant name value :datatype "urn:types:other" :scope (list base a))))
(check "datatype-native:value-and-datatype-independent"
(and (equal value (variant-value v1)) (equal datatype (variant-datatype v1))
(equal "urn:types:other" (variant-datatype other))))
(check "datatype-native:no-topic-creation" (= count (length (topic-map-topics map))))
(check "datatype-native:distinct-datatypes-and-scopes" (= 3 (length (all-variants map))))
(add-item-identifier v1 "urn:variant:first")
(add-item-identifier v2 "urn:variant:second")
(set-item-reifier v1 r1) (set-item-reifier v2 r2)
(merge-topics a b)
(let ((current (canonical-variant v1)))
(check "datatype-native:old-variant-handles-canonical-EQ"
(and (eq current (canonical-variant v2)) (not (eq current v1))
(not (eq current other)) (= 2 (length (all-variants map)))))
(check "datatype-native:parent-and-scope-canonical-EQ"
(and (eq name (canonical-topic-name name)) (eq owner (topic-name-parent name))
(eq name (variant-parent current))
(scope-eq-p (list base (canonical-topic a)) (variant-scope current))))
(check "datatype-native:identifiers-unioned"
(and (= 2 (length (item-identifiers current)))
(member "urn:variant:first" (item-identifiers current) :test #'equal)
(member "urn:variant:second" (item-identifiers current) :test #'equal)))
(check "datatype-native:reifier-inverse-and-old-handles-EQ"
(and (eq (canonical-topic r1) (canonical-topic r2))
(eq (item-reifier current) (canonical-topic r1))
(eq current (topic-reified r1)) (eq current (topic-reified r2))))
(let ((duplicate (add-variant name value :datatype datatype :scope (list base (canonical-topic a)))))
(check "datatype-native:duplicate-add-normalizes"
(and (eq (canonical-variant current) duplicate) (eq duplicate (canonical-variant v1))
(eq duplicate (canonical-variant v2)) (= 2 (length (all-variants map))))))
(check "datatype-native:datatype-survives-normalization"
(and (equal datatype (variant-datatype v1)) (equal value (variant-value v2)))))
(let* ((before (map-reifier-state map))
(actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*)))
(check "datatype-native:cxtm-preserves-custom-lexical-value"
(and (search " 42 & 馬🦊 " (babel:octets-to-string actual :encoding :utf-8))
(search (concatenate 'string "" datatype "")
(babel:octets-to-string actual :encoding :utf-8))))
(check "datatype-native:pure-deterministic-canonicalization"
(and (equal before (map-reifier-state map))
(equalp actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*))
(equal before (map-reifier-state map)) (variant-reifier-current-p map))))
map))
(defvar *datatype-outputs* nil)
(defvar *datatype-controls* nil)
(defun datatype-resource (datatype value)
(format nil "~A"
(if datatype (format nil " datatype='~A'" datatype) "") value))
(defun datatype-variant (datatype value &key (themes '("s")) reifier identifiers)
(variant-reifier-xml reifier :themes themes :identifiers identifiers
:resource (datatype-resource datatype value)))
(defun datatype-body (resource)
(format nil "N~A~A" (scope-xml '("s")) resource))
(defun datatype-original-contract ()
(let* ((id "xtm2/in/variant-datatype-unknown.xtm")
(*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id))
(raw (file-bytes (corpus-path id))) (copy (copy-seq raw))
(baseline (corpus-path "xtm2/baseline/variant-datatype-unknown.xtm.cxtm"))
(expected (file-bytes baseline))
(case (topic-maps.test-foundation:make-test-case :id id :category :valid
:source-pathname (corpus-path id) :document-iri *document* :expected-outcome :cxtm-equal
:expected-baseline baseline :clauses '("XTM 2.0 4.12,4.16.1-4.16.3" "TMDM 4.4,5.5,6.4" "CXTM 3.14,3.16 Other")))
(map nil) (actual nil) (condition nil) (parser-calls 0) (parser-input-eq nil))
(setf *cases* (append *cases* (list case)))
(handler-case
(progn
(let ((saved (symbol-function 'topic-maps.xml:parse-octets)))
(unwind-protect
(progn
(setf (symbol-function 'topic-maps.xml:parse-octets)
(lambda (octets &rest args)
(setf parser-input-eq (eq octets raw))
(incf parser-calls) (apply saved octets args)))
(setf map (deserialize-octets raw :document-iri *document* :source-key id)))
(setf (symbol-function 'topic-maps.xml:parse-octets) saved)))
(let* ((owner (by-id map "topic")) (theme (by-id map "scope"))
(name (first (topic-names owner))) (v (first (all-variants map)))
(type (by-si map (default-name-type-identifier))))
(check "datatype-original:native-model-parent-type-scope-EQ"
(and (= 3 (length (topic-map-topics map))) (null (topic-map-associations map))
(every (lambda (topic) (null (topic-occurrences topic))) (topic-map-topics map))
(= 1 (length (name-reifier-names map))) (= 1 (length (all-variants map)))
(equal "Topic" (topic-name-value name)) (null (topic-name-scope name))
(eq owner (topic-name-parent name)) (eq type (topic-name-type name))
(eq name (canonical-topic-name name)) (eq name (variant-parent v))
(eq v (canonical-variant v)) (scope-eq-p (list theme) (variant-scope v))
(variant-reifier-current-p map)))
(check "datatype-original:value-datatype-not-topic-identities"
(and (equal "BOGUS" (variant-value v)) (equal "http://example.com/bogus" (variant-datatype v))
(equal (list (concatenate 'string *document* "#topic")) (item-identifiers owner))
(equal (list (concatenate 'string *document* "#scope")) (item-identifiers theme))
(null (item-identifiers type)) (equal (list (default-name-type-identifier)) (topic-subject-identifiers type))
(null (topic-subject-identifiers owner)) (null (topic-subject-identifiers theme))
(every (lambda (topic) (null (topic-subject-locators topic))) (topic-map-topics map))
(null (item-identifiers map)) (null (item-identifiers name)) (null (item-identifiers v))
(null (item-reifier map)) (null (item-reifier name)) (null (item-reifier v))
(null (by-si map "http://example.com/bogus")))))
(setf actual (variant-reifier-pure "datatype-original" map (append (all-variants map) (name-reifier-names map) (topic-map-topics map))))
(check "datatype-original:parser-receives-original-octets" parser-input-eq)
(check "datatype-original:source-preserved-parser-once" (and (= parser-calls 1) (equalp raw copy)))
(let ((other (deserialize-octets raw :document-iri *document* :source-key "independent:datatype-original")))
(check "datatype-original:document-IRI-source-key-independence"
(and (equalp (variant-reifier-model-data map) (variant-reifier-model-data other))
(equalp actual (topic-maps.cxtm:canonicalize-octets other :base-locator *document*))))))
(error (c) (setf condition c)))
(record-original-result case actual expected condition)
(setf *datatype-outputs* (list (topic-maps.test-foundation::fields
"id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw)
"sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8)
"qualifiedParserCalls" parser-calls "condition" (map-reifier-condition-data condition)
"model" (when map (variant-reifier-model-data map))
"actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8))
"expectedCXTM" (babel:octets-to-string expected :encoding :utf-8)
"actualBytes" (when actual (length actual)) "expectedBytes" (length expected)
"firstDifferentByte" (when actual (mismatch actual expected))
"status" (string-downcase (string (topic-maps.test-foundation:test-case-status case))))))
(when condition (error condition))
(check "datatype-original:579-byte-exact" (and (= 579 (length expected)) (equalp actual expected)))
(check "datatype-original:72-unique-registered-cases"
(and (= 72 (length *cases*)) (= 64 (count :valid *cases* :key #'topic-maps.test-foundation:test-case-category))
(= 72 (length (remove-duplicates *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))))))
(defun datatype-value-controls ()
(dolist (row (list
(list "http-custom" "http://types.example.invalid/custom" "literal%GG" "literal%GG")
(list "urn-custom" "urn:types:custom" "urn:resource-not-a-topic" "urn:resource-not-a-topic")
(list "unicode" "urn:types:馬" "馬🦊é" "馬🦊é")
(list "datatype-percent-preserved" "urn:types:%61" "42" "42")
(list "whitespace-escaping" "urn:types:text" " A&<>
" (format nil " ~CA&<>~% " #\Tab))
(list "empty" "urn:types:empty" "" "")
(list "integer-text-now-supported" (integer-datatype) "+00042" "+00042")
(list "explicit-string" (string-datatype) "urn:%GG" "urn:%GG")
(list "default-string" nil " x & y " " x & y ")
(list "anyURI-retained" (iri-datatype) "../%E9%A6%AC" "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/xtm2/馬")))
(destructuring-bind (label dtype xml value) row
(variant-reifier-control (concatenate 'string "datatype:" label) (datatype-body (datatype-resource dtype xml))
(lambda (map)
(let ((v (first (all-variants map))))
(check (concatenate 'string "datatype:value-and-exact-datatype:" label)
(and (equal value (variant-value v)) (equal (or dtype (string-datatype)) (variant-datatype v))
(= 3 (length (topic-map-topics map))) (eq (variant-parent v) (first (topic-names (by-id map "a"))))
(scope-eq-p (variant-scope v) (list (by-id map "s"))))))))))
(variant-reifier-control "datatype:normalization-discriminators"
(format nil "~A" (variant-name-xml nil (concatenate 'string
(datatype-variant "urn:type:a" "V") (datatype-variant "urn:type:a" "V")
(datatype-variant "urn:type:a" "W") (datatype-variant "urn:type:b" "V"))))
(lambda (map)
(check "datatype:equality-includes-datatype-and-value"
(and (= 3 (length (all-variants map))) (= 3 (length (topic-map-topics map)))
(= 2 (count "urn:type:a" (all-variants map) :key #'variant-datatype :test #'equal))
(= 2 (count "V" (all-variants map) :key #'variant-value :test #'equal)))))))
(defun datatype-merge-controls ()
(dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity"))
(let* ((held nil) (saved (symbol-function 'topic-maps.xtm::read-variant))
(body (format nil "<~A href='urn:owner'/>~A<~A href='urn:owner'/>~A<~A href='urn:theme'/><~A href='urn:theme'/>"
kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("x") :reifier "#r1" :identifiers '("urn:v1")))
kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("y") :reifier "#r2" :identifiers '("urn:v2"))) kind kind)))
(unwind-protect
(progn
(setf (symbol-function 'topic-maps.xtm::read-variant)
(lambda (&rest args) (let ((v (apply saved args))) (push v held) v)))
(variant-reifier-control (concatenate 'string "datatype:merge-" kind) body
(lambda (map)
(let* ((owner (by-id map "a")) (name (first (topic-names owner))) (v (first (all-variants map)))
(r (by-id map "r1")))
(check (concatenate 'string "datatype:merge-canonical-identities:" kind)
(and (eq owner (by-id map "b")) (eq name (variant-parent v))
(eq owner (topic-name-parent name)) (eq (by-id map "x") (by-id map "y"))
(scope-eq-p (variant-scope v) (list (by-id map "base") (by-id map "x")))
(= 1 (length (all-variants map))) (= 2 (length held)) (not (eq (first held) (second held)))
(every (lambda (old) (eq v (canonical-variant old))) held)
(= 2 (length (item-identifiers v))) (member "urn:v1" (item-identifiers v) :test #'equal)
(member "urn:v2" (item-identifiers v) :test #'equal)
(eq r (by-id map "r2")) (eq r (item-reifier v)) (eq v (topic-reified r))
(equal "urn:datatype:merged" (variant-datatype v)) (equal "V" (variant-value v))))
(variant-reifier-pure (concatenate 'string "datatype:held-" kind) map held)))))
(setf (symbol-function 'topic-maps.xtm::read-variant) saved)))))
(defun datatype-negative-controls ()
(dolist (row '(("relative" "relative" :invalid-resource-data-datatype)
("empty" "" :invalid-resource-data-datatype) ("scheme" "1bad:x" :invalid-resource-data-datatype)
("space" "urn:bad type" :invalid-iri) ("percent" "urn:type:%GG" :invalid-percent-escape)
("percent-short" "urn:type:%" :invalid-percent-escape) ("percent-UTF8" "urn:type:%C3%28" :invalid-percent-utf8)
("percent-surrogate" "urn:type:%ED%A0%80" :invalid-percent-utf8)))
(variant-reifier-error-control (concatenate 'string "datatype:" (first row))
(datatype-body (datatype-resource (second row) "V")) 'xtm-rejection (third row)))
(dolist (dtype '(nil "http://www.w3.org/2001/XMLSchema#string" "http://www.w3.org/2001/XMLSchema#anyURI" "urn:custom"))
(dolist (child '("V" "V"))
(variant-reifier-error-control (format nil "datatype:forbidden-child:~A:~A" dtype child)
(datatype-body (datatype-resource dtype child)) 'xtm-rejection :unexpected-child)))
(dolist (value '("text" "XML"))
(variant-reifier-error-control (concatenate 'string "datatype:anyType:" value)
(datatype-body (datatype-resource "http://www.w3.org/2001/XMLSchema#anyType" value))
'xtm-unsupported :resource-data-datatype))
(variant-reifier-error-control "datatype:unknown-XTM-structure" "" 'xtm-rejection :unexpected-element)
(variant-reifier-error-control "datatype:occurrence-remains-unsupported"
"V"
'xtm-unsupported :unsupported-construct)
(variant-reifier-error-control "datatype:native-reifier-conflict"
(format nil "~A" (variant-name-xml nil (concatenate 'string
(datatype-variant "urn:custom" "A" :reifier "#r") (datatype-variant "urn:custom" "B" :reifier "#r"))))
'reification-error :reifier-conflict)
(variant-reifier-error-control "datatype:native-scope-conflict"
(format nil "~A" (variant-name-xml '("s") (datatype-variant "urn:custom" "V")))
'variant-error :invalid-variant-scope)
(let ((raw (concatenate '(vector (unsigned-byte 8))
(bytes "N")
#(195 40) (bytes ""))))
(variant-reifier-error-control "datatype:raw-invalid-UTF8" nil 'xtm-xml-error :xml-not-well-formed
(lambda () (deserialize-octets raw :document-iri *document* :source-key "variant:datatype:raw-invalid-UTF8")))))
(defun datatype-context-controls ()
;; Retain the resource-operation/native-consumer cross-check from Slice 22D.
;; Slice 22G separately exercises genuine XTM occurrence deserialization.
(dolist (resource (list (datatype-resource "urn:shared:type" " A&B ")
(datatype-resource nil "V")
(datatype-resource (iri-datatype) "../sound")
""))
(let* ((root (topic-maps.xml:parse-octets (bytes (document (datatype-body resource)))
(make-instance 'topic-maps.xtm::source-handler)))
(node (second (topic-maps.xtm::elements
(second (topic-maps.xtm::elements
(first (topic-maps.xtm::elements (first (topic-maps.xtm::elements root)))))))))
(topic-maps.xtm::*document-iri* *document*)
(topic-maps.xtm::*source-key* "datatype:shared-context"))
(topic-maps.xtm::validate-source root)
(multiple-value-bind (value dtype) (topic-maps.xtm::read-resource-value node)
(let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:owner"))
(type (intern-topic map :item-identifier "urn:type")) (theme (intern-topic map :item-identifier "urn:theme"))
(name (add-default-name owner "N")) (count (length (topic-map-topics map)))
(v (add-variant name value :datatype dtype :scope (list theme)))
(o (add-occurrence owner type value :datatype dtype)))
(check (concatenate 'string "datatype:shared-value-consumers:" resource)
(and (equal (variant-value v) (occurrence-value o)) (equal (variant-datatype v) (occurrence-datatype o))
(eq name (variant-parent v)) (eq owner (occurrence-parent o)) (= count (length (topic-map-topics map))))))))))
(defun datatype-tests ()
(datatype-native-preflight)
(let ((*variant-reifier-controls* nil))
(datatype-value-controls) (datatype-merge-controls) (datatype-negative-controls) (datatype-context-controls)
(setf *datatype-controls* (nreverse *variant-reifier-controls*))))