(in-package #:topic-maps.xtm.tests)
(defparameter *item-identity-originals*
'("itemid-occurrence" "occurrence-duplicate-iid" "itemid-association"
"association-duplicate-iid" "association-duplicate-iid2" "association-duplicate-iid3"
"itemid-role" "role-duplicate-iid" "role-duplicate-iid2"))
(defun ik-items (map) (append (all-occurrences map) (topic-map-associations map) (as-roles map)))
(defun ik-state (map)
(list (mm-state map) (loop for i in (topic-maps.tmdm::map-information-items map) collect (list i (item-identifiers i)))))
(defun ik-current-p (map) (and (as-current-p map) (topic-maps.tmdm::validate-map-item-identifiers map)))
(defun ik-model (map)
(mm-fields "nativeModel" (as-model map) "globalItemIdentifiersUnique" (if (ik-current-p map) :true :false)
"items" (coerce (loop for i in (ik-items map) collect
(mm-fields "kind" (string (type-of i)) "identifiers" (coerce (item-identifiers i) 'vector))) 'vector)))
(defun ik-observe (raw key)
;; Observe actual native handles at the shared productive metadata boundary.
;; AS-OBSERVE also retains pre-normalization Association/Role handles, scoped
;; to the returned real map rather than private TMDM preflight graphs.
(let ((reader (symbol-function 'topic-maps.xtm::read-item-identities))
(held nil) (map nil) (native-held nil) (stages nil) (calls 0) (same nil))
(unwind-protect
(progn
(setf (symbol-function 'topic-maps.xtm::read-item-identities)
(lambda (node item) (pushnew item held :test #'eq) (funcall reader node item)))
(multiple-value-setq (map native-held stages calls same) (as-observe raw key)))
(setf (symbol-function 'topic-maps.xtm::read-item-identities) reader))
(values map (union held native-held :test #'eq) stages calls same)))
(defun ik-held-p (map held)
(every (lambda (i) (member (topic-maps.tmdm::canonical-item i) (ik-items map) :test #'eq)) held))
(defun ik-pure (label map)
(let ((before (ik-state map)) (held (topic-maps.tmdm::map-information-items map)) (cx (mm-cxtm map)))
(check (format nil "item-identity:~A:pure-CXTM-identifiers-and-handles" label)
(and (equalp cx (mm-cxtm map)) (equal before (ik-state map)) (ik-current-p map)
(every (lambda (i) (eq i (topic-maps.tmdm::canonical-item i))) held))) cx))
(defun ik-original-model-checks (name map)
(let* ((occurrence-p (search "occurrence" name)) (role-p (search "role" name))
(item (cond (occurrence-p (first (all-occurrences map))) (role-p (first (as-roles map))) (t (first (topic-map-associations map)))))
(suffix (cond (occurrence-p "the-occurrence") (role-p "the-role") (t "the-association")))
(ids (mapcar (lambda (s) (concatenate 'string *document* "#" s))
(if (member name '("association-duplicate-iid2" "association-duplicate-iid3") :test #'equal)
'("the-association" "the-association2") (list suffix)))))
(check (format nil "item-identity-original:~A:exact-native-owner-identifiers-no-extra-topics" name)
(and (= (if occurrence-p 2 3) (length (topic-map-topics map)))
(= (if occurrence-p 1 0) (length (all-occurrences map)))
(= (if occurrence-p 0 1) (length (topic-map-associations map)))
(= (if occurrence-p 0 1) (length (as-roles map)))
(same-identifiers-p ids (item-identifiers item))
(every (lambda (other) (or (eq other item) (null (item-identifiers other)))) (ik-items map))
(every (lambda (topic) (and (= 1 (length (item-identifiers topic)))
(null (topic-subject-identifiers topic)) (null (topic-subject-locators topic))
(null (topic-reified topic)) (null (topic-names topic)))) (topic-map-topics map))
(null (item-identifiers map)) (null (item-reifier map)) (ik-current-p map)))
(check (format nil "item-identity-original:~A:exact-parent-type-player-scope-value" name)
(if occurrence-p
(and (eq (by-id map "topic") (occurrence-parent item)) (eq (by-id map "occtype") (occurrence-type item))
(equal "Testing resource data." (occurrence-value item)) (equal (string-datatype) (occurrence-datatype item))
(null (occurrence-scope item)))
(let* ((a (first (topic-map-associations map))) (r (first (association-roles a))))
(and (eq map (association-parent a)) (eq a (association-role-parent r))
(eq (by-id map "assoctype") (association-type a)) (eq (by-id map "roletype") (association-role-type r))
(eq (by-id map "topic") (association-role-player r)) (null (association-scope a))
(equal (list r) (topic-roles-played (by-id map "topic")))))))
(check (format nil "item-identity-original:~A:each-locator-on-exactly-one-canonical-item" name)
(every (lambda (id) (= 1 (count-if (lambda (i) (member id (item-identifiers i) :test #'equal))
(topic-maps.tmdm::map-information-items map)))) ids))))
(defun item-identity-original-contracts ()
(setf *item-identity-outputs* nil)
(let ((previous (copy-list *cases*)))
(dolist (name *item-identity-originals*)
(let* ((id (format nil "xtm2/in/~A.xtm" name))
(*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id))
(raw (file-bytes (corpus-path id))) (copy (copy-seq raw))
(baseline (corpus-path (format nil "xtm2/baseline/~A.xtm.cxtm" name))) (expected (file-bytes baseline))
(case (topic-maps.test-foundation:make-test-case :id id :category :valid :source-pathname (corpus-path id)
:document-iri *document* :expected-outcome :cxtm-equal :expected-baseline baseline
:clauses '("XTM 4.3.3-4.3.4,4.6,4.15,4.18-4.19" "TMDM 5.1,5.6-5.8,6.5-6.7" "CXTM 3,4")))
(map nil) (held nil) (stages nil) (calls 0) (same nil) (actual nil) (c nil))
(check (format nil "item-identity-original:~A:new-case-once" name)
(null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))
(setf *cases* (append *cases* (list case)))
(handler-case
(progn
(multiple-value-setq (map held stages calls same) (ik-observe raw id))
(ik-original-model-checks name map)
(check (format nil "item-identity-original:~A:real-qualified-parser-original-octets" name)
(and (= 1 calls) same (equalp copy raw)))
(check (format nil "item-identity-original:~A:all-observed-native-handles-live" name) (ik-held-p map held))
(when (search "duplicate" name)
(check (format nil "item-identity-original:~A:actual-native-replacement-handles" name)
(some (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held)))
(setf actual (ik-pure name map))
(let ((other (deserialize-octets raw :document-iri *document* :source-key "different:item-identity-provenance")))
(check (format nil "item-identity-original:~A:source-key-independent-identities-and-bytes" name)
(and (equalp (ik-model map) (ik-model other)) (equalp actual (mm-cxtm other))))))
(error (condition) (setf c condition)))
(record-original-result case actual expected c)
(push (mm-fields "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw)
"sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8)
"qualifiedParserCalls" calls "originalOctetsEQ" (if same :true :false)
"model" (when map (ik-model map)) "normalizationStages" (coerce (mapcar (lambda (s) (coerce s 'vector)) stages) 'vector)
"observedNativeHandles" (length held) "replacedNativeHandles" (count-if (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held)
"oldHandlesCanonicalAndLive" (if (and map (ik-held-p map held)) :true :false)
"actualBytes" (when actual (length actual)) "expectedBytes" (length expected)
"actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8)) "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8)
"firstDifferentByte" (when actual (mismatch actual expected)) "condition" (mm-condition c)
"status" (string-downcase (string (topic-maps.test-foundation:test-case-status case)))) *item-identity-outputs*)
(when c (error c))
(check (format nil "item-identity-original:~A:complete-original-CXTM-byte-exact" name) (equalp actual expected))))
(check "item-identity-original:requested-membership-unique-all-prior-cases-preserved"
(and (every (lambda (case) (and (member case *cases* :test #'eq) (eq :pass (topic-maps.test-foundation:test-case-status case)))) previous)
(same-identifiers-p (mapcar #'topic-maps.test-foundation:test-case-id (set-difference *cases* previous :test #'eq))
(mapcar (lambda (n) (format nil "xtm2/in/~A.xtm" n)) *item-identity-originals*))
(every (lambda (case) (= 1 (count (topic-maps.test-foundation:test-case-id case) *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))) *cases*))))
(setf *item-identity-outputs* (nreverse *item-identity-outputs*)))
(defun ik-ids (ids) (format nil "~{~}" ids))
(defun ik-occ (ids &key (type "#t") (value "V") scope)
(format nil "~A~A~A"
(ik-ids ids) type (if scope (format nil "~{~}" scope) "") value))
(defun ik-role (ids &optional (type "#r") (player "#p"))
(format nil "~A" (ik-ids ids) type player))
(defun ik-assoc (ids roles &key (type "#t") scope)
(format nil "~A~A~{~A~}"
(ik-ids ids) type (if scope (format nil "~{~}" scope) "") roles))
(defun ik-probe (label body verify &optional replacement)
(let* ((raw (bytes (document body))) (copy (copy-seq raw)))
(multiple-value-bind (map held stages calls same) (ik-observe raw label)
(check (format nil "item-identity:~A:native-EQ-parents-types-scopes-players-inverses" label)
(and (= 1 calls) same (equalp copy raw) (ik-current-p map) (ik-held-p map held)))
(funcall verify map)
(when replacement
(check (format nil "item-identity:~A:old-handles-forward-to-current-items" label)
(some (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held)))
(let ((cx (ik-pure label map)))
(push (mm-fields "label" label "documentIRI" *document* "sourceKey" label "sourceText" (document body)
"model" (ik-model map) "CXTM" (babel:octets-to-string cx :encoding :utf-8)
"observedNativeHandles" (length held) "replacedNativeHandles" (count-if (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held)
"oldHandlesCanonicalAndLive" :true "normalizationStages" (coerce (mapcar (lambda (s) (coerce s 'vector)) stages) 'vector)) *item-identity-controls*)) map)))
(defun ik-only-ids (label kind ids count)
(lambda (map)
(let ((items (remove-if-not (lambda (i) (typep i kind)) (ik-items map))))
(check (format nil "item-identity:~A:exact-identifier-union-no-extra-topics" label)
(and (= count (length items))
(= (if (eq kind 'occurrence) 2 3) (length (topic-map-topics map)))
(every (lambda (i) (same-identifiers-p ids (item-identifiers i))) items))))))
(defun ik-error (label body class code &key resolver)
;; Preserve the native condition and show that each failing ID assignment is
;; checked before mutation. XTM/XML source contexts and causes remain intact.
(let ((saved (symbol-function 'add-item-identifier)) (c nil) (failed-assignments nil)
(raw (bytes (document body))))
(unwind-protect
(progn
(setf (symbol-function 'add-item-identifier)
(lambda (item iri)
(let* ((map (topic-maps.tmdm::item-topic-map item)) (before (ik-state map)))
(handler-case (funcall saved item iri)
(error (condition)
(push (list (string (type-of item)) iri (equal before (ik-state map))) failed-assignments)
(error condition))))))
(setf c (condition-from (lambda () (deserialize-octets raw :document-iri *document* :source-key label :resource-resolver resolver)))))
(setf (symbol-function 'add-item-identifier) saved))
(let ((actual-code (cond ((typep c 'topic-maps.xtm:xtm-error) (error-code c))
((typep c 'item-identifier-error) (item-identifier-error-code c)))))
(check (format nil "item-identity-error:~A:exact-class-code-intended-cause" label)
(and (eq (type-of c) class) (eq code actual-code)))
(when (typep c 'topic-maps.xtm:xtm-error)
(check (format nil "item-identity-error:~A:document-source-position-cause" label)
(and (equal *document* (error-document-iri c)) (equal label (error-source-key c))
(if (typep c 'xtm-xml-error) (topic-maps.xtm:error-cause c)
(and (integerp (error-line c)) (integerp (error-column c)))))))
(when (typep c 'item-identifier-error)
(check (format nil "item-identity-error:~A:failed-native-assignment-atomic-not-merged-by-II" label)
(and failed-assignments (every #'third failed-assignments))))
(push (mm-fields "label" label "documentIRI" *document* "sourceKey" label "sourceText" (document body)
"condition" (mm-condition c) "nativeIdentifierCode" (when (typep c 'item-identifier-error) (string-downcase (string actual-code)))
"nativeConditionHasNoSourcePositionSlots" (if (typep c 'item-identifier-error) :true :false)
"rejectedAssignments" (coerce (mapcar (lambda (x) (mm-fields "itemKind" (first x) "locator" (second x) "nativeAssignmentUnchanged" (if (third x) :true :false))) failed-assignments) 'vector)) *item-identity-controls*)) c))
(defun item-identity-tests ()
(setf *item-identity-controls* nil)
(dolist (kind '(occurrence association association-role))
(dolist (row '(("one-II" ("urn:first")) ("multiple-and-repeated-II" ("urn:first" "urn:second" "urn:first"))))
(let* ((label (format nil "~A:~A" kind (first row))) (ids (second row))
(body (ecase kind
(occurrence (format nil "~A" (ik-occ ids)))
(association (ik-assoc ids (list (ik-role nil))))
(association-role (ik-assoc nil (list (ik-role ids)))))))
(ik-probe label body (ik-only-ids label kind (remove-duplicates ids :test #'equal) 1))))
(let* ((label (format nil "~A:equal-items-different-II" kind))
(body (ecase kind
(occurrence (format nil "~A~A" (ik-occ '("urn:first")) (ik-occ '("urn:second"))))
(association (concatenate 'string (ik-assoc '("urn:first") (list (ik-role nil))) (ik-assoc '("urn:second") (list (ik-role nil)))))
(association-role (ik-assoc nil (list (ik-role '("urn:first")) (ik-role '("urn:second"))))))))
(ik-probe label body (ik-only-ids label kind '("urn:first" "urn:second") 1) t)))
(dolist (kind '(occurrence association association-role))
(let* ((label (format nil "~A:relative-percent-Unicode-II" kind))
(ids '("../asset#one" "#two" "../%E9%A6%AC#three" "https://id.example/a#four"))
(body (ecase kind (occurrence (format nil "~A" (ik-occ ids)))
(association (ik-assoc ids (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role ids)))))))
(ik-probe label body (ik-only-ids label kind (mapcar #'mm-child-iri ids) 1))))
;; Corresponding Roles in duplicate Associations occur in reversed source
;; order; II must follow actual Type/Player, never list position.
(ik-probe "role-II-match-by-signature-after-association-normalization"
(concatenate 'string
(ik-assoc '("urn:as1") (list (ik-role '("urn:r1") "#r" "#p") (ik-role '("urn:r2") "#rr" "#q")))
(ik-assoc '("urn:as2") (list (ik-role '("urn:r4") "#rr" "#q") (ik-role '("urn:r3") "#r" "#p"))))
(lambda (map)
(let ((a (first (topic-map-associations map))))
(check "item-identity:role-signature:correct-unions-parent-and-inverses"
(and (= 1 (length (topic-map-associations map))) (= 2 (length (association-roles a)))
(same-identifiers-p '("urn:as1" "urn:as2") (item-identifiers a))
(loop for player in '("p" "q") for ids in '(("urn:r1" "urn:r3") ("urn:r2" "urn:r4")) always
(same-identifiers-p ids (item-identifiers (find (by-id map player) (association-roles a) :key #'association-role-player :test #'eq)))))))) t)
(ik-late-merges)
(ik-negative-tests)
(ik-import-tests)
(setf *item-identity-controls* (nreverse *item-identity-controls*)))
(defun ik-late-merges ()
(dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity"))
(let* ((label (concatenate 'string "late-" kind))
(body (concatenate 'string
(format nil "~A~A"
(ik-occ '("urn:o1") :type "#t" :scope '("#s")) (ik-occ '("urn:o2") :type "#u" :scope '("#v")))
(ik-assoc '("urn:a1") (list (ik-role '("urn:r1") "#r" "#p")) :type "#t" :scope '("#s"))
(ik-assoc '("urn:a2") (list (ik-role '("urn:r2") "#rr" "#q")) :type "#u" :scope '("#v"))
(format nil "<~A href='urn:owner'/><~A href='urn:owner'/><~A href='urn:type'/><~A href='urn:type'/><~A href='urn:role'/><~A href='urn:role'/><~A href='urn:scope'/><~A href='urn:scope'/>" kind kind kind kind kind kind kind kind))))
(ik-probe label body
(lambda (map)
(check (format nil "item-identity:~A:all-three-item-unions-and-native-relations" label)
(and (= 4 (length (topic-map-topics map))) (= 1 (length (all-occurrences map)))
(= 1 (length (topic-map-associations map))) (= 1 (length (as-roles map)))
(same-identifiers-p '("urn:o1" "urn:o2") (item-identifiers (first (all-occurrences map))))
(same-identifiers-p '("urn:a1" "urn:a2") (item-identifiers (first (topic-map-associations map))))
(same-identifiers-p '("urn:r1" "urn:r2") (item-identifiers (first (as-roles map))))
(eq (by-id map "p") (by-id map "q")) (eq (by-id map "s") (by-id map "v"))))) t))))
(defun ik-negative-tests ()
(dolist (kind '(occurrence association association-role))
(dolist (bad '("%ZZ" "%C3%28" "a#b#c"))
(let ((body (ecase kind (occurrence (format nil "~A" (ik-occ (list bad))))
(association (ik-assoc (list bad) (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role (list bad))))))))
(ik-error (format nil "~A:invalid-II:~A" kind bad) body 'xtm-rejection
(cond ((equal bad "%ZZ") :invalid-percent-escape) ((equal bad "%C3%28") :invalid-percent-utf8) (t :invalid-iri)))))
(let* ((label (format nil "~A:II-collides-with-map" kind))
(body (ecase kind (occurrence (format nil "~A" (ik-occ '("urn:collision"))))
(association (ik-assoc '("urn:collision") (list (ik-role nil))))
(association-role (ik-assoc nil (list (ik-role '("urn:collision"))))))))
(ik-error label (concatenate 'string "" body) 'item-identifier-error :duplicate-item-identifier)))
(ik-error "different-occurrences-same-II" (format nil "~A~A" (ik-occ '("urn:collision")) (ik-occ '("urn:collision") :value "different")) 'item-identifier-error :duplicate-item-identifier)
(ik-error "different-associations-same-II" (concatenate 'string (ik-assoc '("urn:collision") (list (ik-role nil))) (ik-assoc '("urn:collision") (list (ik-role nil)) :type "#other")) 'item-identifier-error :duplicate-item-identifier)
(ik-error "different-roles-same-II" (ik-assoc nil (list (ik-role '("urn:collision") "#r" "#p") (ik-role '("urn:collision") "#r" "#q"))) 'item-identifier-error :duplicate-item-identifier)
(ik-error "same-role-signature-different-parents-same-II" (concatenate 'string (ik-assoc nil (list (ik-role '("urn:collision")))) (ik-assoc nil (list (ik-role '("urn:collision"))) :type "#other")) 'item-identifier-error :duplicate-item-identifier)
(ik-error "association-role-cross-type-collision" (ik-assoc '("urn:collision") (list (ik-role '("urn:collision")))) 'item-identifier-error :duplicate-item-identifier)
(ik-error "occurrence-association-cross-type-collision" (concatenate 'string (format nil "~A" (ik-occ '("urn:collision"))) (ik-assoc '("urn:collision") (list (ik-role nil)))) 'item-identifier-error :duplicate-item-identifier)
(ik-error "II-stays-conflict-after-topic-merging"
(concatenate 'string (ik-assoc '("urn:collision") (list (ik-role nil "#r" "#p")))
(ik-assoc '("urn:other") (list (ik-role nil "#r" "#q")))
""
(ik-assoc '("urn:collision") (list (ik-role nil "#r" "#other")))) 'item-identifier-error :duplicate-item-identifier)
(dolist (row '(("occurrence-II-after-type" "V" :element-order)
("association-II-after-type" "" :element-order)
("role-II-after-type" "" :element-order)
("II-in-type" "" :unexpected-element)
("II-in-scope" "" :unexpected-element)
("II-not-empty" "" :unexpected-child)
("II-missing-href" "" :missing-attribute)))
(ik-error (first row) (second row) 'xtm-rejection (third row)))
(ik-error "malformed-XML" "" 'xtm-xml-error :xml-not-well-formed)
(ik-error "occurrence-reification-remains-unsupported" "V" 'xtm-unsupported :resource-data-datatype)
;; Explicitly excluded original: preserve XML-ID uniqueness, no registration.
(let* ((id "xtm2/in/occurrence-duplicate-iid2.xtm") (raw (file-bytes (corpus-path id)))
(c (condition-from (lambda () (deserialize-octets raw :document-iri *document* :source-key id)))))
(check "item-identity:excluded-duplicate-XML-ID-not-registered"
(and (eq (type-of c) 'xtm-rejection) (eq :duplicate-id (error-code c))
(null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))))
(push (mm-fields "label" "excluded-duplicate-XML-ID" "id" id "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list))
"sourceText" (babel:octets-to-string raw :encoding :utf-8) "condition" (mm-condition c) "registered" :false) *item-identity-controls*)))
(defun ik-import-tests ()
(let* ((child (mm-child-iri "child.xtm"))
(raw (bytes (document (concatenate 'string (format nil "~A" (ik-occ '("urn:child-occurrence")))
(ik-assoc '("urn:child-association") (list (ik-role '("urn:child-role")))))))))
(mm-call "item-identities-native-import" (document (mm-directive "child.xtm")) (list (list child raw "identifier-child"))
(lambda (map imports calls) (declare (ignore calls))
(check "item-identity:import:all-three-identifier-sets-native-EQ-source-preserved"
(and (ik-current-p map)
(equal '("urn:child-occurrence") (item-identifiers (first (all-occurrences map))))
(equal '("urn:child-association") (item-identifiers (first (topic-map-associations map))))
(equal '("urn:child-role") (item-identifiers (first (as-roles map))))
(every (lambda (record) (every (lambda (item)
(same-identifiers-p (item-identifiers item) (item-identifiers (funcall (second record) item)))) (ik-items (first record)))) imports))))))
(let* ((child (mm-child-iri "child.xtm"))
(raw (bytes (document (ik-assoc '("urn:map-collision") (list (ik-role nil))))))
(body (concatenate 'string "" (mm-directive "child.xtm")))
(c (ik-error "import-II-conflict-cause" body 'topic-maps.xtm:xtm-resource-error :mergemap-import-failure
:resolver (lambda (&rest args) (declare (ignore args)) (values raw "conflicting-child")))))
(declare (ignore child))
(check "item-identity:import-conflict:original-native-Cause-retained"
(and (typep (topic-maps.xtm:error-cause c) 'item-identifier-error)
(eq :duplicate-item-identifier (item-identifier-error-code (topic-maps.xtm:error-cause c)))))))