(in-package #:topic-maps.xtm.tests) (defparameter *item-identity-originals* '("itemid-occurrence" "occurrence-duplicate-iid" "itemid-association" "association-duplicate-iid" "association-duplicate-iid2" "association-duplicate-iid3" "itemid-role" "role-duplicate-iid" "role-duplicate-iid2")) (defun ik-items (map) (append (all-occurrences map) (topic-map-associations map) (as-roles map))) (defun ik-state (map) (list (mm-state map) (loop for i in (topic-maps.tmdm::map-information-items map) collect (list i (item-identifiers i))))) (defun ik-current-p (map) (and (as-current-p map) (topic-maps.tmdm::validate-map-item-identifiers map))) (defun ik-model (map) (mm-fields "nativeModel" (as-model map) "globalItemIdentifiersUnique" (if (ik-current-p map) :true :false) "items" (coerce (loop for i in (ik-items map) collect (mm-fields "kind" (string (type-of i)) "identifiers" (coerce (item-identifiers i) 'vector))) 'vector))) (defun ik-observe (raw key) ;; Observe actual native handles at the shared productive metadata boundary. ;; AS-OBSERVE also retains pre-normalization Association/Role handles, scoped ;; to the returned real map rather than private TMDM preflight graphs. (let ((reader (symbol-function 'topic-maps.xtm::read-item-identities)) (held nil) (map nil) (native-held nil) (stages nil) (calls 0) (same nil)) (unwind-protect (progn (setf (symbol-function 'topic-maps.xtm::read-item-identities) (lambda (node item) (pushnew item held :test #'eq) (funcall reader node item))) (multiple-value-setq (map native-held stages calls same) (as-observe raw key))) (setf (symbol-function 'topic-maps.xtm::read-item-identities) reader)) (values map (union held native-held :test #'eq) stages calls same))) (defun ik-held-p (map held) (every (lambda (i) (member (topic-maps.tmdm::canonical-item i) (ik-items map) :test #'eq)) held)) (defun ik-pure (label map) (let ((before (ik-state map)) (held (topic-maps.tmdm::map-information-items map)) (cx (mm-cxtm map))) (check (format nil "item-identity:~A:pure-CXTM-identifiers-and-handles" label) (and (equalp cx (mm-cxtm map)) (equal before (ik-state map)) (ik-current-p map) (every (lambda (i) (eq i (topic-maps.tmdm::canonical-item i))) held))) cx)) (defun ik-original-model-checks (name map) (let* ((occurrence-p (search "occurrence" name)) (role-p (search "role" name)) (item (cond (occurrence-p (first (all-occurrences map))) (role-p (first (as-roles map))) (t (first (topic-map-associations map))))) (suffix (cond (occurrence-p "the-occurrence") (role-p "the-role") (t "the-association"))) (ids (mapcar (lambda (s) (concatenate 'string *document* "#" s)) (if (member name '("association-duplicate-iid2" "association-duplicate-iid3") :test #'equal) '("the-association" "the-association2") (list suffix))))) (check (format nil "item-identity-original:~A:exact-native-owner-identifiers-no-extra-topics" name) (and (= (if occurrence-p 2 3) (length (topic-map-topics map))) (= (if occurrence-p 1 0) (length (all-occurrences map))) (= (if occurrence-p 0 1) (length (topic-map-associations map))) (= (if occurrence-p 0 1) (length (as-roles map))) (same-identifiers-p ids (item-identifiers item)) (every (lambda (other) (or (eq other item) (null (item-identifiers other)))) (ik-items map)) (every (lambda (topic) (and (= 1 (length (item-identifiers topic))) (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic)) (null (topic-reified topic)) (null (topic-names topic)))) (topic-map-topics map)) (null (item-identifiers map)) (null (item-reifier map)) (ik-current-p map))) (check (format nil "item-identity-original:~A:exact-parent-type-player-scope-value" name) (if occurrence-p (and (eq (by-id map "topic") (occurrence-parent item)) (eq (by-id map "occtype") (occurrence-type item)) (equal "Testing resource data." (occurrence-value item)) (equal (string-datatype) (occurrence-datatype item)) (null (occurrence-scope item))) (let* ((a (first (topic-map-associations map))) (r (first (association-roles a)))) (and (eq map (association-parent a)) (eq a (association-role-parent r)) (eq (by-id map "assoctype") (association-type a)) (eq (by-id map "roletype") (association-role-type r)) (eq (by-id map "topic") (association-role-player r)) (null (association-scope a)) (equal (list r) (topic-roles-played (by-id map "topic"))))))) (check (format nil "item-identity-original:~A:each-locator-on-exactly-one-canonical-item" name) (every (lambda (id) (= 1 (count-if (lambda (i) (member id (item-identifiers i) :test #'equal)) (topic-maps.tmdm::map-information-items map)))) ids)))) (defun item-identity-original-contracts () (setf *item-identity-outputs* nil) (let ((previous (copy-list *cases*))) (dolist (name *item-identity-originals*) (let* ((id (format nil "xtm2/in/~A.xtm" name)) (*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id)) (raw (file-bytes (corpus-path id))) (copy (copy-seq raw)) (baseline (corpus-path (format nil "xtm2/baseline/~A.xtm.cxtm" name))) (expected (file-bytes baseline)) (case (topic-maps.test-foundation:make-test-case :id id :category :valid :source-pathname (corpus-path id) :document-iri *document* :expected-outcome :cxtm-equal :expected-baseline baseline :clauses '("XTM 4.3.3-4.3.4,4.6,4.15,4.18-4.19" "TMDM 5.1,5.6-5.8,6.5-6.7" "CXTM 3,4"))) (map nil) (held nil) (stages nil) (calls 0) (same nil) (actual nil) (c nil)) (check (format nil "item-identity-original:~A:new-case-once" name) (null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))) (setf *cases* (append *cases* (list case))) (handler-case (progn (multiple-value-setq (map held stages calls same) (ik-observe raw id)) (ik-original-model-checks name map) (check (format nil "item-identity-original:~A:real-qualified-parser-original-octets" name) (and (= 1 calls) same (equalp copy raw))) (check (format nil "item-identity-original:~A:all-observed-native-handles-live" name) (ik-held-p map held)) (when (search "duplicate" name) (check (format nil "item-identity-original:~A:actual-native-replacement-handles" name) (some (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held))) (setf actual (ik-pure name map)) (let ((other (deserialize-octets raw :document-iri *document* :source-key "different:item-identity-provenance"))) (check (format nil "item-identity-original:~A:source-key-independent-identities-and-bytes" name) (and (equalp (ik-model map) (ik-model other)) (equalp actual (mm-cxtm other)))))) (error (condition) (setf c condition))) (record-original-result case actual expected c) (push (mm-fields "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw) "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8) "qualifiedParserCalls" calls "originalOctetsEQ" (if same :true :false) "model" (when map (ik-model map)) "normalizationStages" (coerce (mapcar (lambda (s) (coerce s 'vector)) stages) 'vector) "observedNativeHandles" (length held) "replacedNativeHandles" (count-if (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held) "oldHandlesCanonicalAndLive" (if (and map (ik-held-p map held)) :true :false) "actualBytes" (when actual (length actual)) "expectedBytes" (length expected) "actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8)) "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8) "firstDifferentByte" (when actual (mismatch actual expected)) "condition" (mm-condition c) "status" (string-downcase (string (topic-maps.test-foundation:test-case-status case)))) *item-identity-outputs*) (when c (error c)) (check (format nil "item-identity-original:~A:complete-original-CXTM-byte-exact" name) (equalp actual expected)))) (check "item-identity-original:requested-membership-unique-all-prior-cases-preserved" (and (every (lambda (case) (and (member case *cases* :test #'eq) (eq :pass (topic-maps.test-foundation:test-case-status case)))) previous) (same-identifiers-p (mapcar #'topic-maps.test-foundation:test-case-id (set-difference *cases* previous :test #'eq)) (mapcar (lambda (n) (format nil "xtm2/in/~A.xtm" n)) *item-identity-originals*)) (every (lambda (case) (= 1 (count (topic-maps.test-foundation:test-case-id case) *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))) *cases*)))) (setf *item-identity-outputs* (nreverse *item-identity-outputs*))) (defun ik-ids (ids) (format nil "~{~}" ids)) (defun ik-occ (ids &key (type "#t") (value "V") scope) (format nil "~A~A~A" (ik-ids ids) type (if scope (format nil "~{~}" scope) "") value)) (defun ik-role (ids &optional (type "#r") (player "#p")) (format nil "~A" (ik-ids ids) type player)) (defun ik-assoc (ids roles &key (type "#t") scope) (format nil "~A~A~{~A~}" (ik-ids ids) type (if scope (format nil "~{~}" scope) "") roles)) (defun ik-probe (label body verify &optional replacement) (let* ((raw (bytes (document body))) (copy (copy-seq raw))) (multiple-value-bind (map held stages calls same) (ik-observe raw label) (check (format nil "item-identity:~A:native-EQ-parents-types-scopes-players-inverses" label) (and (= 1 calls) same (equalp copy raw) (ik-current-p map) (ik-held-p map held))) (funcall verify map) (when replacement (check (format nil "item-identity:~A:old-handles-forward-to-current-items" label) (some (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held))) (let ((cx (ik-pure label map))) (push (mm-fields "label" label "documentIRI" *document* "sourceKey" label "sourceText" (document body) "model" (ik-model map) "CXTM" (babel:octets-to-string cx :encoding :utf-8) "observedNativeHandles" (length held) "replacedNativeHandles" (count-if (lambda (i) (not (eq i (topic-maps.tmdm::canonical-item i)))) held) "oldHandlesCanonicalAndLive" :true "normalizationStages" (coerce (mapcar (lambda (s) (coerce s 'vector)) stages) 'vector)) *item-identity-controls*)) map))) (defun ik-only-ids (label kind ids count) (lambda (map) (let ((items (remove-if-not (lambda (i) (typep i kind)) (ik-items map)))) (check (format nil "item-identity:~A:exact-identifier-union-no-extra-topics" label) (and (= count (length items)) (= (if (eq kind 'occurrence) 2 3) (length (topic-map-topics map))) (every (lambda (i) (same-identifiers-p ids (item-identifiers i))) items)))))) (defun ik-error (label body class code &key resolver) ;; Preserve the native condition and show that each failing ID assignment is ;; checked before mutation. XTM/XML source contexts and causes remain intact. (let ((saved (symbol-function 'add-item-identifier)) (c nil) (failed-assignments nil) (raw (bytes (document body)))) (unwind-protect (progn (setf (symbol-function 'add-item-identifier) (lambda (item iri) (let* ((map (topic-maps.tmdm::item-topic-map item)) (before (ik-state map))) (handler-case (funcall saved item iri) (error (condition) (push (list (string (type-of item)) iri (equal before (ik-state map))) failed-assignments) (error condition)))))) (setf c (condition-from (lambda () (deserialize-octets raw :document-iri *document* :source-key label :resource-resolver resolver))))) (setf (symbol-function 'add-item-identifier) saved)) (let ((actual-code (cond ((typep c 'topic-maps.xtm:xtm-error) (error-code c)) ((typep c 'item-identifier-error) (item-identifier-error-code c))))) (check (format nil "item-identity-error:~A:exact-class-code-intended-cause" label) (and (eq (type-of c) class) (eq code actual-code))) (when (typep c 'topic-maps.xtm:xtm-error) (check (format nil "item-identity-error:~A:document-source-position-cause" label) (and (equal *document* (error-document-iri c)) (equal label (error-source-key c)) (if (typep c 'xtm-xml-error) (topic-maps.xtm:error-cause c) (and (integerp (error-line c)) (integerp (error-column c))))))) (when (typep c 'item-identifier-error) (check (format nil "item-identity-error:~A:failed-native-assignment-atomic-not-merged-by-II" label) (and failed-assignments (every #'third failed-assignments)))) (push (mm-fields "label" label "documentIRI" *document* "sourceKey" label "sourceText" (document body) "condition" (mm-condition c) "nativeIdentifierCode" (when (typep c 'item-identifier-error) (string-downcase (string actual-code))) "nativeConditionHasNoSourcePositionSlots" (if (typep c 'item-identifier-error) :true :false) "rejectedAssignments" (coerce (mapcar (lambda (x) (mm-fields "itemKind" (first x) "locator" (second x) "nativeAssignmentUnchanged" (if (third x) :true :false))) failed-assignments) 'vector)) *item-identity-controls*)) c)) (defun item-identity-tests () (setf *item-identity-controls* nil) (dolist (kind '(occurrence association association-role)) (dolist (row '(("one-II" ("urn:first")) ("multiple-and-repeated-II" ("urn:first" "urn:second" "urn:first")))) (let* ((label (format nil "~A:~A" kind (first row))) (ids (second row)) (body (ecase kind (occurrence (format nil "~A" (ik-occ ids))) (association (ik-assoc ids (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role ids))))))) (ik-probe label body (ik-only-ids label kind (remove-duplicates ids :test #'equal) 1)))) (let* ((label (format nil "~A:equal-items-different-II" kind)) (body (ecase kind (occurrence (format nil "~A~A" (ik-occ '("urn:first")) (ik-occ '("urn:second")))) (association (concatenate 'string (ik-assoc '("urn:first") (list (ik-role nil))) (ik-assoc '("urn:second") (list (ik-role nil))))) (association-role (ik-assoc nil (list (ik-role '("urn:first")) (ik-role '("urn:second")))))))) (ik-probe label body (ik-only-ids label kind '("urn:first" "urn:second") 1) t))) (dolist (kind '(occurrence association association-role)) (let* ((label (format nil "~A:relative-percent-Unicode-II" kind)) (ids '("../asset#one" "#two" "../%E9%A6%AC#three" "https://id.example/a#four")) (body (ecase kind (occurrence (format nil "~A" (ik-occ ids))) (association (ik-assoc ids (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role ids))))))) (ik-probe label body (ik-only-ids label kind (mapcar #'mm-child-iri ids) 1)))) ;; Corresponding Roles in duplicate Associations occur in reversed source ;; order; II must follow actual Type/Player, never list position. (ik-probe "role-II-match-by-signature-after-association-normalization" (concatenate 'string (ik-assoc '("urn:as1") (list (ik-role '("urn:r1") "#r" "#p") (ik-role '("urn:r2") "#rr" "#q"))) (ik-assoc '("urn:as2") (list (ik-role '("urn:r4") "#rr" "#q") (ik-role '("urn:r3") "#r" "#p")))) (lambda (map) (let ((a (first (topic-map-associations map)))) (check "item-identity:role-signature:correct-unions-parent-and-inverses" (and (= 1 (length (topic-map-associations map))) (= 2 (length (association-roles a))) (same-identifiers-p '("urn:as1" "urn:as2") (item-identifiers a)) (loop for player in '("p" "q") for ids in '(("urn:r1" "urn:r3") ("urn:r2" "urn:r4")) always (same-identifiers-p ids (item-identifiers (find (by-id map player) (association-roles a) :key #'association-role-player :test #'eq)))))))) t) (ik-late-merges) (ik-negative-tests) (ik-import-tests) (setf *item-identity-controls* (nreverse *item-identity-controls*))) (defun ik-late-merges () (dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity")) (let* ((label (concatenate 'string "late-" kind)) (body (concatenate 'string (format nil "~A~A" (ik-occ '("urn:o1") :type "#t" :scope '("#s")) (ik-occ '("urn:o2") :type "#u" :scope '("#v"))) (ik-assoc '("urn:a1") (list (ik-role '("urn:r1") "#r" "#p")) :type "#t" :scope '("#s")) (ik-assoc '("urn:a2") (list (ik-role '("urn:r2") "#rr" "#q")) :type "#u" :scope '("#v")) (format nil "<~A href='urn:owner'/><~A href='urn:owner'/><~A href='urn:type'/><~A href='urn:type'/><~A href='urn:role'/><~A href='urn:role'/><~A href='urn:scope'/><~A href='urn:scope'/>" kind kind kind kind kind kind kind kind)))) (ik-probe label body (lambda (map) (check (format nil "item-identity:~A:all-three-item-unions-and-native-relations" label) (and (= 4 (length (topic-map-topics map))) (= 1 (length (all-occurrences map))) (= 1 (length (topic-map-associations map))) (= 1 (length (as-roles map))) (same-identifiers-p '("urn:o1" "urn:o2") (item-identifiers (first (all-occurrences map)))) (same-identifiers-p '("urn:a1" "urn:a2") (item-identifiers (first (topic-map-associations map)))) (same-identifiers-p '("urn:r1" "urn:r2") (item-identifiers (first (as-roles map)))) (eq (by-id map "p") (by-id map "q")) (eq (by-id map "s") (by-id map "v"))))) t)))) (defun ik-negative-tests () (dolist (kind '(occurrence association association-role)) (dolist (bad '("%ZZ" "%C3%28" "a#b#c")) (let ((body (ecase kind (occurrence (format nil "~A" (ik-occ (list bad)))) (association (ik-assoc (list bad) (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role (list bad)))))))) (ik-error (format nil "~A:invalid-II:~A" kind bad) body 'xtm-rejection (cond ((equal bad "%ZZ") :invalid-percent-escape) ((equal bad "%C3%28") :invalid-percent-utf8) (t :invalid-iri))))) (let* ((label (format nil "~A:II-collides-with-map" kind)) (body (ecase kind (occurrence (format nil "~A" (ik-occ '("urn:collision")))) (association (ik-assoc '("urn:collision") (list (ik-role nil)))) (association-role (ik-assoc nil (list (ik-role '("urn:collision")))))))) (ik-error label (concatenate 'string "" body) 'item-identifier-error :duplicate-item-identifier))) (ik-error "different-occurrences-same-II" (format nil "~A~A" (ik-occ '("urn:collision")) (ik-occ '("urn:collision") :value "different")) 'item-identifier-error :duplicate-item-identifier) (ik-error "different-associations-same-II" (concatenate 'string (ik-assoc '("urn:collision") (list (ik-role nil))) (ik-assoc '("urn:collision") (list (ik-role nil)) :type "#other")) 'item-identifier-error :duplicate-item-identifier) (ik-error "different-roles-same-II" (ik-assoc nil (list (ik-role '("urn:collision") "#r" "#p") (ik-role '("urn:collision") "#r" "#q"))) 'item-identifier-error :duplicate-item-identifier) (ik-error "same-role-signature-different-parents-same-II" (concatenate 'string (ik-assoc nil (list (ik-role '("urn:collision")))) (ik-assoc nil (list (ik-role '("urn:collision"))) :type "#other")) 'item-identifier-error :duplicate-item-identifier) (ik-error "association-role-cross-type-collision" (ik-assoc '("urn:collision") (list (ik-role '("urn:collision")))) 'item-identifier-error :duplicate-item-identifier) (ik-error "occurrence-association-cross-type-collision" (concatenate 'string (format nil "~A" (ik-occ '("urn:collision"))) (ik-assoc '("urn:collision") (list (ik-role nil)))) 'item-identifier-error :duplicate-item-identifier) (ik-error "II-stays-conflict-after-topic-merging" (concatenate 'string (ik-assoc '("urn:collision") (list (ik-role nil "#r" "#p"))) (ik-assoc '("urn:other") (list (ik-role nil "#r" "#q"))) "" (ik-assoc '("urn:collision") (list (ik-role nil "#r" "#other")))) 'item-identifier-error :duplicate-item-identifier) (dolist (row '(("occurrence-II-after-type" "V" :element-order) ("association-II-after-type" "" :element-order) ("role-II-after-type" "" :element-order) ("II-in-type" "" :unexpected-element) ("II-in-scope" "" :unexpected-element) ("II-not-empty" "" :unexpected-child) ("II-missing-href" "" :missing-attribute))) (ik-error (first row) (second row) 'xtm-rejection (third row))) (ik-error "malformed-XML" "" 'xtm-xml-error :xml-not-well-formed) (ik-error "occurrence-reification-remains-unsupported" "V" 'xtm-unsupported :resource-data-datatype) ;; Explicitly excluded original: preserve XML-ID uniqueness, no registration. (let* ((id "xtm2/in/occurrence-duplicate-iid2.xtm") (raw (file-bytes (corpus-path id))) (c (condition-from (lambda () (deserialize-octets raw :document-iri *document* :source-key id))))) (check "item-identity:excluded-duplicate-XML-ID-not-registered" (and (eq (type-of c) 'xtm-rejection) (eq :duplicate-id (error-code c)) (null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))) (push (mm-fields "label" "excluded-duplicate-XML-ID" "id" id "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8) "condition" (mm-condition c) "registered" :false) *item-identity-controls*))) (defun ik-import-tests () (let* ((child (mm-child-iri "child.xtm")) (raw (bytes (document (concatenate 'string (format nil "~A" (ik-occ '("urn:child-occurrence"))) (ik-assoc '("urn:child-association") (list (ik-role '("urn:child-role"))))))))) (mm-call "item-identities-native-import" (document (mm-directive "child.xtm")) (list (list child raw "identifier-child")) (lambda (map imports calls) (declare (ignore calls)) (check "item-identity:import:all-three-identifier-sets-native-EQ-source-preserved" (and (ik-current-p map) (equal '("urn:child-occurrence") (item-identifiers (first (all-occurrences map)))) (equal '("urn:child-association") (item-identifiers (first (topic-map-associations map)))) (equal '("urn:child-role") (item-identifiers (first (as-roles map)))) (every (lambda (record) (every (lambda (item) (same-identifiers-p (item-identifiers item) (item-identifiers (funcall (second record) item)))) (ik-items (first record)))) imports)))))) (let* ((child (mm-child-iri "child.xtm")) (raw (bytes (document (ik-assoc '("urn:map-collision") (list (ik-role nil)))))) (body (concatenate 'string "" (mm-directive "child.xtm"))) (c (ik-error "import-II-conflict-cause" body 'topic-maps.xtm:xtm-resource-error :mergemap-import-failure :resolver (lambda (&rest args) (declare (ignore args)) (values raw "conflicting-child"))))) (declare (ignore child)) (check "item-identity:import-conflict:original-native-Cause-retained" (and (typep (topic-maps.xtm:error-cause c) 'item-identifier-error) (eq :duplicate-item-identifier (item-identifier-error-code (topic-maps.xtm:error-cause c)))))))