(in-package #:topic-maps.xtm.tests) (defvar *occurrence-outputs* nil) (defvar *occurrence-controls* nil) (defparameter *occurrence-originals* '("occurrence" "occurrence-datatype-unknown" "occurrence-duplicate" "occurrence-scope" "occurrence-scope-duplicate" "occurrence-scope-duplicate-merged" "occurrence-resourcedata-uri" "occurrence-resourcedata-uri-relative" "occurrence-resourceref" "occurrence-resourceref-relative" "merge-itemid-with-occurrences")) (defun all-occurrences (map) (loop for topic in (topic-map-topics map) append (topic-occurrences topic))) (defun occurrence-current-p (map) (and (mm-current-p map) (variant-reifier-current-p map) (every (lambda (o) (and (eq o (canonical-occurrence o)) (eq (occurrence-parent o) (canonical-topic (occurrence-parent o))) (eq (occurrence-type o) (canonical-topic (occurrence-type o))) (member o (topic-occurrences (occurrence-parent o)) :test #'eq) (= (length (occurrence-scope o)) (length (remove-duplicates (occurrence-scope o) :test #'eq))) (every (lambda (theme) (eq theme (canonical-topic theme))) (occurrence-scope o)))) (all-occurrences map)))) (defun occurrence-model (map) (let ((topics (topic-map-topics map))) (labels ((ref (topic) (1+ (position topic topics :test #'eq)))) (mm-fields "existingCharacteristics" (variant-reifier-model-data map) "canonicalOccurrenceReferencesEQ" (if (occurrence-current-p map) :true :false) "occurrenceCount" (length (all-occurrences map)) "occurrences" (coerce (loop for o in (all-occurrences map) collect (mm-fields "parent" (ref (occurrence-parent o)) "type" (ref (occurrence-type o)) "scope" (coerce (mapcar #'ref (occurrence-scope o)) 'vector) "value" (occurrence-value o) "datatype" (occurrence-datatype o) "itemIdentifiers" (coerce (item-identifiers o) 'vector) "reifier" (when (item-reifier o) (ref (item-reifier o))))) 'vector))))) (defun occurrence-pure (label map) (let* ((before (mm-state map)) (held (topic-maps.tmdm::map-information-items map)) (actual (mm-cxtm map))) (check (concatenate 'string "occurrence:" label ":pure-deterministic-cxtm-and-handles") (and (equalp actual (mm-cxtm map)) (equal before (mm-state map)) (occurrence-current-p map) (every (lambda (i) (eq i (topic-maps.tmdm::canonical-item i))) held))) actual)) (defun occurrence-condition (c) (when c (let ((data (mm-condition c))) (when (typep c 'topic-maps.tmdm::occurrence-error) (setf (cdr (assoc "code" (topic-maps.test-foundation::json-object-members data) :test #'equal)) (string-downcase (string (topic-maps.tmdm::occurrence-error-code c))))) data))) (defun occurrence-observe (raw key) ;; Observe real productive calls and retained native handles; return unchanged. (let ((parser (symbol-function 'topic-maps.xml:parse-octets)) (reader (symbol-function 'topic-maps.xtm::read-occurrence)) (calls 0) (same nil) (held nil) (counts nil) (map nil)) (unwind-protect (progn (setf (symbol-function 'topic-maps.xml:parse-octets) (lambda (input &rest args) (incf calls) (setf same (eq input raw)) (apply parser input args))) (setf (symbol-function 'topic-maps.xtm::read-occurrence) (lambda (node owner model) (let ((o (funcall reader node owner model))) (push o held) (push (length (all-occurrences model)) counts) o))) (setf map (deserialize-octets raw :document-iri *document* :source-key key))) (setf (symbol-function 'topic-maps.xml:parse-octets) parser (symbol-function 'topic-maps.xtm::read-occurrence) reader)) (values map (nreverse held) (nreverse counts) calls same))) (defun occurrence-handle-evidence (map held) (mm-fields "readHandleCount" (length held) "distinctReadHandles" (length (remove-duplicates held :test #'eq)) "canonicalGroupsInFinalMap" (coerce (mapcar (lambda (old) (1+ (position (canonical-occurrence old) (all-occurrences map) :test #'eq))) held) 'vector) "replacedReadHandles" (coerce (mapcar (lambda (old) (if (eq old (canonical-occurrence old)) :false :true)) held) 'vector))) (defun occurrence-original-model-checks (name map) (let* ((owner (by-id map "topic")) (type (by-id map "occtype")) (scoped (search "scope" name)) (merged (equal name "merge-itemid-with-occurrences")) (uri (or (search "resourceref" name) (search "resourcedata-uri" name))) (dtype (cond (uri (iri-datatype)) ((equal name "occurrence-datatype-unknown") "http://example.com/bogus") (t (string-datatype)))) (values (if merged '("Some data" "Some more data") (list (cond ((search "relative" name) (mm-child-iri "photo.jpg")) (uri "http://www.isotopicmaps.org/cxtm") ((equal name "occurrence-datatype-unknown") "BOGUS") (t "Testing resource data.")))))) (check (concatenate 'string "occurrence-original:" name ":exact-native-properties") (and (= (if scoped 3 2) (length (topic-map-topics map))) (= (if merged 2 1) (length (all-occurrences map))) (null (topic-map-associations map)) (null (name-reifier-names map)) (null (all-variants map)) (every (lambda (o) (and (eq owner (occurrence-parent o)) (eq type (occurrence-type o)) (scope-eq-p (occurrence-scope o) (when scoped (list (by-id map "english")))) (equal dtype (occurrence-datatype o)) (member (occurrence-value o) values :test #'equal) (null (item-identifiers o)) (null (item-reifier o)))) (all-occurrences map)) (same-identifiers-p values (mapcar #'occurrence-value (all-occurrences map))) (equal (list (concatenate 'string *document* "#occtype")) (item-identifiers type)) (same-identifiers-p (item-identifiers owner) (append (list (concatenate 'string *document* "#topic")) (when merged (list (concatenate 'string *document* "#topic2") "http://example.org/#topic")))) (null (item-identifiers map)) (null (item-reifier map)) (every (lambda (topic) (and (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic)) (null (topic-reified topic)))) (topic-map-topics map)) (occurrence-current-p map))) (when (equal name "occurrence-scope-duplicate-merged") (check "occurrence-original:scope-merged:retained-II-and-EQ-theme" (and (eq (by-id map "en") (by-id map "english")) (same-identifiers-p (item-identifiers (by-id map "en")) (list (concatenate 'string *document* "#en") (concatenate 'string *document* "#english")))))))) (defun occurrence-original-contracts () (setf *occurrence-outputs* nil) (dolist (name *occurrence-originals*) (let* ((id (format nil "xtm2/in/~A.xtm" name)) (*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id)) (source (corpus-path id)) (baseline (corpus-path (format nil "xtm2/baseline/~A.xtm.cxtm" name))) (raw (file-bytes source)) (copy (copy-seq raw)) (expected (file-bytes baseline)) (case (topic-maps.test-foundation:make-test-case :id id :category :valid :source-pathname source :document-iri *document* :expected-outcome :cxtm-equal :expected-baseline baseline :clauses '("XTM 2.0 4.13-4.17,4.20" "TMDM 5.6,6.2,6.5" "CXTM 3,4"))) (map nil) (actual nil) (c nil) (held nil) (counts nil) (calls 0) (same nil)) (check (concatenate 'string "occurrence-original:" name ":not-already-registered") (null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))) (setf *cases* (append *cases* (list case))) (handler-case (progn (multiple-value-setq (map held counts calls same) (occurrence-observe raw id)) (occurrence-original-model-checks name map) (check (concatenate 'string "occurrence-original:" name ":original-octets-qualified-parser") (and (= 1 calls) same (equalp copy raw))) (check (concatenate 'string "occurrence-original:" name ":old-handles-canonical-and-live") (every (lambda (old) (member (canonical-occurrence old) (all-occurrences map) :test #'eq)) held)) (when (member name '("occurrence-duplicate" "occurrence-scope-duplicate-merged") :test #'equal) (check (concatenate 'string "occurrence-original:" name ":distinct-old-handles-normalized") (and (= 2 (length held)) (not (eq (first held) (second held))) (eq (canonical-occurrence (first held)) (canonical-occurrence (second held))) (equal counts (if (equal name "occurrence-duplicate") '(1 1) '(1 2)))))) (setf actual (occurrence-pure name map)) (let ((other (deserialize-octets raw :document-iri *document* :source-key "independent:occurrence-original"))) (check (concatenate 'string "occurrence-original:" name ":source-key-independent") (and (equalp (occurrence-model map) (occurrence-model other)) (equalp actual (mm-cxtm other)))))) (error (condition) (setf c condition))) (record-original-result case actual expected c) (push (mm-fields "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw) "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8) "qualifiedParserCalls" calls "originalOctetsEQ" (if same :true :false) "model" (when map (occurrence-model map)) "occurrenceCountsAfterEachRead" (coerce counts 'vector) "retainedHandleEvidence" (when map (occurrence-handle-evidence map held)) "retainedReadHandles" (length held) "oldHandlesCanonicalAndLive" (if (and map (every (lambda (o) (member (canonical-occurrence o) (all-occurrences map) :test #'eq)) held)) :true :false) "actualBytes" (when actual (length actual)) "expectedBytes" (length expected) "actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8)) "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8) "firstDifferentByte" (when actual (mismatch actual expected)) "condition" (occurrence-condition c) "status" (string-downcase (string (topic-maps.test-foundation:test-case-status case)))) *occurrence-outputs*) (when c (error c)) (check (concatenate 'string "occurrence-original:" name ":complete-original-CXTM-byte-exact") (equalp actual expected)))) (setf *occurrence-outputs* (nreverse *occurrence-outputs*)) (check "occurrence-originals:83-unique-cases-75-positive-eight-negative" (and (= 83 (length *cases*)) (= 75 (count :valid *cases* :key #'topic-maps.test-foundation:test-case-category)) (= 83 (length (remove-duplicates *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))))) (defun occurrence-xml (resource &key (type "t") scope) (format nil "~A~A" type (if scope (scope-xml scope) "") resource)) (defun occurrence-control (label body verify) (let* ((raw (bytes (document body))) (copy (copy-seq raw)) (map nil) (held nil) (counts nil) (calls nil) (same nil)) (multiple-value-setq (map held counts calls same) (occurrence-observe raw label)) (funcall verify map held counts) (let ((actual (occurrence-pure label map))) (check (concatenate 'string "occurrence:" label ":qualified-parser-source-preserved") (and same (= 1 calls) (equalp raw copy))) (push (mm-fields "label" label "sourceText" (document body) "model" (occurrence-model map) "actualCXTM" (babel:octets-to-string actual :encoding :utf-8) "condition" nil "occurrenceCountsAfterEachRead" (coerce counts 'vector) "retainedReadHandles" (length held) "retainedHandleEvidence" (occurrence-handle-evidence map held)) *occurrence-controls*)))) (defun occurrence-value-controls () (dolist (row (list (list "default" " V & 馬🦊 " " V & 馬🦊 " (string-datatype)) (list "whitespace" " A " (format nil " ~CA~% " #\Tab) (string-datatype)) (list "empty" "" "" (string-datatype)) (list "custom" "literal%GG" "literal%GG" "urn:custom") (list "integer-lexical" (datatype-resource (integer-datatype) "+00042") "+00042" (integer-datatype)) (list "uri-relative" (datatype-resource (iri-datatype) "../photo") (mm-child-iri "../photo") (iri-datatype)) (list "ref-relative" "" (mm-child-iri "../photo") (iri-datatype)) (list "ref-absolute" "" "urn:photo" (iri-datatype)))) (destructuring-bind (label resource value dtype) row (occurrence-control label (format nil "~A" (occurrence-xml resource :scope '("s" "s"))) (lambda (map held counts) (declare (ignore held counts)) (let ((o (first (all-occurrences map)))) (check (concatenate 'string "occurrence:" label ":exact-value-datatype-forward-references") (and (= 3 (length (topic-map-topics map))) (= 1 (length (all-occurrences map))) (eq (by-id map "a") (occurrence-parent o)) (eq (by-id map "t") (occurrence-type o)) (scope-eq-p (list (by-id map "s")) (occurrence-scope o)) (equal value (occurrence-value o)) (equal dtype (occurrence-datatype o))))))))) (let* ((resource "V") (base (occurrence-xml resource :scope '("s")))) (occurrence-control "equality-discriminators" (format nil "~A~A~A~A~A~A~A" base base (occurrence-xml "W" :scope '("s")) (occurrence-xml "V" :scope '("s")) (occurrence-xml resource :type "u" :scope '("s")) (occurrence-xml resource :scope '("z")) base) (lambda (map held counts) (declare (ignore counts)) (check "occurrence:equality-complete-parent-type-scope-value-datatype" (and (= 6 (length (all-occurrences map))) (= 5 (length (topic-occurrences (by-id map "a")))) (= 1 (length (topic-occurrences (by-id map "b")))) (= 7 (length held)) (not (eq (first held) (second held))) (eq (canonical-occurrence (first held)) (canonical-occurrence (second held))))))))) (defun occurrence-merge-controls () (dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity")) (dolist (dimension '("parent" "type" "scope")) (let* ((resource "V") (a (if (equal dimension "parent") "a" "x")) (b (if (equal dimension "parent") "b" "y")) (first (occurrence-xml resource :type (if (equal dimension "type") "x" "t") :scope (if (equal dimension "scope") '("x") '("s")))) (second (occurrence-xml resource :type (if (equal dimension "type") "y" "t") :scope (if (equal dimension "scope") '("y") '("s")))) (body (concatenate 'string (if (equal dimension "parent") (format nil "~A~A" first second) (format nil "~A~A" first second)) (format nil "<~A href='urn:shared'/><~A href='urn:shared'/>" a kind b kind)))) (occurrence-control (format nil "merge-~A-~A" dimension kind) body (lambda (map held counts) (check (format nil "occurrence:merge-~A-~A:two-before-one-after-old-handles-EQ" dimension kind) (and (equal '(1 2) counts) (= 1 (length (all-occurrences map))) (= 2 (length held)) (not (eq (first held) (second held))) (eq (canonical-occurrence (first held)) (canonical-occurrence (second held))) (not (eq (first held) (canonical-occurrence (first held)))) (not (eq (second held) (canonical-occurrence (second held)))) (eq (by-id map a) (by-id map b)) (eq (by-id map "alias1") (by-id map "alias2"))))))))) (occurrence-control "other-characteristics-preserved" (format nil "NV~A" (occurrence-xml "O")) (lambda (map held counts) (declare (ignore held counts)) (check "occurrence:existing-names-variants-core-associations-preserved" (and (= 1 (length (all-occurrences map))) (= 1 (length (name-reifier-names map))) (= 1 (length (all-variants map))) (= 1 (length (topic-map-associations map))) (= 2 (length (association-roles (first (topic-map-associations map))))) (equal "N" (topic-name-value (first (topic-names (by-id map "a"))))) (equal "V" (variant-value (first (all-variants map))))))))) (defun occurrence-negative-controls () (dolist (spec '(("missing-type" "V" xtm-rejection :missing-occurrence-type) ("missing-resource" "" xtm-rejection :missing-occurrence-resource) ("two-resources" "V" xtm-rejection :occurrence-resource-cardinality) ("two-types" "V" xtm-rejection :duplicate-occurrence-type) ("empty-type" "V" xtm-rejection :type-cardinality) ("two-type-refs" "V" xtm-rejection :type-cardinality) ("bad-type-ref" "V" xtm-rejection :invalid-topic-ref) ("empty-scope" "V" xtm-rejection :empty-scope) ("bad-scope-ref" "V" xtm-rejection :invalid-topic-ref) ("two-scopes" "V" xtm-rejection :duplicate-occurrence-scope) ("wrong-order" "V" xtm-rejection :element-order) ("early-scope" "V" xtm-rejection :missing-occurrence-type) ("bad-child" "" xtm-rejection :unexpected-element) ("foreign-namespace" "" xtm-rejection :invalid-namespace) ("bad-attribute" "" xtm-rejection :unexpected-attribute) ("text-child" "unexpected" xtm-rejection :unexpected-text) ("relative-datatype" "V" xtm-rejection :invalid-resource-data-datatype) ("embedded-without-anyType" "" xtm-rejection :unexpected-child) ("anyType-text" "V" xtm-unsupported :resource-resolver-required) ("anyType-XML" "" xtm-unsupported :resource-resolver-required) ("item-identity" "V" xtm-unsupported :resource-resolver-required) ("reification" "V" xtm-unsupported :resource-resolver-required))) (destructuring-bind (label body class code) spec (let* ((key (concatenate 'string "occurrence-negative:" label)) (source (ss-unsupported-source (document (format nil "~A" body)) code label)) (c (condition-from (lambda () (deserialize-octets (bytes source) :document-iri *document* :source-key key))))) (check (concatenate 'string key ":actual-class-code-context-position") (and (eq (type-of c) class) (eq code (error-code c)) (equal *document* (error-document-iri c)) (equal key (error-source-key c)) (plusp (error-line c)) (plusp (error-column c)))) (push (mm-fields "label" key "sourceText" source "condition" (occurrence-condition c)) *occurrence-controls*)))) (dolist (body '("" "N")) (let ((c (condition-from (lambda () (read-body body))))) (check (concatenate 'string "occurrence-negative:invalid-parent-context:" body) (and (typep c 'xtm-rejection) (eq :unexpected-element (error-code c)))) (push (mm-fields "label" "invalid-parent-context" "sourceText" (document body) "condition" (occurrence-condition c)) *occurrence-controls*))) (let ((c (condition-from (lambda () (deserialize-octets (bytes "") :document-iri *document* :source-key "occurrence:XML"))))) (check "occurrence-negative:real-XML-error" (and (typep c 'xtm-xml-error) (eq :xml-not-well-formed (error-code c)))) (push (mm-fields "label" "real-XML-error" "sourceText" "" "condition" (occurrence-condition c)) *occurrence-controls*)) (dolist (body (list (format nil "~A" (occurrence-xml (datatype-resource (integer-datatype) "bad"))) (format nil "~A" (occurrence-xml "V")))) (let ((c (condition-from (lambda () (read-body body))))) (check (if (search "bad" body) "occurrence-negative:native-invalid-integer" "occurrence-negative:native-type-identifier-conflict") (if (search "bad" body) (and (typep c 'topic-maps.tmdm::occurrence-error) (eq :invalid-integer-value (topic-maps.tmdm::occurrence-error-code c))) (and (typep c 'item-identifier-error) (eq :duplicate-item-identifier (item-identifier-error-code c))))) (push (mm-fields "label" "occurrence-native-conflict" "sourceText" (document body) "condition" (occurrence-condition c)) *occurrence-controls*)))) (defun occurrence-tests () (setf *occurrence-controls* nil) (occurrence-value-controls) (occurrence-merge-controls) (occurrence-negative-controls) (setf *occurrence-controls* (nreverse *occurrence-controls*)))