(in-package #:topic-maps.xtm.tests)
(defvar *occurrence-outputs* nil)
(defvar *occurrence-controls* nil)
(defparameter *occurrence-originals*
'("occurrence" "occurrence-datatype-unknown" "occurrence-duplicate"
"occurrence-scope" "occurrence-scope-duplicate" "occurrence-scope-duplicate-merged"
"occurrence-resourcedata-uri" "occurrence-resourcedata-uri-relative"
"occurrence-resourceref" "occurrence-resourceref-relative" "merge-itemid-with-occurrences"))
(defun all-occurrences (map)
(loop for topic in (topic-map-topics map) append (topic-occurrences topic)))
(defun occurrence-current-p (map)
(and (mm-current-p map) (variant-reifier-current-p map)
(every (lambda (o)
(and (eq o (canonical-occurrence o))
(eq (occurrence-parent o) (canonical-topic (occurrence-parent o)))
(eq (occurrence-type o) (canonical-topic (occurrence-type o)))
(member o (topic-occurrences (occurrence-parent o)) :test #'eq)
(= (length (occurrence-scope o)) (length (remove-duplicates (occurrence-scope o) :test #'eq)))
(every (lambda (theme) (eq theme (canonical-topic theme))) (occurrence-scope o))))
(all-occurrences map))))
(defun occurrence-model (map)
(let ((topics (topic-map-topics map)))
(labels ((ref (topic) (1+ (position topic topics :test #'eq))))
(mm-fields "existingCharacteristics" (variant-reifier-model-data map)
"canonicalOccurrenceReferencesEQ" (if (occurrence-current-p map) :true :false)
"occurrenceCount" (length (all-occurrences map))
"occurrences" (coerce (loop for o in (all-occurrences map) collect
(mm-fields "parent" (ref (occurrence-parent o)) "type" (ref (occurrence-type o))
"scope" (coerce (mapcar #'ref (occurrence-scope o)) 'vector)
"value" (occurrence-value o) "datatype" (occurrence-datatype o)
"itemIdentifiers" (coerce (item-identifiers o) 'vector)
"reifier" (when (item-reifier o) (ref (item-reifier o))))) 'vector)))))
(defun occurrence-pure (label map)
(let* ((before (mm-state map)) (held (topic-maps.tmdm::map-information-items map))
(actual (mm-cxtm map)))
(check (concatenate 'string "occurrence:" label ":pure-deterministic-cxtm-and-handles")
(and (equalp actual (mm-cxtm map)) (equal before (mm-state map))
(occurrence-current-p map) (every (lambda (i) (eq i (topic-maps.tmdm::canonical-item i))) held)))
actual))
(defun occurrence-condition (c)
(when c
(let ((data (mm-condition c)))
(when (typep c 'topic-maps.tmdm::occurrence-error)
(setf (cdr (assoc "code" (topic-maps.test-foundation::json-object-members data) :test #'equal))
(string-downcase (string (topic-maps.tmdm::occurrence-error-code c)))))
data)))
(defun occurrence-observe (raw key)
;; Observe real productive calls and retained native handles; return unchanged.
(let ((parser (symbol-function 'topic-maps.xml:parse-octets))
(reader (symbol-function 'topic-maps.xtm::read-occurrence))
(calls 0) (same nil) (held nil) (counts nil) (map nil))
(unwind-protect
(progn
(setf (symbol-function 'topic-maps.xml:parse-octets)
(lambda (input &rest args) (incf calls) (setf same (eq input raw)) (apply parser input args)))
(setf (symbol-function 'topic-maps.xtm::read-occurrence)
(lambda (node owner model)
(let ((o (funcall reader node owner model)))
(push o held) (push (length (all-occurrences model)) counts) o)))
(setf map (deserialize-octets raw :document-iri *document* :source-key key)))
(setf (symbol-function 'topic-maps.xml:parse-octets) parser
(symbol-function 'topic-maps.xtm::read-occurrence) reader))
(values map (nreverse held) (nreverse counts) calls same)))
(defun occurrence-handle-evidence (map held)
(mm-fields "readHandleCount" (length held)
"distinctReadHandles" (length (remove-duplicates held :test #'eq))
"canonicalGroupsInFinalMap" (coerce (mapcar (lambda (old)
(1+ (position (canonical-occurrence old) (all-occurrences map) :test #'eq))) held) 'vector)
"replacedReadHandles" (coerce (mapcar (lambda (old)
(if (eq old (canonical-occurrence old)) :false :true)) held) 'vector)))
(defun occurrence-original-model-checks (name map)
(let* ((owner (by-id map "topic")) (type (by-id map "occtype"))
(scoped (search "scope" name)) (merged (equal name "merge-itemid-with-occurrences"))
(uri (or (search "resourceref" name) (search "resourcedata-uri" name)))
(dtype (cond (uri (iri-datatype)) ((equal name "occurrence-datatype-unknown") "http://example.com/bogus") (t (string-datatype))))
(values (if merged '("Some data" "Some more data")
(list (cond ((search "relative" name) (mm-child-iri "photo.jpg"))
(uri "http://www.isotopicmaps.org/cxtm")
((equal name "occurrence-datatype-unknown") "BOGUS") (t "Testing resource data."))))))
(check (concatenate 'string "occurrence-original:" name ":exact-native-properties")
(and (= (if scoped 3 2) (length (topic-map-topics map)))
(= (if merged 2 1) (length (all-occurrences map)))
(null (topic-map-associations map)) (null (name-reifier-names map)) (null (all-variants map))
(every (lambda (o) (and (eq owner (occurrence-parent o)) (eq type (occurrence-type o))
(scope-eq-p (occurrence-scope o) (when scoped (list (by-id map "english"))))
(equal dtype (occurrence-datatype o)) (member (occurrence-value o) values :test #'equal)
(null (item-identifiers o)) (null (item-reifier o)))) (all-occurrences map))
(same-identifiers-p values (mapcar #'occurrence-value (all-occurrences map)))
(equal (list (concatenate 'string *document* "#occtype")) (item-identifiers type))
(same-identifiers-p (item-identifiers owner)
(append (list (concatenate 'string *document* "#topic"))
(when merged (list (concatenate 'string *document* "#topic2") "http://example.org/#topic"))))
(null (item-identifiers map)) (null (item-reifier map))
(every (lambda (topic) (and (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic))
(null (topic-reified topic)))) (topic-map-topics map)) (occurrence-current-p map)))
(when (equal name "occurrence-scope-duplicate-merged")
(check "occurrence-original:scope-merged:retained-II-and-EQ-theme"
(and (eq (by-id map "en") (by-id map "english"))
(same-identifiers-p (item-identifiers (by-id map "en"))
(list (concatenate 'string *document* "#en") (concatenate 'string *document* "#english"))))))))
(defun occurrence-original-contracts ()
(setf *occurrence-outputs* nil)
(dolist (name *occurrence-originals*)
(let* ((id (format nil "xtm2/in/~A.xtm" name))
(*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id))
(source (corpus-path id)) (baseline (corpus-path (format nil "xtm2/baseline/~A.xtm.cxtm" name)))
(raw (file-bytes source)) (copy (copy-seq raw)) (expected (file-bytes baseline))
(case (topic-maps.test-foundation:make-test-case :id id :category :valid :source-pathname source
:document-iri *document* :expected-outcome :cxtm-equal :expected-baseline baseline
:clauses '("XTM 2.0 4.13-4.17,4.20" "TMDM 5.6,6.2,6.5" "CXTM 3,4")))
(map nil) (actual nil) (c nil) (held nil) (counts nil) (calls 0) (same nil))
(check (concatenate 'string "occurrence-original:" name ":not-already-registered")
(null (find id *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))
(setf *cases* (append *cases* (list case)))
(handler-case
(progn
(multiple-value-setq (map held counts calls same) (occurrence-observe raw id))
(occurrence-original-model-checks name map)
(check (concatenate 'string "occurrence-original:" name ":original-octets-qualified-parser")
(and (= 1 calls) same (equalp copy raw)))
(check (concatenate 'string "occurrence-original:" name ":old-handles-canonical-and-live")
(every (lambda (old) (member (canonical-occurrence old) (all-occurrences map) :test #'eq)) held))
(when (member name '("occurrence-duplicate" "occurrence-scope-duplicate-merged") :test #'equal)
(check (concatenate 'string "occurrence-original:" name ":distinct-old-handles-normalized")
(and (= 2 (length held)) (not (eq (first held) (second held)))
(eq (canonical-occurrence (first held)) (canonical-occurrence (second held)))
(equal counts (if (equal name "occurrence-duplicate") '(1 1) '(1 2))))))
(setf actual (occurrence-pure name map))
(let ((other (deserialize-octets raw :document-iri *document* :source-key "independent:occurrence-original")))
(check (concatenate 'string "occurrence-original:" name ":source-key-independent")
(and (equalp (occurrence-model map) (occurrence-model other)) (equalp actual (mm-cxtm other))))))
(error (condition) (setf c condition)))
(record-original-result case actual expected c)
(push (mm-fields "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw)
"sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8)
"qualifiedParserCalls" calls "originalOctetsEQ" (if same :true :false)
"model" (when map (occurrence-model map)) "occurrenceCountsAfterEachRead" (coerce counts 'vector)
"retainedHandleEvidence" (when map (occurrence-handle-evidence map held))
"retainedReadHandles" (length held) "oldHandlesCanonicalAndLive" (if (and map (every (lambda (o) (member (canonical-occurrence o) (all-occurrences map) :test #'eq)) held)) :true :false)
"actualBytes" (when actual (length actual)) "expectedBytes" (length expected)
"actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8)) "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8)
"firstDifferentByte" (when actual (mismatch actual expected)) "condition" (occurrence-condition c)
"status" (string-downcase (string (topic-maps.test-foundation:test-case-status case)))) *occurrence-outputs*)
(when c (error c))
(check (concatenate 'string "occurrence-original:" name ":complete-original-CXTM-byte-exact") (equalp actual expected))))
(setf *occurrence-outputs* (nreverse *occurrence-outputs*))
(check "occurrence-originals:83-unique-cases-75-positive-eight-negative"
(and (= 83 (length *cases*)) (= 75 (count :valid *cases* :key #'topic-maps.test-foundation:test-case-category))
(= 83 (length (remove-duplicates *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))))))
(defun occurrence-xml (resource &key (type "t") scope)
(format nil "~A~A" type
(if scope (scope-xml scope) "") resource))
(defun occurrence-control (label body verify)
(let* ((raw (bytes (document body))) (copy (copy-seq raw)) (map nil) (held nil) (counts nil) (calls nil) (same nil))
(multiple-value-setq (map held counts calls same) (occurrence-observe raw label))
(funcall verify map held counts)
(let ((actual (occurrence-pure label map)))
(check (concatenate 'string "occurrence:" label ":qualified-parser-source-preserved")
(and same (= 1 calls) (equalp raw copy)))
(push (mm-fields "label" label "sourceText" (document body) "model" (occurrence-model map)
"actualCXTM" (babel:octets-to-string actual :encoding :utf-8) "condition" nil
"occurrenceCountsAfterEachRead" (coerce counts 'vector) "retainedReadHandles" (length held)
"retainedHandleEvidence" (occurrence-handle-evidence map held)) *occurrence-controls*))))
(defun occurrence-value-controls ()
(dolist (row (list (list "default" " V & 馬🦊 " " V & 馬🦊 " (string-datatype))
(list "whitespace" " A
" (format nil " ~CA~% " #\Tab) (string-datatype))
(list "empty" "" "" (string-datatype))
(list "custom" "literal%GG" "literal%GG" "urn:custom")
(list "integer-lexical" (datatype-resource (integer-datatype) "+00042") "+00042" (integer-datatype))
(list "uri-relative" (datatype-resource (iri-datatype) "../photo") (mm-child-iri "../photo") (iri-datatype))
(list "ref-relative" "" (mm-child-iri "../photo") (iri-datatype))
(list "ref-absolute" "" "urn:photo" (iri-datatype))))
(destructuring-bind (label resource value dtype) row
(occurrence-control label (format nil "~A"
(occurrence-xml resource :scope '("s" "s")))
(lambda (map held counts) (declare (ignore held counts))
(let ((o (first (all-occurrences map))))
(check (concatenate 'string "occurrence:" label ":exact-value-datatype-forward-references")
(and (= 3 (length (topic-map-topics map))) (= 1 (length (all-occurrences map)))
(eq (by-id map "a") (occurrence-parent o)) (eq (by-id map "t") (occurrence-type o))
(scope-eq-p (list (by-id map "s")) (occurrence-scope o))
(equal value (occurrence-value o)) (equal dtype (occurrence-datatype o)))))))))
(let* ((resource "V")
(base (occurrence-xml resource :scope '("s"))))
(occurrence-control "equality-discriminators"
(format nil "~A~A~A~A~A~A~A" base base
(occurrence-xml "W" :scope '("s"))
(occurrence-xml "V" :scope '("s"))
(occurrence-xml resource :type "u" :scope '("s")) (occurrence-xml resource :scope '("z")) base)
(lambda (map held counts) (declare (ignore counts))
(check "occurrence:equality-complete-parent-type-scope-value-datatype"
(and (= 6 (length (all-occurrences map))) (= 5 (length (topic-occurrences (by-id map "a"))))
(= 1 (length (topic-occurrences (by-id map "b")))) (= 7 (length held))
(not (eq (first held) (second held))) (eq (canonical-occurrence (first held)) (canonical-occurrence (second held)))))))))
(defun occurrence-merge-controls ()
(dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity"))
(dolist (dimension '("parent" "type" "scope"))
(let* ((resource "V")
(a (if (equal dimension "parent") "a" "x")) (b (if (equal dimension "parent") "b" "y"))
(first (occurrence-xml resource :type (if (equal dimension "type") "x" "t") :scope (if (equal dimension "scope") '("x") '("s"))))
(second (occurrence-xml resource :type (if (equal dimension "type") "y" "t") :scope (if (equal dimension "scope") '("y") '("s"))))
(body (concatenate 'string
(if (equal dimension "parent") (format nil "~A~A" first second)
(format nil "~A~A" first second))
(format nil "<~A href='urn:shared'/><~A href='urn:shared'/>" a kind b kind))))
(occurrence-control (format nil "merge-~A-~A" dimension kind) body
(lambda (map held counts)
(check (format nil "occurrence:merge-~A-~A:two-before-one-after-old-handles-EQ" dimension kind)
(and (equal '(1 2) counts) (= 1 (length (all-occurrences map))) (= 2 (length held))
(not (eq (first held) (second held)))
(eq (canonical-occurrence (first held)) (canonical-occurrence (second held)))
(not (eq (first held) (canonical-occurrence (first held))))
(not (eq (second held) (canonical-occurrence (second held))))
(eq (by-id map a) (by-id map b)) (eq (by-id map "alias1") (by-id map "alias2")))))))))
(occurrence-control "other-characteristics-preserved"
(format nil "NV~A"
(occurrence-xml "O"))
(lambda (map held counts) (declare (ignore held counts))
(check "occurrence:existing-names-variants-core-associations-preserved"
(and (= 1 (length (all-occurrences map))) (= 1 (length (name-reifier-names map)))
(= 1 (length (all-variants map))) (= 1 (length (topic-map-associations map)))
(= 2 (length (association-roles (first (topic-map-associations map)))))
(equal "N" (topic-name-value (first (topic-names (by-id map "a")))))
(equal "V" (variant-value (first (all-variants map)))))))))
(defun occurrence-negative-controls ()
(dolist (spec
'(("missing-type" "V" xtm-rejection :missing-occurrence-type)
("missing-resource" "" xtm-rejection :missing-occurrence-resource)
("two-resources" "V" xtm-rejection :occurrence-resource-cardinality)
("two-types" "V" xtm-rejection :duplicate-occurrence-type)
("empty-type" "V" xtm-rejection :type-cardinality)
("two-type-refs" "V" xtm-rejection :type-cardinality)
("bad-type-ref" "V" xtm-rejection :invalid-topic-ref)
("empty-scope" "V" xtm-rejection :empty-scope)
("bad-scope-ref" "V" xtm-rejection :invalid-topic-ref)
("two-scopes" "V" xtm-rejection :duplicate-occurrence-scope)
("wrong-order" "V" xtm-rejection :element-order)
("early-scope" "V" xtm-rejection :missing-occurrence-type)
("bad-child" "" xtm-rejection :unexpected-element)
("foreign-namespace" "" xtm-rejection :invalid-namespace)
("bad-attribute" "" xtm-rejection :unexpected-attribute)
("text-child" "unexpected" xtm-rejection :unexpected-text)
("relative-datatype" "V" xtm-rejection :invalid-resource-data-datatype)
("embedded-without-anyType" "" xtm-rejection :unexpected-child)
("anyType-text" "V" xtm-unsupported :resource-data-datatype)
("anyType-XML" "" xtm-unsupported :resource-data-datatype)
("item-identity" "V" xtm-unsupported :unsupported-construct)
("reification" "V" xtm-unsupported :unsupported-construct)))
(destructuring-bind (label body class code) spec
(let* ((key (concatenate 'string "occurrence-negative:" label))
(c (condition-from (lambda () (deserialize-octets (bytes (document (format nil "~A" body)))
:document-iri *document* :source-key key)))))
(check (concatenate 'string key ":actual-class-code-context-position")
(and (eq (type-of c) class) (eq code (error-code c)) (equal *document* (error-document-iri c))
(equal key (error-source-key c)) (plusp (error-line c)) (plusp (error-column c))))
(push (mm-fields "label" key "sourceText" (document (format nil "~A" body)) "condition" (occurrence-condition c)) *occurrence-controls*))))
(dolist (body '("" "N"))
(let ((c (condition-from (lambda () (read-body body)))))
(check (concatenate 'string "occurrence-negative:invalid-parent-context:" body) (and (typep c 'xtm-rejection) (eq :unexpected-element (error-code c))))
(push (mm-fields "label" "invalid-parent-context" "sourceText" (document body) "condition" (occurrence-condition c)) *occurrence-controls*)))
(let ((c (condition-from (lambda () (deserialize-octets (bytes "") :document-iri *document* :source-key "occurrence:XML")))))
(check "occurrence-negative:real-XML-error" (and (typep c 'xtm-xml-error) (eq :xml-not-well-formed (error-code c))))
(push (mm-fields "label" "real-XML-error" "sourceText" "" "condition" (occurrence-condition c)) *occurrence-controls*))
(dolist (body (list
(format nil "~A" (occurrence-xml (datatype-resource (integer-datatype) "bad")))
(format nil "~A" (occurrence-xml "V"))))
(let ((c (condition-from (lambda () (read-body body)))))
(check (if (search "bad" body) "occurrence-negative:native-invalid-integer" "occurrence-negative:native-type-identifier-conflict")
(if (search "bad" body)
(and (typep c 'topic-maps.tmdm::occurrence-error) (eq :invalid-integer-value (topic-maps.tmdm::occurrence-error-code c)))
(and (typep c 'item-identifier-error) (eq :duplicate-item-identifier (item-identifier-error-code c)))))
(push (mm-fields "label" "occurrence-native-conflict" "sourceText" (document body) "condition" (occurrence-condition c)) *occurrence-controls*))))
(defun occurrence-tests ()
(setf *occurrence-controls* nil)
(occurrence-value-controls) (occurrence-merge-controls) (occurrence-negative-controls)
(setf *occurrence-controls* (nreverse *occurrence-controls*)))