]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/xtm-anytype.lisp
perf(tmdm): index topic normalization merge candidates
[topic-maps.git] / tests / xtm-anytype.lisp
1 (in-package #:topic-maps.xtm.tests)
2 (defvar *ss-retargets* nil)
3 (defvar *ss-results* nil)
4 (defvar *ss-controls* nil)
5 (defvar *ss-replays* nil)
6 (defvar *ss-executed* nil)
7 (defun ss-unsupported-source (source code label)
8 "Explicit 22S control retarget: keep the Unsupported classification/diagnostic
9 check, using a valid mergeMap without a resolver. The old anyType source is
10 retained and replayed separately; no production error is synthesized."
11 (if (and source (member code '(:resource-resolver-required :cyclic-mergemap))
12 (search "http://www.w3.org/2001/XMLSchema#anyType" source))
13 (let* ((end (position #\> source))
14 (new (concatenate 'string (subseq source 0 (1+ end))
15 (if (eq code :cyclic-mergemap) "<mergeMap href=''/>" "<mergeMap href='urn:22s:requires-explicit-resolver'/>") (subseq source (1+ end)))))
16 (assert (and end (search "<topicMap " source :end2 (1+ end))))
17 (push (mm-fields "label" label "sourceBefore" source "sourceAfter" new
18 "formerCode" "resource-data-datatype" "expectedClass" "XTM-UNSUPPORTED"
19 "newCode" (string-downcase (string code)) "purpose" "real Unsupported must not become an original PASS; preserve diagnostic context checks"
20 "originalSourceReplayedSeparately" :true) *ss-retargets*)
21 new)
22 source))
23 (defun ss-occ (resource &key id reifier scope)
24 (format nil "<occurrence~A>~A<type><topicRef href='#type'/></type>~A~A</occurrence>"
25 (if reifier (format nil " reifier='~A'" reifier) "")
26 (if id (format nil "<itemIdentity href='~A'/>" id) "") (if scope (scope-xml scope) "") resource))
27 (defun ss-data (content &optional (datatype "http://www.w3.org/2001/XMLSchema#anyType"))
28 (format nil "<resourceData~A>~A</resourceData>" (if datatype (format nil " datatype='~A'" datatype) "") content))
29 (defun ss-values (map)
30 (append (loop for tpc in (topic-map-topics map) append
31 (loop for o in (topic-occurrences tpc) collect (list "occurrence" (occurrence-value o) (occurrence-datatype o))))
32 (loop for tpc in (topic-map-topics map) append
33 (loop for n in (topic-names tpc) append
34 (loop for v in (topic-name-variants n) collect (list "variant" (variant-value v) (variant-datatype v)))))))
35 (defun ss-positive (id source expected &key (encoding :utf-8) bom verify resolver (doc "https://anytype.example.invalid/root.xtm"))
36 (let* ((raw (bytes source encoding bom)) (copy (copy-seq raw))
37 (map (deserialize-octets raw :document-iri doc :source-key "same-key-is-not-document-identity" :resource-resolver resolver))
38 (native (ss-values map)) (state (ik-state map)) (actual (mm-cxtm map doc)) (again (mm-cxtm map doc)))
39 (check (format nil "anytype:~A:native-values-and-datatypes" id)
40 (and (= (length expected) (length native))
41 (every (lambda (row) (= (count row expected :test #'equal) (count row native :test #'equal))) native)))
42 (check (format nil "anytype:~A:all-canonical-EQ-references" id) (mm-current-p map))
43 (check (format nil "anytype:~A:expected-native-occurrence-parent-type-scope-reifier" id)
44 (every (lambda (o)
45 (let* ((parent (occurrence-parent o))
46 (origin (cond ((eq parent (mm-local map doc "owner")) doc)
47 ((and (equal id "mergeMap-separate-original-document-contexts")
48 (eq parent (mm-local map "https://anytype.example.invalid/child.xtm" "child")))
49 "https://anytype.example.invalid/child.xtm"))))
50 (and origin (eq (occurrence-type o) (mm-local map origin "type"))
51 (if (equal id "metadata-variant-and-occurrence")
52 (and (scope-eq-p (list (mm-local map doc "base") (mm-local map doc "extra")) (occurrence-scope o))
53 (eq (item-reifier o) (mm-local map doc "orr")))
54 (and (null (occurrence-scope o)) (null (item-reifier o))))))) (all-occurrences map)))
55 (when (search "22Q/" id)
56 (check (format nil "anytype:~A:no-topics-from-XML-names" id) (= 2 (length (topic-map-topics map)))))
57 (when verify (funcall verify map))
58 (check (format nil "anytype:~A:source-and-model-pure-repeat-CXTM" id)
59 (and (equalp raw copy) (equalp actual again) (equal state (ik-state map))))
60 (let* ((other (deserialize-octets raw :document-iri doc :source-key "different-diagnostic-key" :resource-resolver resolver))
61 (cx (mm-cxtm other doc)))
62 (check (format nil "anytype:~A:source-key-independent-deterministic-reader" id) (equalp actual cx)))
63 (push (mm-fields "id" id "inputHex" (cr-hex raw) "inputXML" source "encoding" (string encoding)
64 "documentIRI" doc "sourceKey" "same-key-is-not-document-identity"
65 "expectedValues" (coerce (mapcar (lambda (v) (coerce v 'vector)) expected) 'vector)
66 "actualValues" (coerce (mapcar (lambda (v) (coerce v 'vector)) native) 'vector)
67 "nativeModel" (occurrence-model map) "canonicalReferencesEQ" :true "repeatedCXTMPure" :true
68 "actualCXTM" (babel:octets-to-string actual :encoding :utf-8) "actualCXTMHex" (cr-hex actual)) *ss-results*) map))
69 (defun ss-error (id source class code &optional thunk)
70 (let ((c (condition-from (or thunk (lambda () (deserialize-octets (bytes source)
71 :document-iri *document* :source-key id))))))
72 (check (format nil "anytype-error:~A:specific-class-code" id)
73 (and (typep c class) (eq (error-code c) code)))
74 (check (format nil "anytype-error:~A:actual-source-and-cause" id)
75 (and (equal (error-document-iri c) *document*) (equal (error-source-key c) id)
76 (if (eq code :relative-namespace-uri)
77 (and (typep (topic-maps.xtm:error-cause c) 'topic-maps.xml:canonicalization-error)
78 (null (error-line c)) (null (error-column c))) t)))
79 (push (mm-fields "id" id "inputXML" source "condition" (mm-condition c)) *ss-controls*) c))
80 (defun ss-tests ()
81 (setf *ss-results* nil *ss-controls* nil *ss-replays* nil *ss-executed* nil)
82 (let ((dt "http://www.w3.org/2001/XMLSchema#anyType"))
83 ;; Reuse independent, fixed 22Q expectations through full productive Reader
84 ;; documents. Root uses an explicit XTM prefix so embedded unprefixed XML
85 ;; keeps the same expanded names as those fixed inputs.
86 (dolist (spec (cq-specs))
87 (destructuring-bind (id resource want rule) spec
88 (declare (ignore rule))
89 (ss-positive (concatenate 'string "22Q/" id)
90 (format nil "<t:topicMap xmlns:t='http://www.topicmaps.org/xtm/' version='2.0'><t:topic id='owner'><t:occurrence><t:type><t:topicRef href='#type'/></t:type>~A</t:occurrence></t:topic></t:topicMap>" resource)
91 (list (list "occurrence" want dt)))))
92 (dolist (enc '(:utf-8 :utf-16le :utf-16be))
93 (ss-positive (format nil "unicode/~A" enc)
94 (document (format nil "<topic id='owner'>~A</topic>" (ss-occ (ss-data "<p:x xmlns:p='urn:p'>Grüße 水 🦊</p:x>"))))
95 (list (list "occurrence" "<p:x xmlns:p=\"urn:p\">Grüße 水 🦊</p:x>" dt))
96 :encoding enc :bom (not (eq enc :utf-8))))
97 (ss-positive "mixed-datatypes-and-foreign-resourceData"
98 (document (format nil "<topic id='owner'>~A~A~A~A~A~A~A</topic>"
99 (ss-occ (ss-data "<f:resourceData xmlns:f='urn:f'><f:a/></f:resourceData>"))
100 (ss-occ (ss-data "literal" nil)) (ss-occ (ss-data "../asset" (iri-datatype)))
101 (ss-occ (ss-data " 007 " "urn:custom")) (ss-occ (ss-data ""))
102 (ss-occ (ss-data " A&amp;B&lt;C&gt;&quot;&#xD; "))
103 (ss-occ "<resourceRef href='../ref'/>")))
104 (list (list "occurrence" "<f:resourceData xmlns:f=\"urn:f\"><f:a></f:a></f:resourceData>" dt)
105 (list "occurrence" "literal" (string-datatype)) (list "occurrence" "https://anytype.example.invalid/asset" (iri-datatype))
106 (list "occurrence" " 007 " "urn:custom") (list "occurrence" "" dt)
107 (list "occurrence" " A&amp;B&lt;C&gt;\"&#xD; " dt)
108 (list "occurrence" "https://anytype.example.invalid/ref" (iri-datatype))))
109 (ss-positive "metadata-variant-and-occurrence"
110 (document (format nil "<topic id='owner'>~A<name><scope><topicRef href='#base'/></scope><value>N</value><variant reifier='#vr'><itemIdentity href='urn:v'/><scope><topicRef href='#extra'/></scope>~A</variant></name></topic>"
111 (ss-occ (ss-data "<p:x xmlns:p='urn:p'/>") :id "urn:o" :reifier "#orr" :scope '("base" "extra"))
112 (ss-data "<p:y xmlns:p='urn:p'>V</p:y>")))
113 (list (list "occurrence" "<p:x xmlns:p=\"urn:p\"></p:x>" dt)
114 (list "variant" "<p:y xmlns:p=\"urn:p\">V</p:y>" dt))
115 :verify (lambda (map)
116 (let* ((owner (mm-local map "https://anytype.example.invalid/root.xtm" "owner"))
117 (o (first (topic-occurrences owner))) (name (first (topic-names owner))) (v (first (topic-name-variants name)))
118 (doc "https://anytype.example.invalid/root.xtm") (base (mm-local map doc "base")) (extra (mm-local map doc "extra")))
119 (check "anytype:metadata:identifiers-reifiers-parents-types-scopes-EQ"
120 (and (equal '("urn:o") (item-identifiers o)) (equal '("urn:v") (item-identifiers v))
121 (eq o (topic-reified (mm-local map doc "orr"))) (eq v (topic-reified (mm-local map doc "vr")))
122 (eq owner (occurrence-parent o)) (eq name (variant-parent v)) (eq owner (topic-name-parent name))
123 (eq (occurrence-type o) (mm-local map doc "type"))
124 (scope-eq-p (list base extra) (occurrence-scope o)) (scope-eq-p (list base extra) (variant-scope v)))))))
125 (ss-positive "lexical-equivalence-native-normalization"
126 (document (format nil "<topic id='owner'>~A~A</topic>"
127 (ss-occ (ss-data "<x xmlns='' b='2' a='&#65;'><y/><![CDATA[a&b]]></x>") :id "urn:first")
128 (ss-occ (ss-data "<x xmlns='' a='A' b='2'><y></y>a&amp;b</x>") :id "urn:second")))
129 (list (list "occurrence" "<x a=\"A\" b=\"2\"><y></y>a&amp;b</x>" dt))
130 :verify (lambda (map) (let ((o (first (all-occurrences map))))
131 (check "anytype:normalization:one-occurrence-and-identifier-union"
132 (and (= 1 (length (all-occurrences map))) (same-identifiers-p '("urn:first" "urn:second") (item-identifiers o)))))))
133 (let* ((child "https://anytype.example.invalid/child.xtm")
134 (raw (bytes (document (format nil "<topic id='child'>~A</topic>" (ss-occ (ss-data "<p:x xmlns:p='urn:child'>C</p:x>")))) :utf-16le t))
135 (calls nil)
136 (resolver (lambda (iri &rest args) (push (cons iri args) calls)
137 (assert (equal iri child)) (values raw "same-key-is-not-document-identity"))))
138 (ss-positive "mergeMap-separate-original-document-contexts"
139 (document (format nil "<mergeMap href='child.xtm'/><topic id='owner'>~A</topic>" (ss-occ (ss-data "<p:x xmlns:p='urn:parent'>P</p:x>"))))
140 (list (list "occurrence" "<p:x xmlns:p=\"urn:parent\">P</p:x>" dt)
141 (list "occurrence" "<p:x xmlns:p=\"urn:child\">C</p:x>" dt)) :resolver resolver
142 :verify (lambda (map)
143 (check "anytype:mergeMap:child-IRI-local-topics-and-exact-resolver"
144 (and (mm-local map child "child")
145 (equal (first calls) (list child :purpose :mergemap :from-document-iri "https://anytype.example.invalid/root.xtm" :notation "http://psi.topicmaps.org/iso13250/xtm")))))))
146 (dolist (row (list
147 (list "malformed" "<a><b></a>" 'xtm-xml-error :xml-not-well-formed)
148 (list "unbound-prefix" (document (format nil "<topic id='a'>~A</topic>" (ss-occ (ss-data "<missing:x/>")))) 'xtm-xml-error :xml-not-well-formed)
149 (list "relative-namespace" (document (format nil "<topic id='a'>~A</topic>" (ss-occ (ss-data "<p:x xmlns:p='relative'/>")))) 'xtm-xml-error :relative-namespace-uri)
150 (list "DOCTYPE" (concatenate 'string "<!DOCTYPE topicMap>" (document "")) 'xtm-xml-error :doctype-forbidden)
151 (list "external-entity" (concatenate 'string "<!DOCTYPE topicMap [<!ENTITY secret SYSTEM 'file:///tmp/no-read'>]>" (document "&secret;")) 'xtm-xml-error :doctype-forbidden)
152 (list "outside-markup" (document "<x xmlns='urn:foreign'/>") 'xtm-rejection :invalid-namespace)
153 (list "nested-XTM-resourceData" (document (format nil "<topic id='a'>~A</topic>" (ss-occ (ss-data "<x xmlns='urn:f'><resourceData xmlns='http://www.topicmaps.org/xtm/'>X</resourceData></x>")))) 'xtm-rejection :embedded-xtm-element)))
154 (apply #'ss-error row))
155 (dolist (datatype (list nil (string-datatype) (iri-datatype) "urn:custom"))
156 (ss-error (format nil "textual-datatype-disallows-markup/~A" datatype)
157 (document (format nil "<topic id='a'>~A</topic>" (ss-occ (ss-data "<x xmlns='urn:f'/>" datatype)))) 'xtm-rejection :unexpected-child))
158 ;; Deliberately corrupt only the private Reader association, never C14N.
159 (dolist (mode '(wrong-index foreign-node foreign-frame wrong-document changed-octets inventory-mismatch))
160 (let* ((source (document (format nil "<topic id='a'>~A~A</topic>" (ss-occ (ss-data "<x xmlns='urn:a'/>")) (ss-occ (ss-data "<x xmlns='urn:b'/>")))))
161 (prepare (symbol-function 'topic-maps.xtm::prepare-xml-resources))
162 (count (symbol-function 'topic-maps.xml:resource-data-count))
163 (id (format nil "association/~A" mode)))
164 (unwind-protect
165 (progn
166 (if (eq mode 'inventory-mismatch)
167 (setf (symbol-function 'topic-maps.xml:resource-data-count) (lambda (ctx) (1+ (funcall count ctx))))
168 (setf (symbol-function 'topic-maps.xtm::prepare-xml-resources)
169 (lambda (root original snapshot)
170 (let* ((b (funcall prepare root original snapshot))
171 (node (aref (topic-maps.xtm::xml-resource-binding-nodes b) 0)))
172 (case mode
173 (wrong-index (setf (gethash node (topic-maps.xtm::xml-resource-binding-indices b)) 1))
174 (foreign-node (remhash node (topic-maps.xtm::xml-resource-binding-indices b)))
175 (wrong-document (setf (topic-maps.xtm::xml-resource-binding-document-iri b) "urn:foreign"))
176 (changed-octets (setf (aref snapshot 0) 0)))
177 (if (eq mode 'foreign-frame)
178 (funcall prepare (topic-maps.xml:parse-octets original (make-instance 'topic-maps.xtm::source-handler)) original snapshot)
179 b)))))
180 (ss-error id source 'xtm-xml-error :resource-data-association))
181 (setf (symbol-function 'topic-maps.xtm::prepare-xml-resources) prepare
182 (symbol-function 'topic-maps.xml:resource-data-count) count))))
183 ;; Independently retain the old classification controls' original sources and
184 ;; observe their new Reader outcome instead of continuing to call them
185 ;; Unsupported. These are supplementary observations, not corpus contracts.
186 (dolist (r *ss-retargets*)
187 (let* ((f (topic-maps.test-foundation::json-object-members r))
188 (source (cdr (assoc "sourceBefore" f :test #'equal))) (map nil)
189 (c (condition-from (lambda () (setf map (deserialize-octets (bytes source) :document-iri *document* :source-key "22s:former-unsupported-source"))))))
190 (check (format nil "anytype-replay:~D:no-longer-anyType-Unsupported" (length *ss-replays*))
191 (not (typep c 'xtm-unsupported)))
192 (push (mm-fields "sourceText" source "condition" (mm-condition c) "nativeValues" (when map (coerce (mapcar (lambda (v) (coerce v 'vector)) (ss-values map)) 'vector))
193 "actualCXTM" (when map (babel:octets-to-string (mm-cxtm map) :encoding :utf-8))) *ss-replays*)))
194 (setf *ss-results* (nreverse *ss-results*) *ss-controls* (nreverse *ss-controls*) *ss-replays* (nreverse *ss-replays*)
195 *ss-retargets* (nreverse *ss-retargets*) *ss-executed* t)) t)
196 (defun ss-report ()
197 (mm-fields "executed" (if *ss-executed* :true :false) "XTMReaderAnyTypeIntegration" (if *ss-executed* :true :false)
198 "originalRegistrationsAdded" 0 "positiveProbes" (coerce *ss-results* 'vector) "negativeProbes" (coerce *ss-controls* 'vector)
199 "retargetings" (coerce *ss-retargets* 'vector) "formerSourceReplays" (coerce *ss-replays* 'vector)))