]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/xtm-resource-datatypes.lisp
Implement XTM association reification with native conformance evidence
[topic-maps.git] / tests / xtm-resource-datatypes.lisp
1 (in-package #:topic-maps.xtm.tests)
2
3 (defun datatype-native-preflight ()
4 ;; Run before changing the Reader, and again in the regular ASDF test-op.
5 (let* ((map (make-topic-map))
6 (owner (intern-topic map :item-identifier "urn:datatype:owner"))
7 (base (intern-topic map :item-identifier "urn:datatype:base"))
8 (a (intern-topic map :item-identifier "urn:datatype:a"))
9 (b (intern-topic map :item-identifier "urn:datatype:b"))
10 (r1 (intern-topic map :item-identifier "urn:datatype:r1"))
11 (r2 (intern-topic map :item-identifier "urn:datatype:r2"))
12 (name (add-default-name owner "N" :scope (list base)))
13 (count (length (topic-map-topics map)))
14 (datatype "https://types.example.invalid/custom")
15 (value " 42 & 馬🦊 ")
16 (v1 (add-variant name value :datatype datatype :scope (list base a)))
17 (v2 (add-variant name value :datatype datatype :scope (list base b)))
18 (other (add-variant name value :datatype "urn:types:other" :scope (list base a))))
19 (check "datatype-native:value-and-datatype-independent"
20 (and (equal value (variant-value v1)) (equal datatype (variant-datatype v1))
21 (equal "urn:types:other" (variant-datatype other))))
22 (check "datatype-native:no-topic-creation" (= count (length (topic-map-topics map))))
23 (check "datatype-native:distinct-datatypes-and-scopes" (= 3 (length (all-variants map))))
24 (add-item-identifier v1 "urn:variant:first")
25 (add-item-identifier v2 "urn:variant:second")
26 (set-item-reifier v1 r1) (set-item-reifier v2 r2)
27 (merge-topics a b)
28 (let ((current (canonical-variant v1)))
29 (check "datatype-native:old-variant-handles-canonical-EQ"
30 (and (eq current (canonical-variant v2)) (not (eq current v1))
31 (not (eq current other)) (= 2 (length (all-variants map)))))
32 (check "datatype-native:parent-and-scope-canonical-EQ"
33 (and (eq name (canonical-topic-name name)) (eq owner (topic-name-parent name))
34 (eq name (variant-parent current))
35 (scope-eq-p (list base (canonical-topic a)) (variant-scope current))))
36 (check "datatype-native:identifiers-unioned"
37 (and (= 2 (length (item-identifiers current)))
38 (member "urn:variant:first" (item-identifiers current) :test #'equal)
39 (member "urn:variant:second" (item-identifiers current) :test #'equal)))
40 (check "datatype-native:reifier-inverse-and-old-handles-EQ"
41 (and (eq (canonical-topic r1) (canonical-topic r2))
42 (eq (item-reifier current) (canonical-topic r1))
43 (eq current (topic-reified r1)) (eq current (topic-reified r2))))
44 (let ((duplicate (add-variant name value :datatype datatype :scope (list base (canonical-topic a)))))
45 (check "datatype-native:duplicate-add-normalizes"
46 (and (eq (canonical-variant current) duplicate) (eq duplicate (canonical-variant v1))
47 (eq duplicate (canonical-variant v2)) (= 2 (length (all-variants map))))))
48 (check "datatype-native:datatype-survives-normalization"
49 (and (equal datatype (variant-datatype v1)) (equal value (variant-value v2)))))
50 (let* ((before (map-reifier-state map))
51 (actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*)))
52 (check "datatype-native:cxtm-preserves-custom-lexical-value"
53 (and (search "<value> 42 &amp; 馬🦊 </value>" (babel:octets-to-string actual :encoding :utf-8))
54 (search (concatenate 'string "<datatype>" datatype "</datatype>")
55 (babel:octets-to-string actual :encoding :utf-8))))
56 (check "datatype-native:pure-deterministic-canonicalization"
57 (and (equal before (map-reifier-state map))
58 (equalp actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*))
59 (equal before (map-reifier-state map)) (variant-reifier-current-p map))))
60 map))
61
62 (defvar *datatype-outputs* nil)
63 (defvar *datatype-controls* nil)
64 (defun datatype-resource (datatype value)
65 (format nil "<resourceData~A>~A</resourceData>"
66 (if datatype (format nil " datatype='~A'" datatype) "") value))
67 (defun datatype-variant (datatype value &key (themes '("s")) reifier identifiers)
68 (variant-reifier-xml reifier :themes themes :identifiers identifiers
69 :resource (datatype-resource datatype value)))
70 (defun datatype-body (resource)
71 (format nil "<topic id='a'><name><value>N</value><variant>~A~A</variant></name></topic>" (scope-xml '("s")) resource))
72
73 (defun datatype-original-contract ()
74 (let* ((id "xtm2/in/variant-datatype-unknown.xtm")
75 (*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id))
76 (raw (file-bytes (corpus-path id))) (copy (copy-seq raw))
77 (baseline (corpus-path "xtm2/baseline/variant-datatype-unknown.xtm.cxtm"))
78 (expected (file-bytes baseline))
79 (case (topic-maps.test-foundation:make-test-case :id id :category :valid
80 :source-pathname (corpus-path id) :document-iri *document* :expected-outcome :cxtm-equal
81 :expected-baseline baseline :clauses '("XTM 2.0 4.12,4.16.1-4.16.3" "TMDM 4.4,5.5,6.4" "CXTM 3.14,3.16 Other")))
82 (map nil) (actual nil) (condition nil) (parser-calls 0) (parser-input-eq nil))
83 (setf *cases* (append *cases* (list case)))
84 (handler-case
85 (progn
86 (let ((saved (symbol-function 'topic-maps.xml:parse-octets)))
87 (unwind-protect
88 (progn
89 (setf (symbol-function 'topic-maps.xml:parse-octets)
90 (lambda (octets &rest args)
91 (setf parser-input-eq (eq octets raw))
92 (incf parser-calls) (apply saved octets args)))
93 (setf map (deserialize-octets raw :document-iri *document* :source-key id)))
94 (setf (symbol-function 'topic-maps.xml:parse-octets) saved)))
95 (let* ((owner (by-id map "topic")) (theme (by-id map "scope"))
96 (name (first (topic-names owner))) (v (first (all-variants map)))
97 (type (by-si map (default-name-type-identifier))))
98 (check "datatype-original:native-model-parent-type-scope-EQ"
99 (and (= 3 (length (topic-map-topics map))) (null (topic-map-associations map))
100 (every (lambda (topic) (null (topic-occurrences topic))) (topic-map-topics map))
101 (= 1 (length (name-reifier-names map))) (= 1 (length (all-variants map)))
102 (equal "Topic" (topic-name-value name)) (null (topic-name-scope name))
103 (eq owner (topic-name-parent name)) (eq type (topic-name-type name))
104 (eq name (canonical-topic-name name)) (eq name (variant-parent v))
105 (eq v (canonical-variant v)) (scope-eq-p (list theme) (variant-scope v))
106 (variant-reifier-current-p map)))
107 (check "datatype-original:value-datatype-not-topic-identities"
108 (and (equal "BOGUS" (variant-value v)) (equal "http://example.com/bogus" (variant-datatype v))
109 (equal (list (concatenate 'string *document* "#topic")) (item-identifiers owner))
110 (equal (list (concatenate 'string *document* "#scope")) (item-identifiers theme))
111 (null (item-identifiers type)) (equal (list (default-name-type-identifier)) (topic-subject-identifiers type))
112 (null (topic-subject-identifiers owner)) (null (topic-subject-identifiers theme))
113 (every (lambda (topic) (null (topic-subject-locators topic))) (topic-map-topics map))
114 (null (item-identifiers map)) (null (item-identifiers name)) (null (item-identifiers v))
115 (null (item-reifier map)) (null (item-reifier name)) (null (item-reifier v))
116 (null (by-si map "http://example.com/bogus")))))
117 (setf actual (variant-reifier-pure "datatype-original" map (append (all-variants map) (name-reifier-names map) (topic-map-topics map))))
118 (check "datatype-original:parser-receives-original-octets" parser-input-eq)
119 (check "datatype-original:source-preserved-parser-once" (and (= parser-calls 1) (equalp raw copy)))
120 (let ((other (deserialize-octets raw :document-iri *document* :source-key "independent:datatype-original")))
121 (check "datatype-original:document-IRI-source-key-independence"
122 (and (equalp (variant-reifier-model-data map) (variant-reifier-model-data other))
123 (equalp actual (topic-maps.cxtm:canonicalize-octets other :base-locator *document*))))))
124 (error (c) (setf condition c)))
125 (record-original-result case actual expected condition)
126 (setf *datatype-outputs* (list (topic-maps.test-foundation::fields
127 "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw)
128 "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8)
129 "qualifiedParserCalls" parser-calls "condition" (map-reifier-condition-data condition)
130 "model" (when map (variant-reifier-model-data map))
131 "actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8))
132 "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8)
133 "actualBytes" (when actual (length actual)) "expectedBytes" (length expected)
134 "firstDifferentByte" (when actual (mismatch actual expected))
135 "status" (string-downcase (string (topic-maps.test-foundation:test-case-status case))))))
136 (when condition (error condition))
137 (check "datatype-original:579-byte-exact" (and (= 579 (length expected)) (equalp actual expected)))
138 (check "datatype-original:72-unique-registered-cases"
139 (and (= 72 (length *cases*)) (= 64 (count :valid *cases* :key #'topic-maps.test-foundation:test-case-category))
140 (= 72 (length (remove-duplicates *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal)))))))
141
142 (defun datatype-value-controls ()
143 (dolist (row (list
144 (list "http-custom" "http://types.example.invalid/custom" "literal%GG" "literal%GG")
145 (list "urn-custom" "urn:types:custom" "urn:resource-not-a-topic" "urn:resource-not-a-topic")
146 (list "unicode" "urn:types:馬" "馬🦊é" "馬🦊é")
147 (list "datatype-percent-preserved" "urn:types:%61" "42" "42")
148 (list "whitespace-escaping" "urn:types:text" " &#x9;A&amp;&lt;&gt;&#xA; " (format nil " ~CA&<>~% " #\Tab))
149 (list "empty" "urn:types:empty" "" "")
150 (list "integer-text-now-supported" (integer-datatype) "+00042" "+00042")
151 (list "explicit-string" (string-datatype) "urn:%GG" "urn:%GG")
152 (list "default-string" nil " x &amp; y " " x & y ")
153 (list "anyURI-retained" (iri-datatype) "../%E9%A6%AC" "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/xtm2/馬")))
154 (destructuring-bind (label dtype xml value) row
155 (variant-reifier-control (concatenate 'string "datatype:" label) (datatype-body (datatype-resource dtype xml))
156 (lambda (map)
157 (let ((v (first (all-variants map))))
158 (check (concatenate 'string "datatype:value-and-exact-datatype:" label)
159 (and (equal value (variant-value v)) (equal (or dtype (string-datatype)) (variant-datatype v))
160 (= 3 (length (topic-map-topics map))) (eq (variant-parent v) (first (topic-names (by-id map "a"))))
161 (scope-eq-p (variant-scope v) (list (by-id map "s"))))))))))
162 (variant-reifier-control "datatype:normalization-discriminators"
163 (format nil "<topic id='a'>~A</topic>" (variant-name-xml nil (concatenate 'string
164 (datatype-variant "urn:type:a" "V") (datatype-variant "urn:type:a" "V")
165 (datatype-variant "urn:type:a" "W") (datatype-variant "urn:type:b" "V"))))
166 (lambda (map)
167 (check "datatype:equality-includes-datatype-and-value"
168 (and (= 3 (length (all-variants map))) (= 3 (length (topic-map-topics map)))
169 (= 2 (count "urn:type:a" (all-variants map) :key #'variant-datatype :test #'equal))
170 (= 2 (count "V" (all-variants map) :key #'variant-value :test #'equal)))))))
171
172 (defun datatype-merge-controls ()
173 (dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity"))
174 (let* ((held nil) (saved (symbol-function 'topic-maps.xtm::read-variant))
175 (body (format nil "<topic id='a'><~A href='urn:owner'/>~A</topic><topic id='b'><~A href='urn:owner'/>~A</topic><topic id='x'><~A href='urn:theme'/></topic><topic id='y'><~A href='urn:theme'/></topic>"
176 kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("x") :reifier "#r1" :identifiers '("urn:v1")))
177 kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("y") :reifier "#r2" :identifiers '("urn:v2"))) kind kind)))
178 (unwind-protect
179 (progn
180 (setf (symbol-function 'topic-maps.xtm::read-variant)
181 (lambda (&rest args) (let ((v (apply saved args))) (push v held) v)))
182 (variant-reifier-control (concatenate 'string "datatype:merge-" kind) body
183 (lambda (map)
184 (let* ((owner (by-id map "a")) (name (first (topic-names owner))) (v (first (all-variants map)))
185 (r (by-id map "r1")))
186 (check (concatenate 'string "datatype:merge-canonical-identities:" kind)
187 (and (eq owner (by-id map "b")) (eq name (variant-parent v))
188 (eq owner (topic-name-parent name)) (eq (by-id map "x") (by-id map "y"))
189 (scope-eq-p (variant-scope v) (list (by-id map "base") (by-id map "x")))
190 (= 1 (length (all-variants map))) (= 2 (length held)) (not (eq (first held) (second held)))
191 (every (lambda (old) (eq v (canonical-variant old))) held)
192 (= 2 (length (item-identifiers v))) (member "urn:v1" (item-identifiers v) :test #'equal)
193 (member "urn:v2" (item-identifiers v) :test #'equal)
194 (eq r (by-id map "r2")) (eq r (item-reifier v)) (eq v (topic-reified r))
195 (equal "urn:datatype:merged" (variant-datatype v)) (equal "V" (variant-value v))))
196 (variant-reifier-pure (concatenate 'string "datatype:held-" kind) map held)))))
197 (setf (symbol-function 'topic-maps.xtm::read-variant) saved)))))
198
199 (defun datatype-negative-controls ()
200 (dolist (row '(("relative" "relative" :invalid-resource-data-datatype)
201 ("empty" "" :invalid-resource-data-datatype) ("scheme" "1bad:x" :invalid-resource-data-datatype)
202 ("space" "urn:bad type" :invalid-iri) ("percent" "urn:type:%GG" :invalid-percent-escape)
203 ("percent-short" "urn:type:%" :invalid-percent-escape) ("percent-UTF8" "urn:type:%C3%28" :invalid-percent-utf8)
204 ("percent-surrogate" "urn:type:%ED%A0%80" :invalid-percent-utf8)))
205 (variant-reifier-error-control (concatenate 'string "datatype:" (first row))
206 (datatype-body (datatype-resource (second row) "V")) 'xtm-rejection (third row)))
207 (dolist (dtype '(nil "http://www.w3.org/2001/XMLSchema#string" "http://www.w3.org/2001/XMLSchema#anyURI" "urn:custom"))
208 (dolist (child '("<value>V</value>" "<x xmlns='urn:foreign'>V</x>"))
209 (variant-reifier-error-control (format nil "datatype:forbidden-child:~A:~A" dtype child)
210 (datatype-body (datatype-resource dtype child)) 'xtm-rejection :unexpected-child)))
211 (dolist (value '("text" "<x xmlns='urn:foreign'>XML</x>"))
212 (variant-reifier-error-control (concatenate 'string "datatype:anyType:" value)
213 (datatype-body (datatype-resource "http://www.w3.org/2001/XMLSchema#anyType" value))
214 'xtm-unsupported :resource-data-datatype))
215 (variant-reifier-error-control "datatype:unknown-XTM-structure" "<topic id='a'><unknown/></topic>" 'xtm-rejection :unexpected-element)
216 (variant-reifier-error-control "datatype:occurrence-remains-unsupported"
217 "<topic id='a'><occurrence reifier='#r'><type><topicRef href='#t'/></type><resourceData datatype='http://www.w3.org/2001/XMLSchema#anyType'>V</resourceData></occurrence></topic>"
218 'xtm-unsupported :resource-data-datatype)
219 (variant-reifier-error-control "datatype:native-reifier-conflict"
220 (format nil "<topic id='a'>~A</topic>" (variant-name-xml nil (concatenate 'string
221 (datatype-variant "urn:custom" "A" :reifier "#r") (datatype-variant "urn:custom" "B" :reifier "#r"))))
222 'reification-error :reifier-conflict)
223 (variant-reifier-error-control "datatype:native-scope-conflict"
224 (format nil "<topic id='a'>~A</topic>" (variant-name-xml '("s") (datatype-variant "urn:custom" "V")))
225 'variant-error :invalid-variant-scope)
226 (let ((raw (concatenate '(vector (unsigned-byte 8))
227 (bytes "<topicMap xmlns='http://www.topicmaps.org/xtm/' version='2.0'><topic id='a'><name><value>N</value><variant><scope><topicRef href='#s'/></scope><resourceData datatype='urn:custom'>")
228 #(195 40) (bytes "</resourceData></variant></name></topic></topicMap>"))))
229 (variant-reifier-error-control "datatype:raw-invalid-UTF8" nil 'xtm-xml-error :xml-not-well-formed
230 (lambda () (deserialize-octets raw :document-iri *document* :source-key "variant:datatype:raw-invalid-UTF8")))))
231
232 (defun datatype-context-controls ()
233 ;; Retain the resource-operation/native-consumer cross-check from Slice 22D.
234 ;; Slice 22G separately exercises genuine XTM occurrence deserialization.
235 (dolist (resource (list (datatype-resource "urn:shared:type" " A&amp;B ")
236 (datatype-resource nil "V")
237 (datatype-resource (iri-datatype) "../sound")
238 "<resourceRef href='../sound'/>"))
239 (let* ((root (topic-maps.xml:parse-octets (bytes (document (datatype-body resource)))
240 (make-instance 'topic-maps.xtm::source-handler)))
241 (node (second (topic-maps.xtm::elements
242 (second (topic-maps.xtm::elements
243 (first (topic-maps.xtm::elements (first (topic-maps.xtm::elements root)))))))))
244 (topic-maps.xtm::*document-iri* *document*)
245 (topic-maps.xtm::*source-key* "datatype:shared-context"))
246 (topic-maps.xtm::validate-source root)
247 (multiple-value-bind (value dtype) (topic-maps.xtm::read-resource-value node)
248 (let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:owner"))
249 (type (intern-topic map :item-identifier "urn:type")) (theme (intern-topic map :item-identifier "urn:theme"))
250 (name (add-default-name owner "N")) (count (length (topic-map-topics map)))
251 (v (add-variant name value :datatype dtype :scope (list theme)))
252 (o (add-occurrence owner type value :datatype dtype)))
253 (check (concatenate 'string "datatype:shared-value-consumers:" resource)
254 (and (equal (variant-value v) (occurrence-value o)) (equal (variant-datatype v) (occurrence-datatype o))
255 (eq name (variant-parent v)) (eq owner (occurrence-parent o)) (= count (length (topic-map-topics map))))))))))
256
257 (defun datatype-tests ()
258 (datatype-native-preflight)
259 (let ((*variant-reifier-controls* nil))
260 (datatype-value-controls) (datatype-merge-controls) (datatype-negative-controls) (datatype-context-controls)
261 (setf *datatype-controls* (nreverse *variant-reifier-controls*))))