]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/xml-resource-c14n.lisp
feat(tmdm): readers for the TMDM 7.2/7.3 relationships; tests for both default name...
[topic-maps.git] / tests / xml-resource-c14n.lisp
1 (in-package #:topic-maps.xtm.tests)
2
3 (defvar *cr-results* nil)
4 (defvar *cr-native* nil)
5 (defvar *cr-controls* nil)
6 (defvar *cr-executed* nil)
7 (defun cr-hex (octets) (format nil "~{~2,'0X~}" (coerce octets 'list)))
8 (defun cr-dom-state (node)
9 (list (dom:node-type node) (dom:node-name node) (dom:node-value node)
10 (when (eq (dom:node-type node) :element)
11 (loop for i below (dom:length (dom:attributes node)) for a = (dom:item (dom:attributes node) i)
12 collect (list (dom:name a) (dom:namespace-uri a) (dom:local-name a) (dom:value a))))
13 (loop for i below (dom:length (dom:child-nodes node)) collect (cr-dom-state (dom:item (dom:child-nodes node) i)))))
14 (defun cr-native (id value)
15 (let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:cr:owner"))
16 (type (intern-topic map :subject-identifier "urn:cr:type"))
17 (nt (intern-topic map :subject-identifier "urn:cr:name-type"))
18 (base (intern-topic map :subject-identifier "urn:cr:base"))
19 (extra (intern-topic map :subject-identifier "urn:cr:extra"))
20 (name (add-name owner "N" :type nt :scope (list base)))
21 (dt "http://www.w3.org/2001/XMLSchema#anyType")
22 (o (add-occurrence owner type value :datatype dt :scope (list extra)))
23 (v (add-variant name value :datatype dt :scope (list base extra)))
24 (before (ik-state map))
25 (a (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cr:map"))
26 (b (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cr:map")))
27 (check (format nil "resource-c14n-native:~A:lossless-occurrence-variant" id)
28 (and (equal value (occurrence-value o)) (equal value (variant-value v))
29 (equal dt (occurrence-datatype o)) (equal dt (variant-datatype v))))
30 (check (format nil "resource-c14n-native:~A:canonical-parent-type-scope-EQ" id)
31 (and (eq owner (occurrence-parent o)) (eq type (occurrence-type o))
32 (eq name (variant-parent v)) (eq o (canonical-occurrence o)) (eq v (canonical-variant v))
33 (scope-eq-p (list extra) (occurrence-scope o)) (scope-eq-p (list base extra) (variant-scope v))))
34 (check (format nil "resource-c14n-native:~A:pure-CXTM-no-topics-from-XML" id)
35 (and (= 5 (length (topic-map-topics map))) (equalp a b) (equal before (ik-state map))))
36 (push (mm-fields "id" id "value" value "datatype" dt "occurrenceValue" (occurrence-value o)
37 "variantValue" (variant-value v) "canonicalReferencesEQ" :true "topicCount" 5
38 "repeatedCXTMPure" :true "readerIntegration" :false "actualCXTMHex" (cr-hex a)) *cr-native*)))
39 (defun cr-probe (id raw expected origin &key root-vector)
40 (let* ((copy (copy-seq raw)) (ctx (topic-maps.xml:parse-canonical-context raw))
41 (doc (topic-maps.xml::%canonical-context-document ctx)) (state (cr-dom-state doc))
42 (want (bytes expected)))
43 (flet ((run ()
44 (if root-vector
45 ;; Independent W3C examples with document root selected. This
46 ;; private engine is the SAME engine as the public XTM adapter.
47 ;; The examples use all their namespace bindings, so XTM's
48 ;; filter makes no change to their published node set/result.
49 (let ((s (topic-maps.xml::%canonical-subtrees (list (dom:document-element doc)))))
50 (values s (bytes s)))
51 (topic-maps.xml:canonicalize-resource-data ctx))))
52 (multiple-value-bind (value actual) (run)
53 (multiple-value-bind (again repeated) (run)
54 (check (format nil "resource-c14n:~A:exact-independent-bytes" id) (equalp want actual))
55 (check (format nil "resource-c14n:~A:UTF8-string-and-repeat" id)
56 (and (equal expected value) (equalp actual (bytes value)) (equal value again) (equalp actual repeated))))
57 (check (format nil "resource-c14n:~A:input-context-unchanged" id)
58 (and (equalp raw copy) (equal state (cr-dom-state doc))))
59 (unless root-vector (cr-native id value))
60 (push (mm-fields "id" id "origin" origin "inputHex" (cr-hex raw) "expectedHex" (cr-hex want)
61 "actualHex" (cr-hex actual) "expectedXML" expected "actualXML" value
62 "expectedBytes" (length want) "actualBytes" (length actual) "firstDifferentByte" (mismatch want actual)
63 "byteIdentical" :true "repeatedPure" :true "publicXTMProjection" (if root-vector :false :true)) *cr-results*)))))
64
65 (defun cr-extra-specs ()
66 (list
67 (list "prefix-shadowing" (cq-wrap "<p:a><p:b xmlns:p='urn:two'><p:c/></p:b><p:d/></p:a>" " xmlns:p='urn:one'")
68 "<p:a xmlns:p=\"urn:one\"><p:b xmlns:p=\"urn:two\"><p:c></p:c></p:b><p:d></p:d></p:a>" "C14N 2.3: original prefixes, nearest namespace bindings")
69 (list "default-namespace-changes" (cq-wrap "<a><b xmlns=''><c xmlns='urn:inner'/></b><d/></a>" " xmlns='urn:outer'")
70 "<a xmlns=\"urn:outer\"><b xmlns=\"\"><c xmlns=\"urn:inner\"></c></b><d></d></a>" "C14N 2.3: selected-parent default namespace and empty reset")
71 (list "global-namespace-filter" (cq-wrap "<a><p:b/></a>" " xmlns:p='urn:p' xmlns:q='urn:unused'")
72 "<a xmlns:p=\"urn:p\"><p:b></p:b></a>" "XTM 4.16.3: namespace pair used anywhere in selected set; not Exclusive C14N")
73 (list "same-URI-different-prefix" (cq-wrap "<p:a><q:b q:v='1'/></p:a>" " xmlns:p='urn:same' xmlns:q='urn:same' xmlns:z='urn:same'")
74 "<p:a xmlns:p=\"urn:same\" xmlns:q=\"urn:same\"><q:b q:v=\"1\"></q:b></p:a>" "XTM 4.16.3 filters by URI AND prefix, not URI alone")
75 (list "QName-attribute-value-not-namespace-use" (cq-wrap "<a hint='p:x'/>" " xmlns:p='urn:p'")
76 "<a hint=\"p:x\"></a>" "Only element/attribute names establish namespace use, not values")
77 (list "filtered-parent-namespace-gap" (cq-wrap "<p:a><b xmlns:p='urn:unused'><p:c xmlns:p='urn:one'/></b></p:a>" " xmlns:p='urn:one'")
78 "<p:a xmlns:p=\"urn:one\"><b><p:c xmlns:p=\"urn:one\"></p:c></b></p:a>" "C14N 2.3 compares namespace nodes of nearest selected parent, not cumulative rendered map")
79 (list "xml-attributes-inherited" "<outer xml:lang='de' xml:space='preserve'><xtm:resourceData xmlns:xtm='http://www.topicmaps.org/xtm/' xml:lang='fr'><x/><x xml:lang='en'><y/></x></xtm:resourceData></outer>"
80 "<x xml:lang=\"fr\" xml:space=\"preserve\"></x><x xml:lang=\"en\" xml:space=\"preserve\"><y></y></x>" "C14N 1.0 2.4: nearest inherited xml:* on each apex; local override")
81 (list "xml-base-is-1.0-not-1.1" "<outer xml:base='https://example.invalid/' xml:id='ancestor'><xtm:resourceData xmlns:xtm='http://www.topicmaps.org/xtm/' xml:base='child/'><x/></xtm:resourceData></outer>"
82 "<x xml:base=\"child/\" xml:id=\"ancestor\"></x>" "C14N 1.0 2.4: all xml:* inherited, no 1.1 base fixup/id exception")
83 (list "empty-content" (cq-wrap "") "" "XTM descendant node set may be empty")
84 (list "empty-CDATA" (cq-wrap "<![CDATA[]]>") "" "Empty character information contributes no bytes")
85 (list "text-only" (cq-wrap " A&amp;B&lt;C&gt;&quot;&#x9;&#xA;&#xD; ")
86 (format nil " A&amp;B&lt;C&gt;\"~C~C&#xD; " #\Tab #\Newline) "XTM selected characters, C14N text escaping without container")
87 (list "several-apices-mixed-text" (cq-wrap "a<x/>b<y/>c") "a<x></x>b<y></y>c" "Selected document order, no invented fragment wrapper")
88 (list "attribute-order-same-URI" (cq-wrap "<x xmlns:a='urn:s' xmlns:z='urn:s' z:a='1' a:z='2' z:A='3' B='4' a='5'/>")
89 "<x xmlns:a=\"urn:s\" xmlns:z=\"urn:s\" B=\"4\" a=\"5\" z:A=\"3\" z:a=\"1\" a:z=\"2\"></x>" "C14N 2.2: codepoint order, namespace URI then local name")
90 (list "attribute-greater-than-and-whitespace" (cq-wrap (format nil "<x a='>&quot;~C~C&#x9;&#xA;&#xD;'/>" #\Tab #\Newline))
91 "<x a=\">&quot; &#x9;&#xA;&#xD;\"></x>" "XML whitespace normalization vs character references; > not escaped in attributes")
92 (list "no-Unicode-normalization-UTF8" (cq-wrap (format nil "<x>e~C</x>" (code-char #x301)))
93 (format nil "<x>e~C</x>" (code-char #x301)) "C14N 1.0 4.2: no normalization of UCS-based source character data")
94 (list "in-content-BOM-is-text" (cq-wrap (format nil "<x>A~CB</x>" (code-char #xfeff)))
95 (format nil "<x>A~CB</x>" (code-char #xfeff)) "Only encoding BOM is removed, not U+FEFF in character content")))
96
97 (defun cr-official-vectors ()
98 ;; Independent normative examples. The original full examples 3.1/3.3/3.4/
99 ;; 3.5/3.7 contain a DTD or arbitrary XPath selection and are NOT claimed run.
100 ;; The e5 subvector is copied as a complete element with all its namespaces;
101 ;; it has no dependency on the e9 default attribute in the enclosing DTD.
102 (let ((ws (format nil "<doc>~% <clean> </clean>~% <dirty> A B </dirty>~% <mixed>~% A~% <clean> </clean>~% B~% <dirty> A B </dirty>~% C~% </mixed>~%</doc>")))
103 (list
104 (list "w3c-3.2-whitespace" (bytes ws) ws "https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-WhitespaceInContent; complete unchanged example; root node set")
105 (list "w3c-3.3-e5-subvector" (bytes "<e5 a:attr=\"out\" b:attr=\"sorted\" attr2=\"all\" attr=\"I'm\"
106 xmlns:b=\"http://www.ietf.org\"
107 xmlns:a=\"http://www.w3.org\"
108 xmlns=\"http://example.org\"/>")
109 "<e5 xmlns=\"http://example.org\" xmlns:a=\"http://www.w3.org\" xmlns:b=\"http://www.ietf.org\" attr=\"I'm\" attr2=\"all\" b:attr=\"sorted\" a:attr=\"out\"></e5>"
110 "https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-SETags; independent e5 element/output excerpt; NOT the DTD-containing full example")
111 (list "w3c-3.6-ISO8859-1" (bytes (format nil "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>~%<doc>&#169;</doc>") :iso-8859-1)
112 "<doc>©</doc>" "https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-UTF8; complete original input; published C2 A9 byte expectation"))))
113
114 (defun cr-security-tests ()
115 (dolist (source '("<!DOCTYPE x><x/>" "<!DOCTYPE x [<!ENTITY a 'secret'>]><x>&a;</x>"
116 "<!DOCTYPE x SYSTEM 'file:///tmp/no-resource-c14n-read'><x/>"
117 "<!DOCTYPE x SYSTEM 'https://example.invalid/no-resource-c14n'><x/>"
118 "<!DOCTYPE x [<!ENTITY a SYSTEM 'file:///tmp/no-resource-c14n-read'>]><x>&a;</x>"))
119 (let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes source))))))
120 (check (format nil "resource-c14n-security:DOCTYPE-denied:~A" source) (typep c 'topic-maps.xml:doctype-forbidden))
121 (push (mm-fields "source" source "expected" "DOCTYPE-FORBIDDEN" "condition" (cq-condition c)) *cr-controls*)))
122 (dolist (source '("<x><y></x>" "<x xmlns:p='urn:a' xmlns:q='urn:a' p:a='1' q:a='2'/>"
123 "<p:x/>" "<x>&unknown;</x>" "<x a='1' a='2'/>"))
124 (let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes source))))))
125 (check (format nil "resource-c14n-security:XML-rejected:~A" source) (typep c 'cxml:well-formedness-violation))
126 (push (mm-fields "source" source "expected" "WELL-FORMEDNESS-VIOLATION" "condition" (cq-condition c)) *cr-controls*)))
127 (let ((raw (concatenate '(vector (unsigned-byte 8)) (bytes (cq-wrap "")) #(255))))
128 (let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context raw)))))
129 (check "resource-c14n-security:invalid-UTF8-rejected" (typep c 'cxml:well-formedness-violation))
130 (push (mm-fields "id" "invalid-UTF8" "inputHex" (cr-hex raw) "condition" (cq-condition c)) *cr-controls*)))
131 (dolist (marker '(cxml::*topic-maps-encoding-patch* runes-encoding::*topic-maps-encoding-patch*))
132 (let ((c (progv (list marker) '(nil)
133 (condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes "<x/>")))))))
134 (check (format nil "resource-c14n-security:qualification-required:~S" marker) (typep c 'topic-maps.xml:unqualified-parser))
135 (push (mm-fields "id" (format nil "marker:~S" marker) "condition" (cq-condition c)) *cr-controls*)))
136 ;; Observe the actual new parser call, including the deny function. Calling it
137 ;; directly proves it signals rather than returning NIL (which enables OPEN).
138 (let ((original (symbol-function 'cxml:parse-octets)) (args nil))
139 (unwind-protect
140 (progn
141 (setf (symbol-function 'cxml:parse-octets)
142 (lambda (raw handler &rest options) (setf args options) (apply original raw handler options)))
143 (topic-maps.xml:parse-canonical-context (bytes "<x/>")))
144 (setf (symbol-function 'cxml:parse-octets) original))
145 (check "resource-c14n-security:actual-parser-options"
146 (and (null (getf args :validate)) (getf args :disallow-internal-subset)
147 (eq (getf args :entity-resolver) #'topic-maps.xml::deny-external-resource)))
148 (dolist (uri '("file:///tmp/no-resource-c14n-read" "https://example.invalid/no-resource-c14n"))
149 (let ((c (condition-from (lambda () (funcall (getf args :entity-resolver) nil uri)))))
150 (check (format nil "resource-c14n-security:resolver-never-returns:~A" uri)
151 (and (typep c 'topic-maps.xml:external-resource-forbidden) (equal uri (topic-maps.xml:denied-system-id c))))
152 (push (mm-fields "id" uri "condition" (cq-condition c)) *cr-controls*))))
153 (let* ((symbols '(sax:*namespace-processing* sax:*include-xmlns-attributes* sax:*use-xmlns-namespace* cxml:*catalog* cxml:*cache-all-dtds* cxml:*dtd-cache*))
154 (before (mapcar #'symbol-value symbols)))
155 (let ((sax:*namespace-processing* nil) (sax:*include-xmlns-attributes* nil) (sax:*use-xmlns-namespace* nil))
156 (check "resource-c14n-security:explicit-context-overrides-ambient-parser-flags"
157 (equal "<p:x xmlns:p=\"urn:p\"></p:x>"
158 (topic-maps.xml:canonicalize-resource-data (topic-maps.xml:parse-canonical-context (bytes (cq-wrap "<p:x/>" " xmlns:p='urn:p'")))))))
159 (check "resource-c14n-security:all-global-parser-settings-unchanged" (equal before (mapcar #'symbol-value symbols))))
160 (let ((c (condition-from (lambda () (topic-maps.xml:parse-octets (bytes "<x/>") (cxml-dom:make-dom-builder))))))
161 (check "resource-c14n-security:old-XML-byte-boundary-still-unchanged"
162 (and c (search "SAX configuration is incompatible with DOM" (princ-to-string c)))))
163 (dolist (source (list (cq-wrap "<x xmlns:p='relative'/>")
164 "<outer xmlns:p='relative'><xtm:resourceData xmlns:xtm='http://www.topicmaps.org/xtm/'/></outer>"))
165 (let ((c (condition-from (lambda () (topic-maps.xml:canonicalize-resource-data (topic-maps.xml:parse-canonical-context (bytes source)))))))
166 (check (format nil "resource-c14n-security:relative-namespace-rejected:~A" source)
167 (and (typep c 'topic-maps.xml:canonicalization-error) (eq :relative-namespace-uri (topic-maps.xml:canonicalization-error-code c))))
168 (push (mm-fields "source" source "condition" (cq-condition c)) *cr-controls*)))
169 (let* ((ctx (topic-maps.xml:parse-canonical-context (bytes "<x/>")))
170 (c (condition-from (lambda () (topic-maps.xml:canonicalize-resource-data ctx)))))
171 (check "resource-c14n-api:missing-resourceData-is-an-explicit-error"
172 (and (zerop (topic-maps.xml:resource-data-count ctx)) (typep c 'topic-maps.xml:canonicalization-error)
173 (eq :resource-data-index (topic-maps.xml:canonicalization-error-code c)))))
174 (check "resource-c14n-api:raw-context-constructor-not-exported"
175 (not (eq :external (nth-value 1 (find-symbol "%MAKE-CANONICAL-CONTEXT" :topic-maps.xml)))))
176 t)
177
178 (defun cr-tests ()
179 (setf *cr-results* nil *cr-native* nil *cr-controls* nil *cr-executed* nil)
180 ;; Reuse all existing 22Q input/expected strings; never edit its oracle.
181 (dolist (spec (cq-specs))
182 (destructuring-bind (id source expected rule) spec
183 (cr-probe id (bytes source) expected (concatenate 'string "unchanged Slice 22Q: " rule))))
184 (dolist (encoding '(:utf-16le :utf-16be))
185 (destructuring-bind (id source expected rule) (find "unicode" (cq-specs) :key #'first :test #'equal)
186 (cr-probe (format nil "~A/~A" id encoding)
187 (bytes (concatenate 'string "<?xml version='1.0' encoding='UTF-16'?>" source) encoding t) expected
188 (concatenate 'string "unchanged Slice 22Q: " rule))))
189 (dolist (spec (cr-extra-specs))
190 (destructuring-bind (id source expected rule) spec (cr-probe id (bytes source) expected rule)))
191 (dolist (spec '(("UTF8-BOM" :utf-8 t "UTF-8") ("UTF16LE-no-BOM" :utf-16le nil "UTF-16LE")
192 ("UTF16BE-no-BOM" :utf-16be nil "UTF-16BE") ("ISO8859-1" :iso-8859-1 nil "ISO-8859-1")))
193 (destructuring-bind (id encoding bom declaration) spec
194 (let ((text (if (eq encoding :iso-8859-1) "<x>Grüße ©</x>" "<x>Grüße 水 🦊</x>")))
195 (cr-probe (concatenate 'string "encoding/" id)
196 (bytes (concatenate 'string (format nil "<?xml version='1.0' encoding='~A'?>" declaration) (cq-wrap text)) encoding bom)
197 text "Qualified source encoding; mandatory UTF8 output, no BOM"))))
198 (dolist (spec (cr-official-vectors))
199 (destructuring-bind (id raw expected source) spec
200 (cr-probe id raw expected source :root-vector t)
201 ;; Also exercise the PUBLIC adapter with the independently published
202 ;; expectation. Adaptation: insert only the omitted resourceData wrapper;
203 ;; keep the XML declaration and omit outside-root prolog whitespace.
204 ;; All three original inputs contain ASCII only, including the ISO-8859-1
205 ;; vector's numeric character reference, so this transport is lossless.
206 (let* ((text (babel:octets-to-string raw :encoding :utf-8))
207 (end (search "?>" text))
208 (declaration (if end (subseq text 0 (+ end 2)) ""))
209 (content (if end (string-left-trim '(#\Space #\Tab #\Newline #\Return) (subseq text (+ end 2))) text)))
210 (cr-probe (concatenate 'string id "/XTM-wrapper")
211 (bytes (concatenate 'string declaration (cq-wrap content))) expected
212 (concatenate 'string source "; adaptation: wrapper inserted, outside-root whitespace omitted; expectation unchanged")))))
213 (let ((ctx (topic-maps.xml:parse-canonical-context
214 (bytes "<x xmlns:t='http://www.topicmaps.org/xtm/' xmlns:p='urn:p'><t:resourceData><p:a/></t:resourceData><resourceData>ignore</resourceData><t:resourceData>B</t:resourceData></x>"))))
215 (check "resource-c14n-api:expanded-name-and-document-order"
216 (and (= 2 (topic-maps.xml:resource-data-count ctx))
217 (equal "<p:a xmlns:p=\"urn:p\"></p:a>" (topic-maps.xml:canonicalize-resource-data ctx :index 0))
218 (equal "B" (topic-maps.xml:canonicalize-resource-data ctx :index 1)))))
219 (cr-security-tests)
220 (setf *cr-results* (nreverse *cr-results*) *cr-native* (nreverse *cr-native*) *cr-controls* (nreverse *cr-controls*) *cr-executed* t)
221 t)
222 (defun cr-feature-report ()
223 (mm-fields "kind" "XTM-4.16.3-closed-XML-C14N-1.0-infrastructure"
224 "executed" (if *cr-executed* :true :false) "qualifiedForClosedXTMNodeSet" (if *cr-executed* :true :false)
225 "generalXPathC14N" :false "XTMReaderIntegration" :false "corpusRegistrationsAdded" 0
226 "probes" (coerce *cr-results* 'vector) "nativeValues" (coerce *cr-native* 'vector)
227 "controls" (coerce *cr-controls* 'vector)))