]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/xml-c14n-preflight.lisp
feat(tmdm): readers for the TMDM 7.2/7.3 relationships; tests for both default name...
[topic-maps.git] / tests / xml-c14n-preflight.lisp
1 (in-package #:topic-maps.xtm.tests)
2 ;; Characterization only: use the existing walker/sink without implementing
3 ;; namespace repair, node-set projection, sorting, escaping or C14N ourselves.
4 (defvar *cq-results* nil)
5 (defvar *cq-native* nil)
6 (defvar *cq-api* nil)
7 ;; Exact observed subset, frozen after the external exploratory run.
8 ;; These are characterization expectations, NOT an alternative C14N oracle.
9 (defparameter *cq-known-matches*
10 '("simple" "attributes-ba" "attributes-ab" "nested" "comments"
11 "lexical-cdata" "lexical-escaped" "unicode"))
12 (defun cq-wrap (content &optional (attributes ""))
13 (format nil "<xtm:resourceData xmlns:xtm='http://www.topicmaps.org/xtm/' datatype='http://www.w3.org/2001/XMLSchema#anyType'~A>~A</xtm:resourceData>" attributes content))
14 (defun cq-specs ()
15 (list
16 (list "simple" (cq-wrap "<x>Hello</x>") "<x>Hello</x>" "XTM 4.16.3; C14N 2.3: element/text, omit container")
17 (list "attributes-ba" (cq-wrap "<x b='2' a='1'/>") "<x a=\"1\" b=\"2\"></x>" "C14N 2.3: unqualified attribute order and expanded empty element")
18 (list "attributes-ab" (cq-wrap "<x a='1' b='2'></x>") "<x a=\"1\" b=\"2\"></x>" "Same node properties as attributes-ba")
19 (list "inherited-prefix" (cq-wrap "<p:x>A</p:x>" " xmlns:p='urn:p' xmlns:u='urn:unused'") "<p:x xmlns:p=\"urn:p\">A</p:x>" "XTM 4.16.3: needed inherited prefix retained; unused mapping removed")
20 (list "unused-local-namespace" (cq-wrap "<p:x xmlns:p='urn:p' xmlns:u='urn:unused'>A</p:x>") "<p:x xmlns:p=\"urn:p\">A</p:x>" "XTM 4.16.3: unused namespace absent from selected namespace set")
21 (list "inherited-default" (cq-wrap "<x>A</x>" " xmlns='urn:default'") "<x xmlns=\"urn:default\">A</x>" "C14N 2.3-2.4: selected child must retain inherited default namespace")
22 (list "nested" (cq-wrap "<x><y b='2' a='1'>Z</y><empty/></x>") "<x><y a=\"1\" b=\"2\">Z</y><empty></empty></x>" "C14N 2.3: nested elements, attributes, empty element")
23 (list "escaping" (cq-wrap "<x a='&amp;&lt;&quot;&#x9;&#xA;&#xD;'>A&amp;B&lt;C&gt;&quot;&#x9;&#xA;&#xD;</x>")
24 (format nil "<x a=\"&amp;&lt;&quot;&#x9;&#xA;&#xD;\">A&amp;B&lt;C&gt;\"~C~C&#xD;</x>" #\Tab #\Newline)
25 "C14N 2.3: attribute vs text escapes, hexadecimal character references")
26 (list "whitespace-line-endings" (cq-wrap (format nil "<x> a~C~C b~C c~C~C</x>" #\Return #\Newline #\Return #\Tab #\Newline))
27 (format nil "<x> a~% b~% c~C~%</x>" #\Tab) "XML 2.11; C14N 2.1/2.3: normalize CR/CRLF, retain text whitespace")
28 (list "comments" (cq-wrap "<x>A<!--excluded-->B</x>") "<x>AB</x>" "XTM 4.16.3: comments=false")
29 (list "lexical-cdata" (cq-wrap "<x a='&#65;'><y/><![CDATA[a&b]]></x>") "<x a=\"A\"><y></y>a&amp;b</x>" "C14N 2.1/2.3: references, CDATA and tag spelling have equal selected values")
30 (list "lexical-escaped" (cq-wrap "<x a=\"A\"><y></y>a&amp;b</x>") "<x a=\"A\"><y></y>a&amp;b</x>" "Same selected node properties as lexical-cdata")
31 (list "unicode" (cq-wrap "<x>Grüße 水 🦊</x>") "<x>Grüße 水 🦊</x>" "C14N 2.1: strict UTF-8 output for scalar character data")
32 (list "processing-instruction" (cq-wrap "<x>A<?p excluded?>B</x>") "<x>AB</x>" "XTM selects element/attribute/character nodes, not PI nodes")
33 (list "namespace-attribute-order" (cq-wrap "<x xmlns:z='urn:a' xmlns:a='urn:z' z:b='1' a:a='2' plain='0'/>")
34 "<x xmlns:a=\"urn:z\" xmlns:z=\"urn:a\" plain=\"0\" z:b=\"1\" a:a=\"2\"></x>"
35 "C14N 2.3: namespace nodes before attributes, attributes by namespace URI/local name")
36 (list "redundant-binding" (cq-wrap "<p:x xmlns:p='urn:p'><p:y xmlns:p='urn:p'/></p:x>")
37 "<p:x xmlns:p=\"urn:p\"><p:y></p:y></p:x>" "C14N 2.3: suppress already rendered identical namespace binding")))
38 (defun cq-parse-dom (raw)
39 ;; Diagnostic-only CXML configuration. The productive XML wrapper remains
40 ;; unchanged and is NOT claimed compatible with DOM. Same closed-handler,
41 ;; no DOCTYPE, no resolver fallback; DOM requires xmlns attributes and URIs.
42 (assert (equal (symbol-value 'cxml::*topic-maps-encoding-patch*) "19d0b-1"))
43 (assert (equal (symbol-value 'runes-encoding::*topic-maps-encoding-patch*) "19d0b-1"))
44 (let ((sax:*namespace-processing* t) (sax:*include-xmlns-attributes* t)
45 (sax:*use-xmlns-namespace* t) (cxml:*catalog* nil) (cxml:*cache-all-dtds* nil)
46 (cxml:*dtd-cache* (cxml:make-dtd-cache)))
47 (cxml:parse-octets raw
48 (make-instance 'topic-maps.xml::closed-handler :chained-handler (cxml-dom:make-dom-builder))
49 :validate nil :disallow-internal-subset t :entity-resolver #'topic-maps.xml::deny-external-resource)))
50 (defun cq-condition (c)
51 (when c (mm-fields "class" (string (type-of c)) "detail" (princ-to-string c))))
52 (defun cq-write (path value)
53 (with-open-file (s path :direction :output :if-exists :supersede :if-does-not-exist :create)
54 (topic-maps.test-foundation::emit-json-value value s) (terpri s)))
55 (defun cq-native-probe (id value)
56 (let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:cq:owner"))
57 (type (intern-topic map :subject-identifier "urn:cq:type"))
58 (name-type (intern-topic map :subject-identifier "urn:cq:name-type"))
59 (base (intern-topic map :subject-identifier "urn:cq:base"))
60 (theme (intern-topic map :subject-identifier "urn:cq:extra"))
61 (name (add-name owner "N" :type name-type :scope (list base)))
62 (count (length (topic-map-topics map)))
63 (datatype "http://www.w3.org/2001/XMLSchema#anyType")
64 (o (add-occurrence owner type value :datatype datatype :scope (list theme)))
65 (v (add-variant name value :datatype datatype :scope (list base theme)))
66 (before (ik-state map))
67 (a (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cq:base-map"))
68 (b (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cq:base-map")))
69 (check (format nil "c14n-native:~A:occurrence-value-datatype-parent-EQ" id)
70 (and (equal value (occurrence-value o)) (equal datatype (occurrence-datatype o))
71 (eq owner (occurrence-parent o)) (eq type (occurrence-type o)) (eq o (canonical-occurrence o))))
72 (check (format nil "c14n-native:~A:variant-value-datatype-parent-EQ" id)
73 (and (equal value (variant-value v)) (equal datatype (variant-datatype v))
74 (eq name (variant-parent v)) (eq v (canonical-variant v))))
75 (check (format nil "c14n-native:~A:scopes-and-no-topics-from-XML" id)
76 (and (= count (length (topic-map-topics map))) (scope-eq-p (list theme) (occurrence-scope o))
77 (scope-eq-p (list base theme) (variant-scope v))))
78 (check (format nil "c14n-native:~A:repeated-CXTM-preserves-all-properties" id)
79 (and (equalp a b) (equal before (ik-state map)) (equal value (occurrence-value o)) (equal value (variant-value v))))
80 (push (mm-fields "id" id "origin" "actual candidate bytes equal the independent oracle for this probe only"
81 "readerIntegration" :false "datatype" datatype "occurrenceValue" (occurrence-value o) "variantValue" (variant-value v)
82 "topicCountBefore" count "topicCountAfter" (length (topic-map-topics map))
83 "canonicalParentTypeScopeEQ" :true "repeatedCXTMIdentical" :true "modelUnchanged" :true
84 "actualCXTM" (babel:octets-to-string a :encoding :utf-8)) *cq-native*)))
85 (defun cq-probe (spec &optional (encoding :utf-8) bom)
86 (destructuring-bind (label source expected rule) spec
87 (let* ((id (if (eq encoding :utf-8) label (format nil "~A/~A" label encoding)))
88 (raw (bytes source encoding bom)) (copy (copy-seq raw))
89 (dom (cq-parse-dom raw))
90 (container (dom:document-element dom))
91 ;; Existing DOM child walker, applied experimentally to a container.
92 ;; No custom XML serialization/C14N, no namespace repair/filter.
93 (actual (dom:map-document (cxml:make-octet-vector-sink :canonical t) container :include-xmlns-attributes t))
94 (second-form (dom:map-document (cxml:make-octet-vector-sink :canonical 2) container :include-xmlns-attributes t))
95 (whole (dom:map-document (cxml:make-octet-vector-sink :canonical t) dom :include-xmlns-attributes t))
96 (want (bytes expected)) (match (equalp actual want))
97 (string (babel:octets-to-string actual :encoding :utf-8 :errorp t)))
98 (check (format nil "c14n-probe:~A:input-bytes-unchanged" id) (equalp raw copy))
99 (check (format nil "c14n-probe:~A:UTF8-string-roundtrip" id) (equalp actual (bytes string)))
100 (when *cq-known-matches*
101 (check (format nil "c14n-characterization:~A:observed-match-or-limit" id)
102 (eq match (not (null (member label *cq-known-matches* :test #'equal))))))
103 (push (mm-fields "id" id "inputXML" source "inputEncoding" (string encoding)
104 "inputHex" (format nil "~{~2,'0X~}" (coerce raw 'list))
105 "API" "diagnostic CXML DOM configuration with closed-handler/DOCTYPE denial; dom:map-document canonical sink on container child nodes"
106 "candidateOption" "cxml:make-octet-vector-sink :canonical T"
107 "subsetAPIQualified" :false "expectedXML" expected "expectedHex" (format nil "~{~2,'0X~}" (coerce want 'list))
108 "normativeReason" rule "actualXML" string "actualHex" (format nil "~{~2,'0X~}" (coerce actual 'list))
109 "expectedBytes" (length want) "actualBytes" (length actual) "firstDifferentByte" (mismatch want actual)
110 "normativeComparison" (if match "PASS-for-this-probe" "BYTE-DIFFERENCE")
111 "W3C-C14N-1.0-qualified" :false "secondCanonicalFormXML" (babel:octets-to-string second-form :encoding :utf-8)
112 "secondCanonicalFormMatches" (if (equalp second-form want) :true :false)
113 "wholeDocumentCanonicalXML" (babel:octets-to-string whole :encoding :utf-8) "condition" nil) *cq-results*)
114 (when match (cq-native-probe id string)))))
115 (defun cq-tests ()
116 (setf *cq-results* nil *cq-native* nil *cq-api* nil)
117 (let ((c (condition-from (lambda () (topic-maps.xml:parse-octets (bytes "<x/>") (cxml-dom:make-dom-builder))))))
118 (check "c14n-api:productive-boundary-DOM-incompatibility-observed"
119 (and c (search "SAX configuration is incompatible with DOM" (princ-to-string c))))
120 (push (mm-fields "id" "productive-byte-boundary-DOM" "condition" (cq-condition c)) *cq-api*))
121 (dolist (text '("<!DOCTYPE x><x/>" "<!DOCTYPE x SYSTEM 'file:///tmp/no-cq-read'><x/>"))
122 (let ((c (condition-from (lambda () (cq-parse-dom (bytes text))))))
123 (check (format nil "c14n-api:diagnostic-DOM-DOCTYPE-denied:~A" text) (typep c 'topic-maps.xml:doctype-forbidden))
124 (push (mm-fields "id" text "condition" (cq-condition c)) *cq-api*)))
125 (dolist (spec (cq-specs)) (cq-probe spec))
126 (dolist (encoding '(:utf-16le :utf-16be))
127 (let* ((spec (copy-list (find "unicode" (cq-specs) :key #'first :test #'equal))))
128 (setf (second spec) (concatenate 'string "<?xml version='1.0' encoding='UTF-16'?>" (second spec)))
129 (cq-probe spec encoding t)))
130 (dolist (pair (list
131 (list "sink-has-no-node-set-keyword" (lambda () (cxml:make-octet-vector-sink :canonical t :node-set nil)))
132 (list "walker-has-no-node-set-keyword" (lambda ()
133 (let ((doc (cq-parse-dom (bytes "<x/>"))))
134 (dom:map-document (cxml:make-octet-vector-sink :canonical t) doc :node-set nil))))))
135 (let ((c (condition-from (second pair))))
136 (check (concatenate 'string "c14n-api:" (first pair)) (typep c 'program-error))
137 (push (mm-fields "id" (first pair) "condition" (cq-condition c)) *cq-api*)))
138 (setf *cq-results* (nreverse *cq-results*) *cq-native* (nreverse *cq-native*) *cq-api* (nreverse *cq-api*))
139 (dolist (pair '(("attributes-ba" "attributes-ab") ("lexical-cdata" "lexical-escaped")))
140 (let* ((a (position (first pair) (cq-specs) :test #'equal :key #'first))
141 (b (position (second pair) (cq-specs) :test #'equal :key #'first)))
142 ;; Compare actual bytes in the existing report objects; never repair them.
143 (check (format nil "c14n-characterization:paired-actual-bytes:~A/~A" (first pair) (second pair))
144 (equal (cdr (assoc "actualHex" (topic-maps.test-foundation::json-object-members (nth a *cq-results*)) :test #'equal))
145 (cdr (assoc "actualHex" (topic-maps.test-foundation::json-object-members (nth b *cq-results*)) :test #'equal))))))
146 (let ((path (uiop:getenv "TOPIC_MAPS_C14N_REPORT")))
147 (when path (cq-write path (mm-fields "kind" "slice-22q-existing-candidate-characterization-not-reader-support"
148 "W3CC14NQualified" :false "XTMReaderIntegration" :false "ownC14NImplementation" :false
149 "probes" (coerce *cq-results* 'vector) "nativeValues" (coerce *cq-native* 'vector)
150 "APILimits" (coerce *cq-api* 'vector) "checks" (coerce (reverse *checks*) 'vector)))))
151 t)
152
153 (defun cq-feature-report ()
154 (mm-fields "kind" "existing-component-characterization-not-XTM-support"
155 "executed" (if *cq-results* :true :false)
156 "W3CC14NQualified" :false "XTMReaderIntegration" :false
157 "probes" (coerce *cq-results* 'vector) "nativeValues" (coerce *cq-native* 'vector)
158 "APILimits" (coerce *cq-api* 'vector)))