(in-package #:topic-maps.xtm.tests)
(defvar *cr-results* nil)
(defvar *cr-native* nil)
(defvar *cr-controls* nil)
(defvar *cr-executed* nil)
(defun cr-hex (octets) (format nil "~{~2,'0X~}" (coerce octets 'list)))
(defun cr-dom-state (node)
(list (dom:node-type node) (dom:node-name node) (dom:node-value node)
(when (eq (dom:node-type node) :element)
(loop for i below (dom:length (dom:attributes node)) for a = (dom:item (dom:attributes node) i)
collect (list (dom:name a) (dom:namespace-uri a) (dom:local-name a) (dom:value a))))
(loop for i below (dom:length (dom:child-nodes node)) collect (cr-dom-state (dom:item (dom:child-nodes node) i)))))
(defun cr-native (id value)
(let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:cr:owner"))
(type (intern-topic map :subject-identifier "urn:cr:type"))
(nt (intern-topic map :subject-identifier "urn:cr:name-type"))
(base (intern-topic map :subject-identifier "urn:cr:base"))
(extra (intern-topic map :subject-identifier "urn:cr:extra"))
(name (add-name owner "N" :type nt :scope (list base)))
(dt "http://www.w3.org/2001/XMLSchema#anyType")
(o (add-occurrence owner type value :datatype dt :scope (list extra)))
(v (add-variant name value :datatype dt :scope (list base extra)))
(before (ik-state map))
(a (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cr:map"))
(b (topic-maps.cxtm:canonicalize-octets map :base-locator "urn:cr:map")))
(check (format nil "resource-c14n-native:~A:lossless-occurrence-variant" id)
(and (equal value (occurrence-value o)) (equal value (variant-value v))
(equal dt (occurrence-datatype o)) (equal dt (variant-datatype v))))
(check (format nil "resource-c14n-native:~A:canonical-parent-type-scope-EQ" id)
(and (eq owner (occurrence-parent o)) (eq type (occurrence-type o))
(eq name (variant-parent v)) (eq o (canonical-occurrence o)) (eq v (canonical-variant v))
(scope-eq-p (list extra) (occurrence-scope o)) (scope-eq-p (list base extra) (variant-scope v))))
(check (format nil "resource-c14n-native:~A:pure-CXTM-no-topics-from-XML" id)
(and (= 5 (length (topic-map-topics map))) (equalp a b) (equal before (ik-state map))))
(push (mm-fields "id" id "value" value "datatype" dt "occurrenceValue" (occurrence-value o)
"variantValue" (variant-value v) "canonicalReferencesEQ" :true "topicCount" 5
"repeatedCXTMPure" :true "readerIntegration" :false "actualCXTMHex" (cr-hex a)) *cr-native*)))
(defun cr-probe (id raw expected origin &key root-vector)
(let* ((copy (copy-seq raw)) (ctx (topic-maps.xml:parse-canonical-context raw))
(doc (topic-maps.xml::%canonical-context-document ctx)) (state (cr-dom-state doc))
(want (bytes expected)))
(flet ((run ()
(if root-vector
;; Independent W3C examples with document root selected. This
;; private engine is the SAME engine as the public XTM adapter.
;; The examples use all their namespace bindings, so XTM's
;; filter makes no change to their published node set/result.
(let ((s (topic-maps.xml::%canonical-subtrees (list (dom:document-element doc)))))
(values s (bytes s)))
(topic-maps.xml:canonicalize-resource-data ctx))))
(multiple-value-bind (value actual) (run)
(multiple-value-bind (again repeated) (run)
(check (format nil "resource-c14n:~A:exact-independent-bytes" id) (equalp want actual))
(check (format nil "resource-c14n:~A:UTF8-string-and-repeat" id)
(and (equal expected value) (equalp actual (bytes value)) (equal value again) (equalp actual repeated))))
(check (format nil "resource-c14n:~A:input-context-unchanged" id)
(and (equalp raw copy) (equal state (cr-dom-state doc))))
(unless root-vector (cr-native id value))
(push (mm-fields "id" id "origin" origin "inputHex" (cr-hex raw) "expectedHex" (cr-hex want)
"actualHex" (cr-hex actual) "expectedXML" expected "actualXML" value
"expectedBytes" (length want) "actualBytes" (length actual) "firstDifferentByte" (mismatch want actual)
"byteIdentical" :true "repeatedPure" :true "publicXTMProjection" (if root-vector :false :true)) *cr-results*)))))
(defun cr-extra-specs ()
(list
(list "prefix-shadowing" (cq-wrap "" " xmlns:p='urn:one'")
"" "C14N 2.3: original prefixes, nearest namespace bindings")
(list "default-namespace-changes" (cq-wrap "" " xmlns='urn:outer'")
"" "C14N 2.3: selected-parent default namespace and empty reset")
(list "global-namespace-filter" (cq-wrap "" " xmlns:p='urn:p' xmlns:q='urn:unused'")
"" "XTM 4.16.3: namespace pair used anywhere in selected set; not Exclusive C14N")
(list "same-URI-different-prefix" (cq-wrap "" " xmlns:p='urn:same' xmlns:q='urn:same' xmlns:z='urn:same'")
"" "XTM 4.16.3 filters by URI AND prefix, not URI alone")
(list "QName-attribute-value-not-namespace-use" (cq-wrap "" " xmlns:p='urn:p'")
"" "Only element/attribute names establish namespace use, not values")
(list "filtered-parent-namespace-gap" (cq-wrap "" " xmlns:p='urn:one'")
"" "C14N 2.3 compares namespace nodes of nearest selected parent, not cumulative rendered map")
(list "xml-attributes-inherited" ""
"" "C14N 1.0 2.4: nearest inherited xml:* on each apex; local override")
(list "xml-base-is-1.0-not-1.1" ""
"" "C14N 1.0 2.4: all xml:* inherited, no 1.1 base fixup/id exception")
(list "empty-content" (cq-wrap "") "" "XTM descendant node set may be empty")
(list "empty-CDATA" (cq-wrap "") "" "Empty character information contributes no bytes")
(list "text-only" (cq-wrap " A&B<C>"
")
(format nil " A&B<C>\"~C~C
" #\Tab #\Newline) "XTM selected characters, C14N text escaping without container")
(list "several-apices-mixed-text" (cq-wrap "abc") "abc" "Selected document order, no invented fragment wrapper")
(list "attribute-order-same-URI" (cq-wrap "")
"" "C14N 2.2: codepoint order, namespace URI then local name")
(list "attribute-greater-than-and-whitespace" (cq-wrap (format nil "" #\Tab #\Newline))
""
\">" "XML whitespace normalization vs character references; > not escaped in attributes")
(list "no-Unicode-normalization-UTF8" (cq-wrap (format nil "e~C" (code-char #x301)))
(format nil "e~C" (code-char #x301)) "C14N 1.0 4.2: no normalization of UCS-based source character data")
(list "in-content-BOM-is-text" (cq-wrap (format nil "A~CB" (code-char #xfeff)))
(format nil "A~CB" (code-char #xfeff)) "Only encoding BOM is removed, not U+FEFF in character content")))
(defun cr-official-vectors ()
;; Independent normative examples. The original full examples 3.1/3.3/3.4/
;; 3.5/3.7 contain a DTD or arbitrary XPath selection and are NOT claimed run.
;; The e5 subvector is copied as a complete element with all its namespaces;
;; it has no dependency on the e9 default attribute in the enclosing DTD.
(let ((ws (format nil "~% ~% A B ~% ~% A~% ~% B~% A B ~% C~% ~%")))
(list
(list "w3c-3.2-whitespace" (bytes ws) ws "https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-WhitespaceInContent; complete unchanged example; root node set")
(list "w3c-3.3-e5-subvector" (bytes "")
""
"https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-SETags; independent e5 element/output excerpt; NOT the DTD-containing full example")
(list "w3c-3.6-ISO8859-1" (bytes (format nil "~%©") :iso-8859-1)
"©" "https://www.w3.org/TR/2001/REC-xml-c14n-20010315#Example-UTF8; complete original input; published C2 A9 byte expectation"))))
(defun cr-security-tests ()
(dolist (source '("" "]>&a;"
""
""
"]>&a;"))
(let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes source))))))
(check (format nil "resource-c14n-security:DOCTYPE-denied:~A" source) (typep c 'topic-maps.xml:doctype-forbidden))
(push (mm-fields "source" source "expected" "DOCTYPE-FORBIDDEN" "condition" (cq-condition c)) *cr-controls*)))
(dolist (source '("" ""
"" "&unknown;" ""))
(let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes source))))))
(check (format nil "resource-c14n-security:XML-rejected:~A" source) (typep c 'cxml:well-formedness-violation))
(push (mm-fields "source" source "expected" "WELL-FORMEDNESS-VIOLATION" "condition" (cq-condition c)) *cr-controls*)))
(let ((raw (concatenate '(vector (unsigned-byte 8)) (bytes (cq-wrap "")) #(255))))
(let ((c (condition-from (lambda () (topic-maps.xml:parse-canonical-context raw)))))
(check "resource-c14n-security:invalid-UTF8-rejected" (typep c 'cxml:well-formedness-violation))
(push (mm-fields "id" "invalid-UTF8" "inputHex" (cr-hex raw) "condition" (cq-condition c)) *cr-controls*)))
(dolist (marker '(cxml::*topic-maps-encoding-patch* runes-encoding::*topic-maps-encoding-patch*))
(let ((c (progv (list marker) '(nil)
(condition-from (lambda () (topic-maps.xml:parse-canonical-context (bytes "")))))))
(check (format nil "resource-c14n-security:qualification-required:~S" marker) (typep c 'topic-maps.xml:unqualified-parser))
(push (mm-fields "id" (format nil "marker:~S" marker) "condition" (cq-condition c)) *cr-controls*)))
;; Observe the actual new parser call, including the deny function. Calling it
;; directly proves it signals rather than returning NIL (which enables OPEN).
(let ((original (symbol-function 'cxml:parse-octets)) (args nil))
(unwind-protect
(progn
(setf (symbol-function 'cxml:parse-octets)
(lambda (raw handler &rest options) (setf args options) (apply original raw handler options)))
(topic-maps.xml:parse-canonical-context (bytes "")))
(setf (symbol-function 'cxml:parse-octets) original))
(check "resource-c14n-security:actual-parser-options"
(and (null (getf args :validate)) (getf args :disallow-internal-subset)
(eq (getf args :entity-resolver) #'topic-maps.xml::deny-external-resource)))
(dolist (uri '("file:///tmp/no-resource-c14n-read" "https://example.invalid/no-resource-c14n"))
(let ((c (condition-from (lambda () (funcall (getf args :entity-resolver) nil uri)))))
(check (format nil "resource-c14n-security:resolver-never-returns:~A" uri)
(and (typep c 'topic-maps.xml:external-resource-forbidden) (equal uri (topic-maps.xml:denied-system-id c))))
(push (mm-fields "id" uri "condition" (cq-condition c)) *cr-controls*))))
(let* ((symbols '(sax:*namespace-processing* sax:*include-xmlns-attributes* sax:*use-xmlns-namespace* cxml:*catalog* cxml:*cache-all-dtds* cxml:*dtd-cache*))
(before (mapcar #'symbol-value symbols)))
(let ((sax:*namespace-processing* nil) (sax:*include-xmlns-attributes* nil) (sax:*use-xmlns-namespace* nil))
(check "resource-c14n-security:explicit-context-overrides-ambient-parser-flags"
(equal ""
(topic-maps.xml:canonicalize-resource-data (topic-maps.xml:parse-canonical-context (bytes (cq-wrap "" " xmlns:p='urn:p'")))))))
(check "resource-c14n-security:all-global-parser-settings-unchanged" (equal before (mapcar #'symbol-value symbols))))
(let ((c (condition-from (lambda () (topic-maps.xml:parse-octets (bytes "") (cxml-dom:make-dom-builder))))))
(check "resource-c14n-security:old-XML-byte-boundary-still-unchanged"
(and c (search "SAX configuration is incompatible with DOM" (princ-to-string c)))))
(dolist (source (list (cq-wrap "")
""))
(let ((c (condition-from (lambda () (topic-maps.xml:canonicalize-resource-data (topic-maps.xml:parse-canonical-context (bytes source)))))))
(check (format nil "resource-c14n-security:relative-namespace-rejected:~A" source)
(and (typep c 'topic-maps.xml:canonicalization-error) (eq :relative-namespace-uri (topic-maps.xml:canonicalization-error-code c))))
(push (mm-fields "source" source "condition" (cq-condition c)) *cr-controls*)))
(let* ((ctx (topic-maps.xml:parse-canonical-context (bytes "")))
(c (condition-from (lambda () (topic-maps.xml:canonicalize-resource-data ctx)))))
(check "resource-c14n-api:missing-resourceData-is-an-explicit-error"
(and (zerop (topic-maps.xml:resource-data-count ctx)) (typep c 'topic-maps.xml:canonicalization-error)
(eq :resource-data-index (topic-maps.xml:canonicalization-error-code c)))))
(check "resource-c14n-api:raw-context-constructor-not-exported"
(not (eq :external (nth-value 1 (find-symbol "%MAKE-CANONICAL-CONTEXT" :topic-maps.xml)))))
t)
(defun cr-tests ()
(setf *cr-results* nil *cr-native* nil *cr-controls* nil *cr-executed* nil)
;; Reuse all existing 22Q input/expected strings; never edit its oracle.
(dolist (spec (cq-specs))
(destructuring-bind (id source expected rule) spec
(cr-probe id (bytes source) expected (concatenate 'string "unchanged Slice 22Q: " rule))))
(dolist (encoding '(:utf-16le :utf-16be))
(destructuring-bind (id source expected rule) (find "unicode" (cq-specs) :key #'first :test #'equal)
(cr-probe (format nil "~A/~A" id encoding)
(bytes (concatenate 'string "" source) encoding t) expected
(concatenate 'string "unchanged Slice 22Q: " rule))))
(dolist (spec (cr-extra-specs))
(destructuring-bind (id source expected rule) spec (cr-probe id (bytes source) expected rule)))
(dolist (spec '(("UTF8-BOM" :utf-8 t "UTF-8") ("UTF16LE-no-BOM" :utf-16le nil "UTF-16LE")
("UTF16BE-no-BOM" :utf-16be nil "UTF-16BE") ("ISO8859-1" :iso-8859-1 nil "ISO-8859-1")))
(destructuring-bind (id encoding bom declaration) spec
(let ((text (if (eq encoding :iso-8859-1) "Grüße ©" "Grüße 水 🦊")))
(cr-probe (concatenate 'string "encoding/" id)
(bytes (concatenate 'string (format nil "" declaration) (cq-wrap text)) encoding bom)
text "Qualified source encoding; mandatory UTF8 output, no BOM"))))
(dolist (spec (cr-official-vectors))
(destructuring-bind (id raw expected source) spec
(cr-probe id raw expected source :root-vector t)
;; Also exercise the PUBLIC adapter with the independently published
;; expectation. Adaptation: insert only the omitted resourceData wrapper;
;; keep the XML declaration and omit outside-root prolog whitespace.
;; All three original inputs contain ASCII only, including the ISO-8859-1
;; vector's numeric character reference, so this transport is lossless.
(let* ((text (babel:octets-to-string raw :encoding :utf-8))
(end (search "?>" text))
(declaration (if end (subseq text 0 (+ end 2)) ""))
(content (if end (string-left-trim '(#\Space #\Tab #\Newline #\Return) (subseq text (+ end 2))) text)))
(cr-probe (concatenate 'string id "/XTM-wrapper")
(bytes (concatenate 'string declaration (cq-wrap content))) expected
(concatenate 'string source "; adaptation: wrapper inserted, outside-root whitespace omitted; expectation unchanged")))))
(let ((ctx (topic-maps.xml:parse-canonical-context
(bytes "ignoreB"))))
(check "resource-c14n-api:expanded-name-and-document-order"
(and (= 2 (topic-maps.xml:resource-data-count ctx))
(equal "" (topic-maps.xml:canonicalize-resource-data ctx :index 0))
(equal "B" (topic-maps.xml:canonicalize-resource-data ctx :index 1)))))
(cr-security-tests)
(setf *cr-results* (nreverse *cr-results*) *cr-native* (nreverse *cr-native*) *cr-controls* (nreverse *cr-controls*) *cr-executed* t)
t)
(defun cr-feature-report ()
(mm-fields "kind" "XTM-4.16.3-closed-XML-C14N-1.0-infrastructure"
"executed" (if *cr-executed* :true :false) "qualifiedForClosedXTMNodeSet" (if *cr-executed* :true :false)
"generalXPathC14N" :false "XTMReaderIntegration" :false "corpusRegistrationsAdded" 0
"probes" (coerce *cr-results* 'vector) "nativeValues" (coerce *cr-native* 'vector)
"controls" (coerce *cr-controls* 'vector)))