1 ;;;; Slice 19D0: parser qualification only; no XTM deserializer or model.
3 (setf *compile-verbose* nil *load-verbose* nil)
4 (assert (string= (lisp-implementation-version) "2.5.4"))
5 (assert (asdf:find-system "cxml" nil))
6 (asdf:load-system "cxml")
8 (defpackage #:topic-maps.cxml-probe (:use #:cl))
9 (in-package #:topic-maps.cxml-probe)
12 (defvar *failed-checks* nil)
13 (defvar *resolver-calls* nil)
14 (defvar *fallback-calls* 0)
16 (defun check (name truth)
18 (unless truth (push name *failed-checks*))
19 (format t "CXML-CHECK-~A ~A~%" (if truth "PASS" "FAIL") name))
21 (defun expect-condition (name class thunk)
22 ;; An I/O failure or a different generic exception is never a PASS.
23 (let ((condition (handler-case (progn (funcall thunk) nil)
25 (check name (typep condition class))
26 (format t "CXML-REJECTION ~A ~A~%" name (type-of condition))))
28 (define-condition doctype-forbidden (error) ())
29 (define-condition external-resource-forbidden (error)
30 ((system-id :initarg :system-id :reader denied-system-id)))
31 (define-condition fallback-reached (error) ())
33 (defclass observation (sax:default-handler)
34 ((elements :initform nil :accessor elements)
35 (text :initform "" :accessor text)
36 ;; Enabled only in tests that exercise the resolver defense separately.
37 (allow-doctype :initform nil :initarg :allow-doctype :reader allow-doctype)))
39 (defmethod sax:start-element ((handler observation) uri local qname attributes)
40 (push (list uri local qname
42 (list (sax:attribute-namespace-uri a)
43 (sax:attribute-local-name a)
44 (sax:attribute-qname a) (sax:attribute-value a)))
48 (defmethod sax:characters ((handler observation) data)
49 (setf (text handler) (concatenate 'string (text handler) data)))
51 (defmethod sax:end-document ((handler observation)) handler)
53 (defmethod sax:start-dtd ((handler observation) name public-id system-id)
54 (declare (ignore name public-id system-id))
55 (unless (allow-doctype handler) (error 'doctype-forbidden)))
57 (defun deny-external-resource (public-id system-id)
58 (declare (ignore public-id))
59 (push (puri:render-uri system-id nil) *resolver-calls*)
60 ;; NIL would permit CXML's file-opening fallback (xml-parse.lisp:1064).
61 (error 'external-resource-forbidden :system-id system-id))
63 (defun parse-bytes (bytes &key resolver-probe)
64 (let ((sax:*namespace-processing* t)
65 (sax:*include-xmlns-attributes* nil)
67 (cxml:*cache-all-dtds* nil)
68 (cxml:*dtd-cache* (cxml:make-dtd-cache)))
70 bytes (make-instance 'observation :allow-doctype resolver-probe)
71 :validate nil :disallow-internal-subset (not resolver-probe)
72 :entity-resolver #'deny-external-resource)))
74 (defun octets (string &optional (encoding :utf-8) bom)
75 ;; Explicit BOM bytes: Babel's endian-specific encoders do not supply a
76 ;; BOM vector for :USE-BOM T. Encoding itself stays with the existing library.
77 (concatenate '(vector (unsigned-byte 8))
78 (if bom (ecase encoding
79 (:utf-8 #(239 187 191))
80 (:utf-16le #(255 254))
81 (:utf-16be #(254 255))) #())
82 (babel:string-to-octets string :encoding encoding :use-bom nil)))
84 (defun parse-text (string) (parse-bytes (octets string)))
86 (defun check-decoded-text (name bytes expected)
87 (handler-case (check name (string= (text (parse-bytes bytes)) expected))
90 (format t "CXML-UNEXPECTED-REJECTION ~A ~A: ~A~%" name (type-of c) c))))
92 (check "asdf-cxml-found-and-loaded" (find-package :cxml))
93 (dolist (name '("closure-common" "puri" "babel" "alexandria"
94 "trivial-features" "trivial-gray-streams"))
95 (check (concatenate 'string "asdf-dependency:" name)
96 (asdf:component-loaded-p (asdf:find-system name))))
98 (let* ((source "<x:root xmlns:x='urn:element' xmlns:y='urn:attr' plain='v' y:a='ä'>🦊</x:root>")
99 (doc (parse-text source))
100 (root (first (elements doc))))
101 (check "namespace-expanded-element" (equal (subseq root 0 3) '("urn:element" "root" "x:root")))
102 (check "unprefixed-attribute-has-no-namespace"
103 (member '(nil "plain" "plain" "v") (fourth root) :test #'equal))
104 (check "prefixed-attribute-namespace"
105 (member '("urn:attr" "a" "y:a" "ä") (fourth root) :test #'equal))
106 (check "unicode-astral-character" (string= (text doc) "🦊")))
107 (let ((roots (reverse (elements (parse-text "<r xmlns='urn:a'><s xmlns='urn:b'/><t/></r>")))))
108 (check "default-namespace-and-nested-rebinding"
109 (equal (mapcar #'first roots) '("urn:a" "urn:b" "urn:a"))))
111 (dolist (spec '(("utf8-no-bom" :utf-8 nil "UTF-8")
112 ("utf8-bom" :utf-8 t "UTF-8")
113 ("utf16le-bom" :utf-16le t "UTF-16")
114 ("utf16be-bom" :utf-16be t "UTF-16")
115 ("utf16le-explicit-no-bom" :utf-16le nil "UTF-16LE")
116 ("utf16be-explicit-no-bom" :utf-16be nil "UTF-16BE")))
117 (destructuring-bind (name encoding bom declaration) spec
118 (dolist (content '("Grüße 水" "Grüße 水 🦊"))
119 (let ((source (format nil "<?xml version='1.0' encoding='~A'?><r>~A</r>" declaration content)))
120 (check-decoded-text (concatenate 'string name (if (find #\🦊 content) ":astral" ":bmp"))
121 (octets source encoding bom) content)))))
122 (check-decoded-text "utf16-bom-without-declaration:bmp"
123 (octets "<r>水</r>" :utf-16le t) "水")
124 (check-decoded-text "utf16-bom-without-declaration:astral"
125 (octets "<r>水🦊</r>" :utf-16le t) "水🦊")
126 (check "predefined-entities-and-character-references"
127 (string= (text (parse-text "<r><>&"'ä🦊</r>")) "<>&\"'ä🦊"))
128 (check "xml-line-end-normalization"
129 (string= (text (parse-text (format nil "<r>a~C~Cb~Cc</r>" #\Return #\Newline #\Return)))
130 (format nil "a~%b~%c")))
131 (check "xml-attribute-whitespace-normalization"
132 (equal (fourth (first (elements (parse-text (format nil "<r a='a~Cb~Cc'/>" #\Tab #\Newline)))))
133 '((nil "a" "a" "a b c"))))
135 (dolist (pair '(("misnested" . "<r><a></r>")
136 ("duplicate-attribute" . "<r a='1' a='2'/>")
137 ("duplicate-expanded-attribute" . "<r xmlns:a='urn:x' xmlns:b='urn:x' a:v='1' b:v='2'/>")
138 ("undeclared-prefix" . "<q:r/>")
139 ("multiple-roots" . "<r/><s/>")
140 ("unclosed-root" . "<r>")
141 ("undefined-entity" . "<r>&missing;</r>")
142 ("forbidden-nul-reference" . "<r>�</r>")
143 ("forbidden-surrogate-reference" . "<r>�</r>")))
144 (expect-condition (car pair) 'cxml:well-formedness-violation
145 (lambda () (parse-text (cdr pair)))))
146 (expect-condition "invalid-utf8" 'cxml:well-formedness-violation
147 (lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
148 (octets "<r>") #(195 40) (octets "</r>")))))
150 (format t "CXML-OBSERVATION invalid-utf8 decoded:~{ U+~4,'0X~}~%"
151 (map 'list #'char-code
152 (text (parse-bytes (concatenate '(vector (unsigned-byte 8))
153 (octets "<r>") #(195 40) (octets "</r>"))))))
154 (cxml:well-formedness-violation () nil))
156 ;; Every DOCTYPE is deliberately outside the selected safe input policy.
157 (dolist (pair '(("doctype-without-subset" . "<!DOCTYPE r><r/>")
158 ("internal-entity" . "<!DOCTYPE r [<!ENTITY e 'value'>]><r>&e;</r>")
159 ("external-file-dtd" . "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
160 ("external-network-dtd" . "<!DOCTYPE r SYSTEM 'http://127.0.0.1:9/must-not-fetch.dtd'><r/>")
161 ("external-general-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'file:///tmp/cxml-must-not-read'>]><r>&e;</r>")
162 ("external-parameter-entity" . "<!DOCTYPE r [<!ENTITY % e SYSTEM 'file:///tmp/cxml-must-not-read'>%e;]><r/>")))
163 (expect-condition (car pair) 'doctype-forbidden
164 (lambda () (parse-text (cdr pair)))))
165 (check "safe-policy-never-invokes-external-resolver" (null *resolver-calls*))
167 ;; Exercise the second defense without the DOCTYPE guard. Trap the exact
168 ;; CXML fallback before it can turn a URI into a pathname and call OPEN.
169 ;; This test-only interception is restored in this disposable process.
170 (let ((original (fdefinition 'cxml::uri-to-pathname)))
173 (setf (fdefinition 'cxml::uri-to-pathname)
174 (lambda (uri) (declare (ignore uri))
175 (incf *fallback-calls*) (error 'fallback-reached)))
176 (dolist (pair '(("resolver-file-dtd" . "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
177 ("resolver-network-dtd" . "<!DOCTYPE r SYSTEM 'http://127.0.0.1:9/must-not-fetch.dtd'><r/>")
178 ("resolver-file-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'file:///tmp/cxml-must-not-read'>]><r>&e;</r>")
179 ("resolver-network-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'http://127.0.0.1:9/must-not-fetch'>]><r>&e;</r>")
180 ("resolver-parameter-entity" . "<!DOCTYPE r [<!ENTITY % e SYSTEM 'file:///tmp/cxml-must-not-read'>%e;]><r/>")))
181 (let ((before (length *resolver-calls*)))
182 (expect-condition (car pair) 'external-resource-forbidden
183 (lambda () (parse-bytes (octets (cdr pair)) :resolver-probe t)))
184 (check (concatenate 'string (car pair) ":called")
185 (= (length *resolver-calls*) (1+ before)))))
186 ;; Counterprobe: returning NIL really would reach the fallback.
187 (expect-condition "nil-resolver-reaches-cxml-fallback" 'fallback-reached
190 (octets "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
191 (make-instance 'observation :allow-doctype t)
192 :entity-resolver (lambda (public system)
193 (declare (ignore public system)) nil))))
194 (check "exactly-one-fallback-only-in-nil-counterprobe" (= *fallback-calls* 1)))
195 (setf (fdefinition 'cxml::uri-to-pathname) original)))
197 (format t "CXML-SUMMARY ~D PASS ~D FAIL~%"
198 (- (length *checks*) (length *failed-checks*)) (length *failed-checks*))
199 (when *failed-checks* (uiop:quit 1))