]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/cxml-probe.lisp
Register five original XTM mergeMap contracts (Slice 22H)
[topic-maps.git] / tests / cxml-probe.lisp
1 ;;;; Slice 19D0: parser qualification only; no XTM deserializer or model.
2 (require :asdf)
3 (setf *compile-verbose* nil *load-verbose* nil)
4 (assert (string= (lisp-implementation-version) "2.5.4"))
5 (assert (asdf:find-system "cxml" nil))
6 (asdf:load-system "cxml")
7 (asdf:load-asd (merge-pathnames "../topic-maps.asd" *load-truename*))
8 (asdf:load-system "topic-maps/xml")
9
10 (defpackage #:topic-maps.cxml-probe (:use #:cl))
11 (in-package #:topic-maps.cxml-probe)
12
13 (defvar *checks* nil)
14 (defvar *failed-checks* nil)
15 (defvar *resolver-calls* nil)
16 (defvar *fallback-calls* 0)
17
18 (defun check (name truth)
19 (push name *checks*)
20 (unless truth (push name *failed-checks*))
21 (format t "CXML-CHECK-~A ~A~%" (if truth "PASS" "FAIL") name))
22
23 (defun expect-condition (name class thunk)
24 ;; An I/O failure or a different generic exception is never a PASS.
25 (let ((condition (handler-case (progn (funcall thunk) nil)
26 (error (c) c))))
27 (check name (typep condition class))
28 (format t "CXML-REJECTION ~A ~A~%" name (type-of condition))))
29
30 (deftype doctype-forbidden () 'topic-maps.xml:doctype-forbidden)
31 (define-condition external-resource-forbidden (error)
32 ((system-id :initarg :system-id :reader denied-system-id)))
33 (define-condition fallback-reached (error) ())
34
35 (defclass observation (sax:default-handler)
36 ((elements :initform nil :accessor elements)
37 (text :initform "" :accessor text)
38 ;; Enabled only in tests that exercise the resolver defense separately.
39 (allow-doctype :initform nil :initarg :allow-doctype :reader allow-doctype)))
40
41 (defmethod sax:start-element ((handler observation) uri local qname attributes)
42 (push (list uri local qname
43 (mapcar (lambda (a)
44 (list (sax:attribute-namespace-uri a)
45 (sax:attribute-local-name a)
46 (sax:attribute-qname a) (sax:attribute-value a)))
47 attributes))
48 (elements handler)))
49
50 (defmethod sax:characters ((handler observation) data)
51 (setf (text handler) (concatenate 'string (text handler) data)))
52
53 (defmethod sax:end-document ((handler observation)) handler)
54
55 (defmethod sax:start-dtd ((handler observation) name public-id system-id)
56 (declare (ignore name public-id system-id))
57 (unless (allow-doctype handler) (error 'topic-maps.xml:doctype-forbidden)))
58
59 (defun deny-external-resource (public-id system-id)
60 (declare (ignore public-id))
61 (push (puri:render-uri system-id nil) *resolver-calls*)
62 ;; NIL would permit CXML's file-opening fallback (xml-parse.lisp:1064).
63 (error 'external-resource-forbidden :system-id system-id))
64
65 (defun parse-bytes (bytes &key resolver-probe)
66 (unless resolver-probe
67 (return-from parse-bytes
68 (topic-maps.xml:parse-octets bytes (make-instance 'observation))))
69 (let ((sax:*namespace-processing* t)
70 (sax:*include-xmlns-attributes* nil)
71 (cxml:*catalog* nil)
72 (cxml:*cache-all-dtds* nil)
73 (cxml:*dtd-cache* (cxml:make-dtd-cache)))
74 (cxml:parse-octets
75 bytes (make-instance 'observation :allow-doctype resolver-probe)
76 :validate nil :disallow-internal-subset (not resolver-probe)
77 :entity-resolver #'deny-external-resource)))
78
79 (defun octets (string &optional (encoding :utf-8) bom)
80 ;; Explicit BOM bytes: Babel's endian-specific encoders do not supply a
81 ;; BOM vector for :USE-BOM T. Encoding itself stays with the existing library.
82 (concatenate '(vector (unsigned-byte 8))
83 (if bom (ecase encoding
84 (:utf-8 #(239 187 191))
85 (:utf-16le #(255 254))
86 (:utf-16be #(254 255))) #())
87 (babel:string-to-octets string :encoding encoding :use-bom nil)))
88
89 (defun parse-text (string) (parse-bytes (octets string)))
90
91 (defun check-decoded-text (name bytes expected)
92 (handler-case (check name (string= (text (parse-bytes bytes)) expected))
93 (error (c)
94 (check name nil)
95 (format t "CXML-UNEXPECTED-REJECTION ~A ~A: ~A~%" name (type-of c) c))))
96
97 (check "asdf-cxml-found-and-loaded" (find-package :cxml))
98 (dolist (name '("closure-common" "puri" "babel" "alexandria"
99 "trivial-features" "trivial-gray-streams"))
100 (check (concatenate 'string "asdf-dependency:" name)
101 (asdf:component-loaded-p (asdf:find-system name))))
102
103 (let* ((source "<x:root xmlns:x='urn:element' xmlns:y='urn:attr' plain='v' y:a='ä'>🦊</x:root>")
104 (doc (parse-text source))
105 (root (first (elements doc))))
106 (check "namespace-expanded-element" (equal (subseq root 0 3) '("urn:element" "root" "x:root")))
107 (check "unprefixed-attribute-has-no-namespace"
108 (member '(nil "plain" "plain" "v") (fourth root) :test #'equal))
109 (check "prefixed-attribute-namespace"
110 (member '("urn:attr" "a" "y:a" "ä") (fourth root) :test #'equal))
111 (check "unicode-astral-character" (string= (text doc) "🦊")))
112 (let ((roots (reverse (elements (parse-text "<r xmlns='urn:a'><s xmlns='urn:b'/><t/></r>")))))
113 (check "default-namespace-and-nested-rebinding"
114 (equal (mapcar #'first roots) '("urn:a" "urn:b" "urn:a"))))
115
116 (dolist (spec '(("utf8-no-bom" :utf-8 nil "UTF-8")
117 ("utf8-bom" :utf-8 t "UTF-8")
118 ("utf16le-bom" :utf-16le t "UTF-16")
119 ("utf16be-bom" :utf-16be t "UTF-16")
120 ("utf16le-explicit-no-bom" :utf-16le nil "UTF-16LE")
121 ("utf16be-explicit-no-bom" :utf-16be nil "UTF-16BE")))
122 (destructuring-bind (name encoding bom declaration) spec
123 (dolist (content '("Grüße 水" "Grüße 水 🦊"))
124 (let ((source (format nil "<?xml version='1.0' encoding='~A'?><r>~A</r>" declaration content)))
125 (check-decoded-text (concatenate 'string name (if (find #\🦊 content) ":astral" ":bmp"))
126 (octets source encoding bom) content)))))
127 (check-decoded-text "utf16-bom-without-declaration:bmp"
128 (octets "<r>水</r>" :utf-16le t) "水")
129 (check-decoded-text "utf16-bom-without-declaration:astral"
130 (octets "<r>水🦊</r>" :utf-16le t) "水🦊")
131 (check "predefined-entities-and-character-references"
132 (string= (text (parse-text "<r>&lt;&gt;&amp;&quot;&apos;&#xE4;&#x1F98A;</r>")) "<>&\"'ä🦊"))
133 (check "xml-line-end-normalization"
134 (string= (text (parse-text (format nil "<r>a~C~Cb~Cc</r>" #\Return #\Newline #\Return)))
135 (format nil "a~%b~%c")))
136 (check "xml-attribute-whitespace-normalization"
137 (equal (fourth (first (elements (parse-text (format nil "<r a='a~Cb~Cc'/>" #\Tab #\Newline)))))
138 '((nil "a" "a" "a b c"))))
139
140 (dolist (pair '(("misnested" . "<r><a></r>")
141 ("duplicate-attribute" . "<r a='1' a='2'/>")
142 ("duplicate-expanded-attribute" . "<r xmlns:a='urn:x' xmlns:b='urn:x' a:v='1' b:v='2'/>")
143 ("undeclared-prefix" . "<q:r/>")
144 ("multiple-roots" . "<r/><s/>")
145 ("unclosed-root" . "<r>")
146 ("undefined-entity" . "<r>&missing;</r>")
147 ("forbidden-nul-reference" . "<r>&#0;</r>")
148 ("forbidden-surrogate-reference" . "<r>&#xD800;</r>")))
149 (expect-condition (car pair) 'cxml:well-formedness-violation
150 (lambda () (parse-text (cdr pair)))))
151 (expect-condition "invalid-utf8" 'cxml:well-formedness-violation
152 (lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
153 (octets "<r>") #(195 40) (octets "</r>")))))
154 (handler-case
155 (format t "CXML-OBSERVATION invalid-utf8 decoded:~{ U+~4,'0X~}~%"
156 (map 'list #'char-code
157 (text (parse-bytes (concatenate '(vector (unsigned-byte 8))
158 (octets "<r>") #(195 40) (octets "</r>"))))))
159 (cxml:well-formedness-violation () nil))
160
161 ;; Every DOCTYPE is deliberately outside the selected safe input policy.
162 (dolist (pair '(("doctype-without-subset" . "<!DOCTYPE r><r/>")
163 ("internal-entity" . "<!DOCTYPE r [<!ENTITY e 'value'>]><r>&e;</r>")
164 ("external-file-dtd" . "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
165 ("external-network-dtd" . "<!DOCTYPE r SYSTEM 'http://127.0.0.1:9/must-not-fetch.dtd'><r/>")
166 ("external-general-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'file:///tmp/cxml-must-not-read'>]><r>&e;</r>")
167 ("external-parameter-entity" . "<!DOCTYPE r [<!ENTITY % e SYSTEM 'file:///tmp/cxml-must-not-read'>%e;]><r/>")))
168 (expect-condition (car pair) 'doctype-forbidden
169 (lambda () (parse-text (cdr pair)))))
170 (check "safe-policy-never-invokes-external-resolver" (null *resolver-calls*))
171
172 ;; Exercise the second defense without the DOCTYPE guard. Trap the exact
173 ;; CXML fallback before it can turn a URI into a pathname and call OPEN.
174 ;; This test-only interception is restored in this disposable process.
175 (let ((original (fdefinition 'cxml::uri-to-pathname)))
176 (unwind-protect
177 (progn
178 (setf (fdefinition 'cxml::uri-to-pathname)
179 (lambda (uri) (declare (ignore uri))
180 (incf *fallback-calls*) (error 'fallback-reached)))
181 (dolist (pair '(("resolver-file-dtd" . "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
182 ("resolver-network-dtd" . "<!DOCTYPE r SYSTEM 'http://127.0.0.1:9/must-not-fetch.dtd'><r/>")
183 ("resolver-file-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'file:///tmp/cxml-must-not-read'>]><r>&e;</r>")
184 ("resolver-network-entity" . "<!DOCTYPE r [<!ENTITY e SYSTEM 'http://127.0.0.1:9/must-not-fetch'>]><r>&e;</r>")
185 ("resolver-parameter-entity" . "<!DOCTYPE r [<!ENTITY % e SYSTEM 'file:///tmp/cxml-must-not-read'>%e;]><r/>")))
186 (let ((before (length *resolver-calls*)))
187 (expect-condition (car pair) 'external-resource-forbidden
188 (lambda () (parse-bytes (octets (cdr pair)) :resolver-probe t)))
189 (check (concatenate 'string (car pair) ":called")
190 (= (length *resolver-calls*) (1+ before)))))
191 ;; Counterprobe: returning NIL really would reach the fallback.
192 (expect-condition "nil-resolver-reaches-cxml-fallback" 'fallback-reached
193 (lambda ()
194 (cxml:parse-octets
195 (octets "<!DOCTYPE r SYSTEM 'file:///tmp/cxml-must-not-read.dtd'><r/>")
196 (make-instance 'observation :allow-doctype t)
197 :entity-resolver (lambda (public system)
198 (declare (ignore public system)) nil))))
199 (check "exactly-one-fallback-only-in-nil-counterprobe" (= *fallback-calls* 1)))
200 (setf (fdefinition 'cxml::uri-to-pathname) original)))
201
202 (load (merge-pathnames "cxml-encoding-regressions.lisp" *load-truename*))
203
204 (format t "CXML-SUMMARY ~D PASS ~D FAIL~%"
205 (- (length *checks*) (length *failed-checks*)) (length *failed-checks*))
206 (when *failed-checks* (uiop:quit 1))