]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/cxml-encoding-regressions.lisp
Support XTM topic name item identifiers with original contracts and offline evidence
[topic-maps.git] / tests / cxml-encoding-regressions.lisp
1 ;;;; Additional checks. The original 58 check names/expectations are retained.
2 (in-package #:topic-maps.cxml-probe)
3
4 (defun encoded-document (encoding content &key bom (declaration "UTF-8"))
5 (octets (if declaration
6 (format nil "<?xml version='1.0' encoding='~A'?><r>~A</r>" declaration content)
7 (format nil "<r>~A</r>" content))
8 encoding bom))
9
10 (defun bad-content (name content encoding &key bom declaration)
11 (let* ((header (if declaration
12 (format nil "<?xml version='1.0' encoding='~A'?><r>" declaration)
13 "<r>"))
14 (input (concatenate '(vector (unsigned-byte 8))
15 (octets header encoding bom) content (octets "</r>" encoding))))
16 (expect-condition name 'cxml:well-formedness-violation
17 (lambda () (parse-bytes input)))))
18
19 (dolist (spec '(("stray-continuation" 128)
20 ("bad-continuation" 195 40)
21 ("overlong-two" 192 175)
22 ("overlong-three" 224 128 175)
23 ("overlong-four" 240 128 128 175)
24 ("encoded-surrogate" 237 160 128)
25 ("above-unicode-limit" 244 144 128 128)
26 ("illegal-leading-f5" 245 128 128 128)
27 ("obsolete-five-byte-form" 248 128 128 128 175)
28 ("obsolete-six-byte-form" 252 128 128 128 128 175)
29 ("illegal-leading-ff" 255)))
30 (bad-content (concatenate 'string "utf8-invalid:" (car spec))
31 (coerce (cdr spec) '(vector (unsigned-byte 8))) :utf-8))
32
33 ;; EOF is deliberately inside a sequence, rather than followed by XML markup.
34 (dolist (bytes '(#(194) #(224 160) #(240 159 166)))
35 (expect-condition (format nil "utf8-truncated:~{~2,'0X~}" (coerce bytes 'list))
36 'cxml:well-formedness-violation
37 (lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
38 (octets "<r>") bytes)))))
39
40 (dolist (encoding '(:utf-16le :utf-16be))
41 (let* ((le (eq encoding :utf-16le))
42 (label (if le "le" "be")))
43 (dolist (spec (if le
44 '(("isolated-low" 0 220) ("high-then-letter" 0 216 65 0)
45 ("high-then-high" 0 216 0 216))
46 '(("isolated-low" 220 0) ("high-then-letter" 216 0 0 65)
47 ("high-then-high" 216 0 216 0))))
48 (bad-content (concatenate 'string "utf16-invalid:" label ":" (car spec))
49 (coerce (cdr spec) '(vector (unsigned-byte 8))) encoding :bom t))
50 (dolist (spec (if le '(("odd-octet" 65) ("unfinished-high" 0 216))
51 '(("odd-octet" 65) ("unfinished-high" 216 0))))
52 (expect-condition (concatenate 'string "utf16-truncated:" label ":" (car spec))
53 'cxml:well-formedness-violation
54 (lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
55 (octets "<r>" encoding t)
56 (coerce (cdr spec) '(vector (unsigned-byte 8))))))))))
57
58 (dolist (encoding '(:utf-8 :utf-16le :utf-16be))
59 (let* ((bom (not (eq encoding :utf-8)))
60 (declaration (if bom "UTF-16" "UTF-8"))
61 (characters (coerce (mapcar #'code-char '(#xD7FF #xE000 #xFFFD #x10000 #x10FFFF)) 'string)))
62 (check-decoded-text (format nil "scalar-boundaries:~A" encoding)
63 (encoded-document encoding characters :bom bom :declaration declaration)
64 characters)
65 (check-decoded-text (format nil "line-ending-normalization:~A" encoding)
66 (encoded-document encoding (format nil "a~C~Cb~Cc" #\Return #\Newline #\Return)
67 :bom bom :declaration declaration)
68 (format nil "a~%b~%c"))
69 ;; Cross the parser's 8192-byte refill with a multi-unit scalar.
70 (dolist (length '(8189 8190 8191 8192))
71 (let ((content (concatenate 'string (make-string length :initial-element #\x) "🦊水")))
72 (check-decoded-text (format nil "refill:~A:~D" encoding length)
73 (encoded-document encoding content :bom bom :declaration declaration)
74 content)))))
75
76 ;; Signatures alone do not authorize silently treating generic UTF-16 as LE/BE.
77 (dolist (encoding '(:utf-16le :utf-16be))
78 (dolist (declaration '("UTF-16" nil "UTF-8"))
79 (let ((source (if declaration
80 (format nil "<?xml version='1.0' encoding='~A'?><r/>" declaration)
81 "<?xml version='1.0'?><r/>")))
82 (expect-condition (format nil "bomless-missing-or-wrong-label:~A:~A" encoding declaration)
83 'cxml:well-formedness-violation
84 (lambda () (parse-bytes (octets source encoding))))))
85 (expect-condition (format nil "bomless-without-declaration:~A" encoding)
86 'cxml:well-formedness-violation
87 (lambda () (parse-bytes (octets "<r/>" encoding))))
88 (expect-condition (format nil "bomless-pi-without-xml-declaration:~A" encoding)
89 'cxml:well-formedness-violation
90 (lambda () (parse-bytes (octets "<?pi?><r/>" encoding))))
91 (expect-condition (format nil "bom-declaration-disagreement:~A" encoding)
92 'cxml:well-formedness-violation
93 (lambda () (parse-bytes (encoded-document encoding "x" :bom t :declaration "UTF-8"))))
94 (expect-condition (format nil "opposite-endian-declaration:~A" encoding)
95 'cxml:well-formedness-violation
96 (lambda () (parse-bytes (encoded-document encoding "x"
97 :declaration (if (eq encoding :utf-16le) "UTF-16BE" "UTF-16LE")))))
98 (expect-condition (format nil "explicit-endian-label-with-bom:~A" encoding)
99 'cxml:well-formedness-violation
100 (lambda () (parse-bytes (encoded-document encoding "x" :bom t
101 :declaration (if (eq encoding :utf-16le) "UTF-16LE" "UTF-16BE"))))))
102 (dolist (declaration '("UTF-16" "UTF-16LE" "UTF-16BE" "ISO-8859-1"))
103 (expect-condition (concatenate 'string "utf8-bom-wrong-declaration:" declaration)
104 'cxml:well-formedness-violation
105 (lambda () (parse-bytes (encoded-document :utf-8 "x" :bom t :declaration declaration)))))
106 (expect-condition "unknown-encoding-is-fatal" 'cxml:well-formedness-violation
107 (lambda () (parse-bytes (encoded-document :utf-8 "x" :declaration "x-not-a-real-encoding"))))
108 (check-decoded-text "encoding-name-case-insensitive"
109 (encoded-document :utf-16le "🦊" :declaration "utf-16le") "🦊")
110
111 ;; The public boundary enforces the policy even for a permissive SAX receiver.
112 (expect-condition "public-api-rejects-string-input" 'type-error
113 (lambda () (topic-maps.xml:parse-octets "<r/>" nil)))
114 (expect-condition "public-api-no-resolver-override" 'program-error
115 (lambda () (uiop:symbol-call :topic-maps.xml :parse-octets
116 (octets "<r/>") nil :entity-resolver (constantly nil))))
117 (expect-condition "public-api-guard-with-nil-handler" 'topic-maps.xml:doctype-forbidden
118 (lambda () (topic-maps.xml:parse-octets
119 (octets "<!DOCTYPE r SYSTEM 'file:///tmp/must-not-open'><r/>") nil)))
120 (expect-condition "public-resolver-signals" 'topic-maps.xml:external-resource-forbidden
121 (lambda () (topic-maps.xml::deny-external-resource
122 nil (puri:parse-uri "http://127.0.0.1:9/must-not-fetch"))))
123 (let ((original (symbol-value 'cxml::*topic-maps-encoding-patch*)))
124 (unwind-protect
125 (progn
126 (setf (symbol-value 'cxml::*topic-maps-encoding-patch*) nil)
127 (expect-condition "public-api-refuses-unqualified-dependency" 'topic-maps.xml:unqualified-parser
128 (lambda () (topic-maps.xml:parse-octets (octets "<r/>") nil))))
129 (setf (symbol-value 'cxml::*topic-maps-encoding-patch*) original)))
130 (let ((handler (make-instance 'observation)))
131 (check "public-api-returns-identical-sax-result"
132 (eq handler (topic-maps.xml:parse-octets (octets "<r/>") handler))))
133
134 ;; Exercise the existing decoder's partial-input contract (no second decoder).
135 (dolist (encoding '(:utf-8 :utf-16-little-endian :utf-16-big-endian))
136 (let* ((input (octets "🦊" (ecase encoding (:utf-8 :utf-8)
137 (:utf-16-little-endian :utf-16le)
138 (:utf-16-big-endian :utf-16be))))
139 (output (make-array 8 :element-type '(unsigned-byte 32))))
140 (dolist (end '(1 2 3))
141 (multiple-value-bind (written read)
142 (runes-encoding:decode-sequence encoding input 0 end output 0 8 nil)
143 (check (format nil "decoder-needs-more:~A:~D" encoding end)
144 (and (zerop written) (zerop read))))
145 (expect-condition (format nil "decoder-incomplete-at-eof:~A:~D" encoding end)
146 'runes-encoding:encoding-error
147 (lambda () (runes-encoding:decode-sequence encoding input 0 end output 0 8 t))))))