;;;; Additional checks. The original 58 check names/expectations are retained.
(in-package #:topic-maps.cxml-probe)
(defun encoded-document (encoding content &key bom (declaration "UTF-8"))
(octets (if declaration
(format nil "~A" declaration content)
(format nil "~A" content))
encoding bom))
(defun bad-content (name content encoding &key bom declaration)
(let* ((header (if declaration
(format nil "" declaration)
""))
(input (concatenate '(vector (unsigned-byte 8))
(octets header encoding bom) content (octets "" encoding))))
(expect-condition name 'cxml:well-formedness-violation
(lambda () (parse-bytes input)))))
(dolist (spec '(("stray-continuation" 128)
("bad-continuation" 195 40)
("overlong-two" 192 175)
("overlong-three" 224 128 175)
("overlong-four" 240 128 128 175)
("encoded-surrogate" 237 160 128)
("above-unicode-limit" 244 144 128 128)
("illegal-leading-f5" 245 128 128 128)
("obsolete-five-byte-form" 248 128 128 128 175)
("obsolete-six-byte-form" 252 128 128 128 128 175)
("illegal-leading-ff" 255)))
(bad-content (concatenate 'string "utf8-invalid:" (car spec))
(coerce (cdr spec) '(vector (unsigned-byte 8))) :utf-8))
;; EOF is deliberately inside a sequence, rather than followed by XML markup.
(dolist (bytes '(#(194) #(224 160) #(240 159 166)))
(expect-condition (format nil "utf8-truncated:~{~2,'0X~}" (coerce bytes 'list))
'cxml:well-formedness-violation
(lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
(octets "") bytes)))))
(dolist (encoding '(:utf-16le :utf-16be))
(let* ((le (eq encoding :utf-16le))
(label (if le "le" "be")))
(dolist (spec (if le
'(("isolated-low" 0 220) ("high-then-letter" 0 216 65 0)
("high-then-high" 0 216 0 216))
'(("isolated-low" 220 0) ("high-then-letter" 216 0 0 65)
("high-then-high" 216 0 216 0))))
(bad-content (concatenate 'string "utf16-invalid:" label ":" (car spec))
(coerce (cdr spec) '(vector (unsigned-byte 8))) encoding :bom t))
(dolist (spec (if le '(("odd-octet" 65) ("unfinished-high" 0 216))
'(("odd-octet" 65) ("unfinished-high" 216 0))))
(expect-condition (concatenate 'string "utf16-truncated:" label ":" (car spec))
'cxml:well-formedness-violation
(lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
(octets "" encoding t)
(coerce (cdr spec) '(vector (unsigned-byte 8))))))))))
(dolist (encoding '(:utf-8 :utf-16le :utf-16be))
(let* ((bom (not (eq encoding :utf-8)))
(declaration (if bom "UTF-16" "UTF-8"))
(characters (coerce (mapcar #'code-char '(#xD7FF #xE000 #xFFFD #x10000 #x10FFFF)) 'string)))
(check-decoded-text (format nil "scalar-boundaries:~A" encoding)
(encoded-document encoding characters :bom bom :declaration declaration)
characters)
(check-decoded-text (format nil "line-ending-normalization:~A" encoding)
(encoded-document encoding (format nil "a~C~Cb~Cc" #\Return #\Newline #\Return)
:bom bom :declaration declaration)
(format nil "a~%b~%c"))
;; Cross the parser's 8192-byte refill with a multi-unit scalar.
(dolist (length '(8189 8190 8191 8192))
(let ((content (concatenate 'string (make-string length :initial-element #\x) "🦊水")))
(check-decoded-text (format nil "refill:~A:~D" encoding length)
(encoded-document encoding content :bom bom :declaration declaration)
content)))))
;; Signatures alone do not authorize silently treating generic UTF-16 as LE/BE.
(dolist (encoding '(:utf-16le :utf-16be))
(dolist (declaration '("UTF-16" nil "UTF-8"))
(let ((source (if declaration
(format nil "" declaration)
"")))
(expect-condition (format nil "bomless-missing-or-wrong-label:~A:~A" encoding declaration)
'cxml:well-formedness-violation
(lambda () (parse-bytes (octets source encoding))))))
(expect-condition (format nil "bomless-without-declaration:~A" encoding)
'cxml:well-formedness-violation
(lambda () (parse-bytes (octets "" encoding))))
(expect-condition (format nil "bomless-pi-without-xml-declaration:~A" encoding)
'cxml:well-formedness-violation
(lambda () (parse-bytes (octets "" encoding))))
(expect-condition (format nil "bom-declaration-disagreement:~A" encoding)
'cxml:well-formedness-violation
(lambda () (parse-bytes (encoded-document encoding "x" :bom t :declaration "UTF-8"))))
(expect-condition (format nil "opposite-endian-declaration:~A" encoding)
'cxml:well-formedness-violation
(lambda () (parse-bytes (encoded-document encoding "x"
:declaration (if (eq encoding :utf-16le) "UTF-16BE" "UTF-16LE")))))
(expect-condition (format nil "explicit-endian-label-with-bom:~A" encoding)
'cxml:well-formedness-violation
(lambda () (parse-bytes (encoded-document encoding "x" :bom t
:declaration (if (eq encoding :utf-16le) "UTF-16LE" "UTF-16BE"))))))
(dolist (declaration '("UTF-16" "UTF-16LE" "UTF-16BE" "ISO-8859-1"))
(expect-condition (concatenate 'string "utf8-bom-wrong-declaration:" declaration)
'cxml:well-formedness-violation
(lambda () (parse-bytes (encoded-document :utf-8 "x" :bom t :declaration declaration)))))
(expect-condition "unknown-encoding-is-fatal" 'cxml:well-formedness-violation
(lambda () (parse-bytes (encoded-document :utf-8 "x" :declaration "x-not-a-real-encoding"))))
(check-decoded-text "encoding-name-case-insensitive"
(encoded-document :utf-16le "🦊" :declaration "utf-16le") "🦊")
;; The public boundary enforces the policy even for a permissive SAX receiver.
(expect-condition "public-api-rejects-string-input" 'type-error
(lambda () (topic-maps.xml:parse-octets "" nil)))
(expect-condition "public-api-no-resolver-override" 'program-error
(lambda () (uiop:symbol-call :topic-maps.xml :parse-octets
(octets "") nil :entity-resolver (constantly nil))))
(expect-condition "public-api-guard-with-nil-handler" 'topic-maps.xml:doctype-forbidden
(lambda () (topic-maps.xml:parse-octets
(octets "") nil)))
(expect-condition "public-resolver-signals" 'topic-maps.xml:external-resource-forbidden
(lambda () (topic-maps.xml::deny-external-resource
nil (puri:parse-uri "http://127.0.0.1:9/must-not-fetch"))))
(let ((original (symbol-value 'cxml::*topic-maps-encoding-patch*)))
(unwind-protect
(progn
(setf (symbol-value 'cxml::*topic-maps-encoding-patch*) nil)
(expect-condition "public-api-refuses-unqualified-dependency" 'topic-maps.xml:unqualified-parser
(lambda () (topic-maps.xml:parse-octets (octets "") nil))))
(setf (symbol-value 'cxml::*topic-maps-encoding-patch*) original)))
(let ((handler (make-instance 'observation)))
(check "public-api-returns-identical-sax-result"
(eq handler (topic-maps.xml:parse-octets (octets "") handler))))
;; Exercise the existing decoder's partial-input contract (no second decoder).
(dolist (encoding '(:utf-8 :utf-16-little-endian :utf-16-big-endian))
(let* ((input (octets "🦊" (ecase encoding (:utf-8 :utf-8)
(:utf-16-little-endian :utf-16le)
(:utf-16-big-endian :utf-16be))))
(output (make-array 8 :element-type '(unsigned-byte 32))))
(dolist (end '(1 2 3))
(multiple-value-bind (written read)
(runes-encoding:decode-sequence encoding input 0 end output 0 8 nil)
(check (format nil "decoder-needs-more:~A:~D" encoding end)
(and (zerop written) (zerop read))))
(expect-condition (format nil "decoder-incomplete-at-eof:~A:~D" encoding end)
'runes-encoding:encoding-error
(lambda () (runes-encoding:decode-sequence encoding input 0 end output 0 8 t))))))