;;;; Slice 19D0: parser qualification only; no XTM deserializer or model.
(require :asdf)
(setf *compile-verbose* nil *load-verbose* nil)
(assert (string= (lisp-implementation-version) "2.5.4"))
(assert (asdf:find-system "cxml" nil))
(asdf:load-system "cxml")
(defpackage #:topic-maps.cxml-probe (:use #:cl))
(in-package #:topic-maps.cxml-probe)
(defvar *checks* nil)
(defvar *failed-checks* nil)
(defvar *resolver-calls* nil)
(defvar *fallback-calls* 0)
(defun check (name truth)
(push name *checks*)
(unless truth (push name *failed-checks*))
(format t "CXML-CHECK-~A ~A~%" (if truth "PASS" "FAIL") name))
(defun expect-condition (name class thunk)
;; An I/O failure or a different generic exception is never a PASS.
(let ((condition (handler-case (progn (funcall thunk) nil)
(error (c) c))))
(check name (typep condition class))
(format t "CXML-REJECTION ~A ~A~%" name (type-of condition))))
(define-condition doctype-forbidden (error) ())
(define-condition external-resource-forbidden (error)
((system-id :initarg :system-id :reader denied-system-id)))
(define-condition fallback-reached (error) ())
(defclass observation (sax:default-handler)
((elements :initform nil :accessor elements)
(text :initform "" :accessor text)
;; Enabled only in tests that exercise the resolver defense separately.
(allow-doctype :initform nil :initarg :allow-doctype :reader allow-doctype)))
(defmethod sax:start-element ((handler observation) uri local qname attributes)
(push (list uri local qname
(mapcar (lambda (a)
(list (sax:attribute-namespace-uri a)
(sax:attribute-local-name a)
(sax:attribute-qname a) (sax:attribute-value a)))
attributes))
(elements handler)))
(defmethod sax:characters ((handler observation) data)
(setf (text handler) (concatenate 'string (text handler) data)))
(defmethod sax:end-document ((handler observation)) handler)
(defmethod sax:start-dtd ((handler observation) name public-id system-id)
(declare (ignore name public-id system-id))
(unless (allow-doctype handler) (error 'doctype-forbidden)))
(defun deny-external-resource (public-id system-id)
(declare (ignore public-id))
(push (puri:render-uri system-id nil) *resolver-calls*)
;; NIL would permit CXML's file-opening fallback (xml-parse.lisp:1064).
(error 'external-resource-forbidden :system-id system-id))
(defun parse-bytes (bytes &key resolver-probe)
(let ((sax:*namespace-processing* t)
(sax:*include-xmlns-attributes* nil)
(cxml:*catalog* nil)
(cxml:*cache-all-dtds* nil)
(cxml:*dtd-cache* (cxml:make-dtd-cache)))
(cxml:parse-octets
bytes (make-instance 'observation :allow-doctype resolver-probe)
:validate nil :disallow-internal-subset (not resolver-probe)
:entity-resolver #'deny-external-resource)))
(defun octets (string &optional (encoding :utf-8) bom)
;; Explicit BOM bytes: Babel's endian-specific encoders do not supply a
;; BOM vector for :USE-BOM T. Encoding itself stays with the existing library.
(concatenate '(vector (unsigned-byte 8))
(if bom (ecase encoding
(:utf-8 #(239 187 191))
(:utf-16le #(255 254))
(:utf-16be #(254 255))) #())
(babel:string-to-octets string :encoding encoding :use-bom nil)))
(defun parse-text (string) (parse-bytes (octets string)))
(defun check-decoded-text (name bytes expected)
(handler-case (check name (string= (text (parse-bytes bytes)) expected))
(error (c)
(check name nil)
(format t "CXML-UNEXPECTED-REJECTION ~A ~A: ~A~%" name (type-of c) c))))
(check "asdf-cxml-found-and-loaded" (find-package :cxml))
(dolist (name '("closure-common" "puri" "babel" "alexandria"
"trivial-features" "trivial-gray-streams"))
(check (concatenate 'string "asdf-dependency:" name)
(asdf:component-loaded-p (asdf:find-system name))))
(let* ((source "🦊")
(doc (parse-text source))
(root (first (elements doc))))
(check "namespace-expanded-element" (equal (subseq root 0 3) '("urn:element" "root" "x:root")))
(check "unprefixed-attribute-has-no-namespace"
(member '(nil "plain" "plain" "v") (fourth root) :test #'equal))
(check "prefixed-attribute-namespace"
(member '("urn:attr" "a" "y:a" "ä") (fourth root) :test #'equal))
(check "unicode-astral-character" (string= (text doc) "🦊")))
(let ((roots (reverse (elements (parse-text "")))))
(check "default-namespace-and-nested-rebinding"
(equal (mapcar #'first roots) '("urn:a" "urn:b" "urn:a"))))
(dolist (spec '(("utf8-no-bom" :utf-8 nil "UTF-8")
("utf8-bom" :utf-8 t "UTF-8")
("utf16le-bom" :utf-16le t "UTF-16")
("utf16be-bom" :utf-16be t "UTF-16")
("utf16le-explicit-no-bom" :utf-16le nil "UTF-16LE")
("utf16be-explicit-no-bom" :utf-16be nil "UTF-16BE")))
(destructuring-bind (name encoding bom declaration) spec
(dolist (content '("Grüße 水" "Grüße 水 🦊"))
(let ((source (format nil "~A" declaration content)))
(check-decoded-text (concatenate 'string name (if (find #\🦊 content) ":astral" ":bmp"))
(octets source encoding bom) content)))))
(check-decoded-text "utf16-bom-without-declaration:bmp"
(octets "水" :utf-16le t) "水")
(check-decoded-text "utf16-bom-without-declaration:astral"
(octets "水🦊" :utf-16le t) "水🦊")
(check "predefined-entities-and-character-references"
(string= (text (parse-text "<>&"'ä🦊")) "<>&\"'ä🦊"))
(check "xml-line-end-normalization"
(string= (text (parse-text (format nil "a~C~Cb~Cc" #\Return #\Newline #\Return)))
(format nil "a~%b~%c")))
(check "xml-attribute-whitespace-normalization"
(equal (fourth (first (elements (parse-text (format nil "" #\Tab #\Newline)))))
'((nil "a" "a" "a b c"))))
(dolist (pair '(("misnested" . "")
("duplicate-attribute" . "")
("duplicate-expanded-attribute" . "")
("undeclared-prefix" . "")
("multiple-roots" . "")
("unclosed-root" . "")
("undefined-entity" . "&missing;")
("forbidden-nul-reference" . "")
("forbidden-surrogate-reference" . "")))
(expect-condition (car pair) 'cxml:well-formedness-violation
(lambda () (parse-text (cdr pair)))))
(expect-condition "invalid-utf8" 'cxml:well-formedness-violation
(lambda () (parse-bytes (concatenate '(vector (unsigned-byte 8))
(octets "") #(195 40) (octets "")))))
(handler-case
(format t "CXML-OBSERVATION invalid-utf8 decoded:~{ U+~4,'0X~}~%"
(map 'list #'char-code
(text (parse-bytes (concatenate '(vector (unsigned-byte 8))
(octets "") #(195 40) (octets ""))))))
(cxml:well-formedness-violation () nil))
;; Every DOCTYPE is deliberately outside the selected safe input policy.
(dolist (pair '(("doctype-without-subset" . "")
("internal-entity" . "]>&e;")
("external-file-dtd" . "")
("external-network-dtd" . "")
("external-general-entity" . "]>&e;")
("external-parameter-entity" . "%e;]>")))
(expect-condition (car pair) 'doctype-forbidden
(lambda () (parse-text (cdr pair)))))
(check "safe-policy-never-invokes-external-resolver" (null *resolver-calls*))
;; Exercise the second defense without the DOCTYPE guard. Trap the exact
;; CXML fallback before it can turn a URI into a pathname and call OPEN.
;; This test-only interception is restored in this disposable process.
(let ((original (fdefinition 'cxml::uri-to-pathname)))
(unwind-protect
(progn
(setf (fdefinition 'cxml::uri-to-pathname)
(lambda (uri) (declare (ignore uri))
(incf *fallback-calls*) (error 'fallback-reached)))
(dolist (pair '(("resolver-file-dtd" . "")
("resolver-network-dtd" . "")
("resolver-file-entity" . "]>&e;")
("resolver-network-entity" . "]>&e;")
("resolver-parameter-entity" . "%e;]>")))
(let ((before (length *resolver-calls*)))
(expect-condition (car pair) 'external-resource-forbidden
(lambda () (parse-bytes (octets (cdr pair)) :resolver-probe t)))
(check (concatenate 'string (car pair) ":called")
(= (length *resolver-calls*) (1+ before)))))
;; Counterprobe: returning NIL really would reach the fallback.
(expect-condition "nil-resolver-reaches-cxml-fallback" 'fallback-reached
(lambda ()
(cxml:parse-octets
(octets "")
(make-instance 'observation :allow-doctype t)
:entity-resolver (lambda (public system)
(declare (ignore public system)) nil))))
(check "exactly-one-fallback-only-in-nil-counterprobe" (= *fallback-calls* 1)))
(setf (fdefinition 'cxml::uri-to-pathname) original)))
(format t "CXML-SUMMARY ~D PASS ~D FAIL~%"
(- (length *checks*) (length *failed-checks*)) (length *failed-checks*))
(when *failed-checks* (uiop:quit 1))