(in-package #:topic-maps.xtm.tests) (defun datatype-native-preflight () ;; Run before changing the Reader, and again in the regular ASDF test-op. (let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:datatype:owner")) (base (intern-topic map :item-identifier "urn:datatype:base")) (a (intern-topic map :item-identifier "urn:datatype:a")) (b (intern-topic map :item-identifier "urn:datatype:b")) (r1 (intern-topic map :item-identifier "urn:datatype:r1")) (r2 (intern-topic map :item-identifier "urn:datatype:r2")) (name (add-default-name owner "N" :scope (list base))) (count (length (topic-map-topics map))) (datatype "https://types.example.invalid/custom") (value " 42 & 馬🦊 ") (v1 (add-variant name value :datatype datatype :scope (list base a))) (v2 (add-variant name value :datatype datatype :scope (list base b))) (other (add-variant name value :datatype "urn:types:other" :scope (list base a)))) (check "datatype-native:value-and-datatype-independent" (and (equal value (variant-value v1)) (equal datatype (variant-datatype v1)) (equal "urn:types:other" (variant-datatype other)))) (check "datatype-native:no-topic-creation" (= count (length (topic-map-topics map)))) (check "datatype-native:distinct-datatypes-and-scopes" (= 3 (length (all-variants map)))) (add-item-identifier v1 "urn:variant:first") (add-item-identifier v2 "urn:variant:second") (set-item-reifier v1 r1) (set-item-reifier v2 r2) (merge-topics a b) (let ((current (canonical-variant v1))) (check "datatype-native:old-variant-handles-canonical-EQ" (and (eq current (canonical-variant v2)) (not (eq current v1)) (not (eq current other)) (= 2 (length (all-variants map))))) (check "datatype-native:parent-and-scope-canonical-EQ" (and (eq name (canonical-topic-name name)) (eq owner (topic-name-parent name)) (eq name (variant-parent current)) (scope-eq-p (list base (canonical-topic a)) (variant-scope current)))) (check "datatype-native:identifiers-unioned" (and (= 2 (length (item-identifiers current))) (member "urn:variant:first" (item-identifiers current) :test #'equal) (member "urn:variant:second" (item-identifiers current) :test #'equal))) (check "datatype-native:reifier-inverse-and-old-handles-EQ" (and (eq (canonical-topic r1) (canonical-topic r2)) (eq (item-reifier current) (canonical-topic r1)) (eq current (topic-reified r1)) (eq current (topic-reified r2)))) (let ((duplicate (add-variant name value :datatype datatype :scope (list base (canonical-topic a))))) (check "datatype-native:duplicate-add-normalizes" (and (eq (canonical-variant current) duplicate) (eq duplicate (canonical-variant v1)) (eq duplicate (canonical-variant v2)) (= 2 (length (all-variants map)))))) (check "datatype-native:datatype-survives-normalization" (and (equal datatype (variant-datatype v1)) (equal value (variant-value v2))))) (let* ((before (map-reifier-state map)) (actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*))) (check "datatype-native:cxtm-preserves-custom-lexical-value" (and (search " 42 & 馬🦊 " (babel:octets-to-string actual :encoding :utf-8)) (search (concatenate 'string "" datatype "") (babel:octets-to-string actual :encoding :utf-8)))) (check "datatype-native:pure-deterministic-canonicalization" (and (equal before (map-reifier-state map)) (equalp actual (topic-maps.cxtm:canonicalize-octets map :base-locator *document*)) (equal before (map-reifier-state map)) (variant-reifier-current-p map)))) map)) (defvar *datatype-outputs* nil) (defvar *datatype-controls* nil) (defun datatype-resource (datatype value) (format nil "~A" (if datatype (format nil " datatype='~A'" datatype) "") value)) (defun datatype-variant (datatype value &key (themes '("s")) reifier identifiers) (variant-reifier-xml reifier :themes themes :identifiers identifiers :resource (datatype-resource datatype value))) (defun datatype-body (resource) (format nil "N~A~A" (scope-xml '("s")) resource)) (defun datatype-original-contract () (let* ((id "xtm2/in/variant-datatype-unknown.xtm") (*document* (concatenate 'string "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/" id)) (raw (file-bytes (corpus-path id))) (copy (copy-seq raw)) (baseline (corpus-path "xtm2/baseline/variant-datatype-unknown.xtm.cxtm")) (expected (file-bytes baseline)) (case (topic-maps.test-foundation:make-test-case :id id :category :valid :source-pathname (corpus-path id) :document-iri *document* :expected-outcome :cxtm-equal :expected-baseline baseline :clauses '("XTM 2.0 4.12,4.16.1-4.16.3" "TMDM 4.4,5.5,6.4" "CXTM 3.14,3.16 Other"))) (map nil) (actual nil) (condition nil) (parser-calls 0) (parser-input-eq nil)) (setf *cases* (append *cases* (list case))) (handler-case (progn (let ((saved (symbol-function 'topic-maps.xml:parse-octets))) (unwind-protect (progn (setf (symbol-function 'topic-maps.xml:parse-octets) (lambda (octets &rest args) (setf parser-input-eq (eq octets raw)) (incf parser-calls) (apply saved octets args))) (setf map (deserialize-octets raw :document-iri *document* :source-key id))) (setf (symbol-function 'topic-maps.xml:parse-octets) saved))) (let* ((owner (by-id map "topic")) (theme (by-id map "scope")) (name (first (topic-names owner))) (v (first (all-variants map))) (type (by-si map (default-name-type-identifier)))) (check "datatype-original:native-model-parent-type-scope-EQ" (and (= 3 (length (topic-map-topics map))) (null (topic-map-associations map)) (every (lambda (topic) (null (topic-occurrences topic))) (topic-map-topics map)) (= 1 (length (name-reifier-names map))) (= 1 (length (all-variants map))) (equal "Topic" (topic-name-value name)) (null (topic-name-scope name)) (eq owner (topic-name-parent name)) (eq type (topic-name-type name)) (eq name (canonical-topic-name name)) (eq name (variant-parent v)) (eq v (canonical-variant v)) (scope-eq-p (list theme) (variant-scope v)) (variant-reifier-current-p map))) (check "datatype-original:value-datatype-not-topic-identities" (and (equal "BOGUS" (variant-value v)) (equal "http://example.com/bogus" (variant-datatype v)) (equal (list (concatenate 'string *document* "#topic")) (item-identifiers owner)) (equal (list (concatenate 'string *document* "#scope")) (item-identifiers theme)) (null (item-identifiers type)) (equal (list (default-name-type-identifier)) (topic-subject-identifiers type)) (null (topic-subject-identifiers owner)) (null (topic-subject-identifiers theme)) (every (lambda (topic) (null (topic-subject-locators topic))) (topic-map-topics map)) (null (item-identifiers map)) (null (item-identifiers name)) (null (item-identifiers v)) (null (item-reifier map)) (null (item-reifier name)) (null (item-reifier v)) (null (by-si map "http://example.com/bogus"))))) (setf actual (variant-reifier-pure "datatype-original" map (append (all-variants map) (name-reifier-names map) (topic-map-topics map)))) (check "datatype-original:parser-receives-original-octets" parser-input-eq) (check "datatype-original:source-preserved-parser-once" (and (= parser-calls 1) (equalp raw copy))) (let ((other (deserialize-octets raw :document-iri *document* :source-key "independent:datatype-original"))) (check "datatype-original:document-IRI-source-key-independence" (and (equalp (variant-reifier-model-data map) (variant-reifier-model-data other)) (equalp actual (topic-maps.cxtm:canonicalize-octets other :base-locator *document*)))))) (error (c) (setf condition c))) (record-original-result case actual expected condition) (setf *datatype-outputs* (list (topic-maps.test-foundation::fields "id" id "documentIRI" *document* "sourceKey" id "sourceBytes" (length raw) "sourceHex" (format nil "~{~2,'0X~}" (coerce raw 'list)) "sourceText" (babel:octets-to-string raw :encoding :utf-8) "qualifiedParserCalls" parser-calls "condition" (map-reifier-condition-data condition) "model" (when map (variant-reifier-model-data map)) "actualCXTM" (when actual (babel:octets-to-string actual :encoding :utf-8)) "expectedCXTM" (babel:octets-to-string expected :encoding :utf-8) "actualBytes" (when actual (length actual)) "expectedBytes" (length expected) "firstDifferentByte" (when actual (mismatch actual expected)) "status" (string-downcase (string (topic-maps.test-foundation:test-case-status case)))))) (when condition (error condition)) (check "datatype-original:579-byte-exact" (and (= 579 (length expected)) (equalp actual expected))) (check "datatype-original:72-unique-registered-cases" (and (= 72 (length *cases*)) (= 64 (count :valid *cases* :key #'topic-maps.test-foundation:test-case-category)) (= 72 (length (remove-duplicates *cases* :key #'topic-maps.test-foundation:test-case-id :test #'equal))))))) (defun datatype-value-controls () (dolist (row (list (list "http-custom" "http://types.example.invalid/custom" "literal%GG" "literal%GG") (list "urn-custom" "urn:types:custom" "urn:resource-not-a-topic" "urn:resource-not-a-topic") (list "unicode" "urn:types:馬" "馬🦊é" "馬🦊é") (list "datatype-percent-preserved" "urn:types:%61" "42" "42") (list "whitespace-escaping" "urn:types:text" " A&<> " (format nil " ~CA&<>~% " #\Tab)) (list "empty" "urn:types:empty" "" "") (list "integer-text-now-supported" (integer-datatype) "+00042" "+00042") (list "explicit-string" (string-datatype) "urn:%GG" "urn:%GG") (list "default-string" nil " x & y " " x & y ") (list "anyURI-retained" (iri-datatype) "../%E9%A6%AC" "https://xtm-tests.example.invalid/cd165b150176c1d6842b37be9cc9051b40d4996b/xtm2/馬"))) (destructuring-bind (label dtype xml value) row (variant-reifier-control (concatenate 'string "datatype:" label) (datatype-body (datatype-resource dtype xml)) (lambda (map) (let ((v (first (all-variants map)))) (check (concatenate 'string "datatype:value-and-exact-datatype:" label) (and (equal value (variant-value v)) (equal (or dtype (string-datatype)) (variant-datatype v)) (= 3 (length (topic-map-topics map))) (eq (variant-parent v) (first (topic-names (by-id map "a")))) (scope-eq-p (variant-scope v) (list (by-id map "s")))))))))) (variant-reifier-control "datatype:normalization-discriminators" (format nil "~A" (variant-name-xml nil (concatenate 'string (datatype-variant "urn:type:a" "V") (datatype-variant "urn:type:a" "V") (datatype-variant "urn:type:a" "W") (datatype-variant "urn:type:b" "V")))) (lambda (map) (check "datatype:equality-includes-datatype-and-value" (and (= 3 (length (all-variants map))) (= 3 (length (topic-map-topics map))) (= 2 (count "urn:type:a" (all-variants map) :key #'variant-datatype :test #'equal)) (= 2 (count "V" (all-variants map) :key #'variant-value :test #'equal))))))) (defun datatype-merge-controls () (dolist (kind '("subjectIdentifier" "subjectLocator" "itemIdentity")) (let* ((held nil) (saved (symbol-function 'topic-maps.xtm::read-variant)) (body (format nil "<~A href='urn:owner'/>~A<~A href='urn:owner'/>~A<~A href='urn:theme'/><~A href='urn:theme'/>" kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("x") :reifier "#r1" :identifiers '("urn:v1"))) kind (variant-name-xml '("base") (datatype-variant "urn:datatype:merged" "V" :themes '("y") :reifier "#r2" :identifiers '("urn:v2"))) kind kind))) (unwind-protect (progn (setf (symbol-function 'topic-maps.xtm::read-variant) (lambda (&rest args) (let ((v (apply saved args))) (push v held) v))) (variant-reifier-control (concatenate 'string "datatype:merge-" kind) body (lambda (map) (let* ((owner (by-id map "a")) (name (first (topic-names owner))) (v (first (all-variants map))) (r (by-id map "r1"))) (check (concatenate 'string "datatype:merge-canonical-identities:" kind) (and (eq owner (by-id map "b")) (eq name (variant-parent v)) (eq owner (topic-name-parent name)) (eq (by-id map "x") (by-id map "y")) (scope-eq-p (variant-scope v) (list (by-id map "base") (by-id map "x"))) (= 1 (length (all-variants map))) (= 2 (length held)) (not (eq (first held) (second held))) (every (lambda (old) (eq v (canonical-variant old))) held) (= 2 (length (item-identifiers v))) (member "urn:v1" (item-identifiers v) :test #'equal) (member "urn:v2" (item-identifiers v) :test #'equal) (eq r (by-id map "r2")) (eq r (item-reifier v)) (eq v (topic-reified r)) (equal "urn:datatype:merged" (variant-datatype v)) (equal "V" (variant-value v)))) (variant-reifier-pure (concatenate 'string "datatype:held-" kind) map held))))) (setf (symbol-function 'topic-maps.xtm::read-variant) saved))))) (defun datatype-negative-controls () (dolist (row '(("relative" "relative" :invalid-resource-data-datatype) ("empty" "" :invalid-resource-data-datatype) ("scheme" "1bad:x" :invalid-resource-data-datatype) ("space" "urn:bad type" :invalid-iri) ("percent" "urn:type:%GG" :invalid-percent-escape) ("percent-short" "urn:type:%" :invalid-percent-escape) ("percent-UTF8" "urn:type:%C3%28" :invalid-percent-utf8) ("percent-surrogate" "urn:type:%ED%A0%80" :invalid-percent-utf8))) (variant-reifier-error-control (concatenate 'string "datatype:" (first row)) (datatype-body (datatype-resource (second row) "V")) 'xtm-rejection (third row))) (dolist (dtype '(nil "http://www.w3.org/2001/XMLSchema#string" "http://www.w3.org/2001/XMLSchema#anyURI" "urn:custom")) (dolist (child '("V" "V")) (variant-reifier-error-control (format nil "datatype:forbidden-child:~A:~A" dtype child) (datatype-body (datatype-resource dtype child)) 'xtm-rejection :unexpected-child))) (dolist (value '("text" "XML")) (variant-reifier-error-control (concatenate 'string "datatype:anyType:" value) (datatype-body (datatype-resource "http://www.w3.org/2001/XMLSchema#anyType" value)) 'xtm-unsupported :resource-data-datatype)) (variant-reifier-error-control "datatype:unknown-XTM-structure" "" 'xtm-rejection :unexpected-element) (variant-reifier-error-control "datatype:occurrence-remains-unsupported" "V" 'xtm-unsupported :resource-data-datatype) (variant-reifier-error-control "datatype:native-reifier-conflict" (format nil "~A" (variant-name-xml nil (concatenate 'string (datatype-variant "urn:custom" "A" :reifier "#r") (datatype-variant "urn:custom" "B" :reifier "#r")))) 'reification-error :reifier-conflict) (variant-reifier-error-control "datatype:native-scope-conflict" (format nil "~A" (variant-name-xml '("s") (datatype-variant "urn:custom" "V"))) 'variant-error :invalid-variant-scope) (let ((raw (concatenate '(vector (unsigned-byte 8)) (bytes "N") #(195 40) (bytes "")))) (variant-reifier-error-control "datatype:raw-invalid-UTF8" nil 'xtm-xml-error :xml-not-well-formed (lambda () (deserialize-octets raw :document-iri *document* :source-key "variant:datatype:raw-invalid-UTF8"))))) (defun datatype-context-controls () ;; Retain the resource-operation/native-consumer cross-check from Slice 22D. ;; Slice 22G separately exercises genuine XTM occurrence deserialization. (dolist (resource (list (datatype-resource "urn:shared:type" " A&B ") (datatype-resource nil "V") (datatype-resource (iri-datatype) "../sound") "")) (let* ((root (topic-maps.xml:parse-octets (bytes (document (datatype-body resource))) (make-instance 'topic-maps.xtm::source-handler))) (node (second (topic-maps.xtm::elements (second (topic-maps.xtm::elements (first (topic-maps.xtm::elements (first (topic-maps.xtm::elements root))))))))) (topic-maps.xtm::*document-iri* *document*) (topic-maps.xtm::*source-key* "datatype:shared-context")) (topic-maps.xtm::validate-source root) (multiple-value-bind (value dtype) (topic-maps.xtm::read-resource-value node) (let* ((map (make-topic-map)) (owner (intern-topic map :item-identifier "urn:owner")) (type (intern-topic map :item-identifier "urn:type")) (theme (intern-topic map :item-identifier "urn:theme")) (name (add-default-name owner "N")) (count (length (topic-map-topics map))) (v (add-variant name value :datatype dtype :scope (list theme))) (o (add-occurrence owner type value :datatype dtype))) (check (concatenate 'string "datatype:shared-value-consumers:" resource) (and (equal (variant-value v) (occurrence-value o)) (equal (variant-datatype v) (occurrence-datatype o)) (eq name (variant-parent v)) (eq owner (occurrence-parent o)) (= count (length (topic-map-topics map)))))))))) (defun datatype-tests () (datatype-native-preflight) (let ((*variant-reifier-controls* nil)) (datatype-value-controls) (datatype-merge-controls) (datatype-negative-controls) (datatype-context-controls) (setf *datatype-controls* (nreverse *variant-reifier-controls*))))