]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/ctm-registered-originals.lisp
test(ctm): register seven reviewed negative Slice 20B contracts
[topic-maps.git] / tests / ctm-registered-originals.lisp
1 (defpackage #:topic-maps.registered-original.tests
2 (:use #:cl #:topic-maps.tmdm)
3 (:export #:run-tests #:write-reports #:candidate-groups
4 #:run-negative-tests #:write-negative-reports))
5 (in-package #:topic-maps.registered-original.tests)
6
7 ;; One selection of the existing discovered Case Records, not another registry.
8 (defparameter *groups*
9 '((:comments "comment-ml" "comment-ml-nested" "comment-ml-nested2" "comment-sl" "comment-sl2"
10 "comment-sl3" "comment-sl4" "comment-sl5" "comment-sl6" "empty")
11 (:identity "association-qname-iid" "item-identifier-INF" "item-identifier-relative" "qname-date" "qname-digits"
12 "rootless-uri" "subject-identifier-dot" "subject-identifier-non-http" "subject-identifier-relative"
13 "subject-locator-dot" "subject-locator-relative" "topic-identifier-INF" "topic-identifier-unicode")
14 (:occurrences "occurrence-decimal-dot-neg2" "occurrence-decimal-dot-pos" "occurrence-decimal-leading-zeros2"
15 "occurrence-duplicate-string" "occurrence-duplicate-uri" "occurrence-qname-value"
16 "occurrence-string-multiline" "occurrence-string-multiline2" "occurrence-uri-semicolon")
17 (:prefixes "prefix-multiline2" "prefix-multiline3" "prefix-multiline4" "prefix-with-qm")
18 (:escapes "string-escape" "string-escape2" "unicode-escape-u4" "unicode-escape-u6")))
19 (defun candidate-groups () (copy-tree *groups*))
20 (defvar *checks* nil)
21 (defvar *rows* nil)
22 (defvar *results* nil)
23 (defvar *entry-statuses* nil)
24 (defun check (label truth)
25 (unless truth (error "Registered original check failed: ~A" label))
26 (push label *checks*))
27 (defun fields (&rest args) (apply #'topic-maps.test-foundation::fields args))
28 (defun binary (path) (topic-maps.ctm.tests::binary-file path))
29 (defun same-set (a b &optional (test #'equal))
30 (and (= (length a) (length b)) (subsetp a b :test test) (subsetp b a :test test)))
31 (defun local (map doc name)
32 (find-if (lambda (topic) (member (concatenate 'string doc "#" name) (topic-item-identifiers topic) :test #'equal))
33 (topic-map-topics map)))
34 (defun single-identity (map kind value)
35 (let ((topics (topic-map-topics map)))
36 (and (= 1 (length topics)) (null (topic-map-associations map))
37 (let ((topic (first topics)))
38 (and (equal (topic-item-identifiers topic) (when (eq kind :ii) (list value)))
39 (equal (topic-subject-identifiers topic) (when (eq kind :si) (list value)))
40 (equal (topic-subject-locators topic) (when (eq kind :sl) (list value)))
41 (null (topic-names topic)) (null (topic-occurrences topic)))))))
42 (defun plain-locals-p (map doc names)
43 (and (= (length names) (length (topic-map-topics map))) (null (topic-map-associations map))
44 (every (lambda (name) (let ((topic (local map doc name)))
45 (and topic (equal (list (concatenate 'string doc "#" name)) (topic-item-identifiers topic))
46 (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic))
47 (null (topic-names topic)) (null (topic-occurrences topic))))) names)))
48 (defun semantic-witness (name group map doc)
49 "Independent concrete object properties, in addition to exact XML equality."
50 (ecase group
51 (:comments
52 (plain-locals-p map doc
53 (cond ((member name '("empty" "comment-sl3") :test #'equal) nil)
54 ((member name '("comment-ml" "comment-ml-nested" "comment-ml-nested2" "comment-sl2") :test #'equal) '("topic1" "topic3"))
55 ((equal name "comment-sl6") '("topic" "topic2")) (t '("topic")))))
56 (:identity
57 (if (equal name "association-qname-iid")
58 (let* ((base "http://www.example.org/my-ctm#")
59 (a (first (topic-map-associations map))) (roles (when a (association-roles a)))
60 (topics (topic-map-topics map)))
61 (flet ((named (suffix) (find-if (lambda (topic) (equal (list (concatenate 'string base suffix)) (topic-item-identifiers topic))) topics)))
62 (and (= 3 (length topics)) (= 1 (length (topic-map-associations map))) (= 1 (length roles))
63 (every (lambda (topic) (and (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic)))) topics)
64 (eq (named "assoctype") (association-type a)) (eq (named "roletype") (association-role-type (first roles)))
65 (eq (named "topic") (association-role-player (first roles))) (eq a (association-role-parent (first roles))))))
66 (let ((expected (cond
67 ((member name '("item-identifier-INF" "topic-identifier-INF") :test #'equal) (list :ii (concatenate 'string doc "#INF")))
68 ((equal name "item-identifier-relative") (list :ii (concatenate 'string doc "#topic")))
69 ((equal name "topic-identifier-unicode") (list :ii (concatenate 'string doc "#ü͡ØΨ㬟͡")))
70 ((equal name "qname-date") '(:si "http://psi.semagia.com/iso8601/1947-01-08"))
71 ((equal name "qname-digits") '(:si "http://psi.example.org/007"))
72 ((equal name "rootless-uri") '(:si "urn:uuid:d84e2777-8928-4bd4-a3e4-8ca835f92304"))
73 ((equal name "subject-identifier-dot") '(:si "http://psi.example.org/fake"))
74 ((equal name "subject-identifier-non-http") '(:si "ftp://psi.example.org/photo.jpg"))
75 ((equal name "subject-identifier-relative") (list :si (concatenate 'string doc "#fake")))
76 ((equal name "subject-locator-dot") '(:sl "http://www.example.org/"))
77 ((equal name "subject-locator-relative") (list :sl (concatenate 'string doc "#fake"))))))
78 (and expected (single-identity map (first expected) (second expected))))))
79 (:prefixes
80 (if (equal name "prefix-with-qm")
81 (single-identity map :sl "http://www.youtube.com/watch?v=fd85SK7dESU")
82 (single-identity map :si "http://psi.example.org/fake")))
83 (:occurrences
84 (let* ((owner (local map doc (if (equal name "occurrence-qname-value") "john" "topic")))
85 (type (local map doc (if (equal name "occurrence-qname-value") "wikipedia" "occtype")))
86 (items (when owner (topic-occurrences owner))) (o (first items))
87 (expected (cond
88 ((equal name "occurrence-decimal-dot-neg2") (list "-0.03" (decimal-datatype)))
89 ((equal name "occurrence-decimal-dot-pos") (list "0.3" (decimal-datatype)))
90 ((equal name "occurrence-decimal-leading-zeros2") (list "2.3" (decimal-datatype)))
91 ((equal name "occurrence-duplicate-string") (list "Occurrence" (string-datatype)))
92 ((equal name "occurrence-duplicate-uri") (list "http://www.example.org/resource" (iri-datatype)))
93 ((equal name "occurrence-qname-value") (list "http://en.wikipedia.org/wiki/John_Lennon" (iri-datatype)))
94 ((equal name "occurrence-string-multiline") (list (format nil "Occurrence~%Value") (string-datatype)))
95 ((equal name "occurrence-string-multiline2") (list (format nil "Occurrence~C~CValue" #\Return #\Newline) (string-datatype)))
96 ((equal name "occurrence-uri-semicolon") (list "http://www.example.org/;something;;somewhere" (iri-datatype))))))
97 (and owner type (= 2 (length (topic-map-topics map))) (= 1 (length items)) (null (topic-map-associations map))
98 (every (lambda (topic) (null (topic-names topic))) (topic-map-topics map))
99 (eq owner (occurrence-parent o)) (eq type (occurrence-type o))
100 (equal expected (list (occurrence-value o) (occurrence-datatype o)))
101 (null (occurrence-scope o)) (null (item-identifiers o)))))
102 (:escapes
103 (let* ((owner (local map doc (cond ((equal name "string-escape") "topic") ((equal name "string-escape2") "A00") (t "zztop"))))
104 (names (when owner (topic-names owner)))
105 (type (find-if (lambda (topic) (member (default-name-type-identifier) (topic-subject-identifiers topic) :test #'equal)) (topic-map-topics map)))
106 (expected (cond ((equal name "string-escape") (list "\\" "Quote (\") character" (format nil "Some whitespace ~C~C~C" #\Newline #\Return #\Tab)))
107 ((equal name "string-escape2") '("A00")) (t '("ZZ Top")))))
108 (and owner type (= 2 (length (topic-map-topics map))) (null (topic-map-associations map))
109 (same-set expected (mapcar #'topic-name-value names))
110 (every (lambda (n) (and (eq owner (topic-name-parent n)) (eq type (topic-name-type n))
111 (null (topic-name-scope n)) (null (topic-name-variants n)))) names))))))
112
113 (defun identity-fields (case)
114 (list (topic-maps.test-foundation:test-case-id case) (topic-maps.test-foundation:test-case-category case)
115 (topic-maps.test-foundation:test-case-source-pathname case) (topic-maps.test-foundation:test-case-document-iri case)
116 (topic-maps.test-foundation:test-case-expected-outcome case) (topic-maps.test-foundation:test-case-expected-baseline case)
117 (topic-maps.test-foundation:test-case-clauses case)))
118 (defun exercise-case (case group)
119 "Record only an actual comparison. Rejections/errors never become empty Maps."
120 (let* ((id (topic-maps.test-foundation:test-case-id case)) (doc (topic-maps.test-foundation:test-case-document-iri case))
121 (name (pathname-name (topic-maps.test-foundation:test-case-source-pathname case)))
122 (identity (copy-tree (identity-fields case))) (source (binary (topic-maps.test-foundation:test-case-source-pathname case)))
123 (evidence nil) (result nil))
124 (handler-case
125 (let* ((map (topic-maps.ctm:deserialize-file (topic-maps.test-foundation:test-case-source-pathname case) :document-iri doc :source-name id))
126 (held (topic-maps.import.tests::handles map)) (before (topic-maps.import.tests::state map held))
127 (actual (topic-maps.cxtm:canonicalize-octets map :base-locator doc))
128 (expected (binary (topic-maps.test-foundation:test-case-expected-baseline case))) (different (mismatch actual expected)))
129 (check (concatenate 'string id ":actual-TMDM-properties-and-EQ-references") (semantic-witness name group map doc))
130 (check (concatenate 'string id ":canonicalization-preserves-raw-objects-and-handles")
131 (equal before (topic-maps.import.tests::state map held)))
132 (check (concatenate 'string id ":deterministic-canonical-octets")
133 (equalp actual (topic-maps.cxtm:canonicalize-octets map :base-locator doc)))
134 (check (concatenate 'string id ":unmodified-source-bytes") (equalp source (binary (topic-maps.test-foundation:test-case-source-pathname case))))
135 (topic-maps.test-foundation:record-result case (if different :cxtm-mismatch :cxtm-match)
136 :diagnostics (when different (list (format nil "First different byte ~D; actual ~D bytes, expected ~D bytes." different (length actual) (length expected)))))
137 (setf evidence (fields "actualBytes" (length actual) "expectedBytes" (length expected)
138 "firstDifferentByte" different "model" (topic-maps.template.tests::observe map))
139 result (list :id id :base doc :actual (topic-maps.ctm::decode-utf8 actual)
140 :expected (topic-maps.ctm::decode-utf8 expected) :match (null different) :bytes (length actual))))
141 ((or topic-maps.ctm:ctm-unsupported topic-maps.cxtm:cxtm-unsupported) (condition)
142 (topic-maps.test-foundation:record-result case :unsupported :diagnostics (list (format nil "~A: ~A" (type-of condition) condition))))
143 (error (condition)
144 (topic-maps.test-foundation:record-result case :execution-error :diagnostics (list (format nil "~A: ~A" (type-of condition) condition)))))
145 (check (concatenate 'string id ":same-contract-identity-after-execution") (equal identity (identity-fields case)))
146 (values
147 (fields "id" id "group" (string-downcase (symbol-name group))
148 "sourcePathname" id "documentIRI" doc
149 "baseline" (format nil "ctm/baseline/~A.ctm.cxtm" name)
150 "status" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-status case))
151 "actualOutcome" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-actual-outcome case))
152 "diagnostics" (coerce (topic-maps.test-foundation:test-case-diagnostics case) 'vector)
153 "sourceBytes" (length source) "comparison" evidence)
154 result)))
155
156
157 (defun case-state (case)
158 (copy-tree
159 (list (identity-fields case)
160 (topic-maps.test-foundation:test-case-status case)
161 (topic-maps.test-foundation:test-case-actual-outcome case)
162 (topic-maps.test-foundation:test-case-diagnostics case))))
163
164 (defun run-tests ()
165 ;; Use the same Case Records populated by the retained suites. A direct ASDF
166 ;; invocation may initialize those suites, but must never replace their corpus.
167 (unless topic-maps.ctm.tests::*cxtm-corpus*
168 (topic-maps.include.tests:run-tests))
169 (setf *checks* nil *rows* nil *results* nil)
170 (let* ((corpus topic-maps.ctm.tests::*cxtm-corpus*)
171 (root (topic-maps.test-foundation:corpus-root corpus))
172 (ids (loop for (group . names) in *groups*
173 append (mapcar (lambda (name) (format nil "ctm/in/~A.ctm" name)) names)))
174 (cases (topic-maps.test-foundation:corpus-cases corpus))
175 (previous (mapcar (lambda (case) (cons case (case-state case))) cases))
176 (old-results (copy-tree topic-maps.ctm.tests::*cxtm-results*))
177 (base (format nil "https://ctm-tests.example.invalid/~A/"
178 (getf (topic-maps.test-foundation:corpus-metadata corpus) :commit))))
179 (setf *entry-statuses*
180 (copy-list (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses)))
181 (check "registration:forty-distinct-selected-originals"
182 (and (= 40 (length ids)) (= 40 (length (remove-duplicates ids :test #'equal)))
183 (equal '(10 13 9 4 4) (mapcar (lambda (group) (length (cdr group))) *groups*))))
184 (check "registration:existing-corpus-correspondence"
185 (getf (topic-maps.test-foundation:verify-correspondence corpus) :ok))
186 (dolist (group *groups*)
187 (dolist (name (cdr group))
188 (let* ((id (format nil "ctm/in/~A.ctm" name))
189 (selected (topic-maps.test-foundation:select-cases corpus :id id))
190 (case (first selected)))
191 (check (concatenate 'string id ":unique-existing-positive-contract")
192 (and (= 1 (length selected)) (member case cases :test #'eq)
193 (eq :valid (topic-maps.test-foundation:test-case-category case))
194 (eq :cxtm-equal (topic-maps.test-foundation:test-case-expected-outcome case))
195 (eq :not-run (topic-maps.test-foundation:test-case-status case))
196 (null (topic-maps.test-foundation:test-case-actual-outcome case))))
197 (check (concatenate 'string id ":pinned-source-baseline-and-document-IRI")
198 (and (equal (truename (merge-pathnames id root))
199 (truename (topic-maps.test-foundation:test-case-source-pathname case)))
200 (equal (truename (merge-pathnames (format nil "ctm/baseline/~A.ctm.cxtm" name) root))
201 (truename (topic-maps.test-foundation:test-case-expected-baseline case)))
202 (equal (concatenate 'string base id)
203 (topic-maps.test-foundation:test-case-document-iri case))))
204 (multiple-value-bind (row result) (exercise-case case (car group))
205 (push row *rows*)
206 (when result (push result *results*)))
207 (format t "~&REGISTERED-ORIGINAL ~A: ~A~{ ~A~}~%" id
208 (topic-maps.test-foundation:test-case-status case)
209 (topic-maps.test-foundation:test-case-diagnostics case)))))
210 (setf *rows* (nreverse *rows*) *results* (nreverse *results*))
211 (check "registration:forty-actual-byte-comparisons-pass"
212 (and (= 40 (length *rows*) (length *results*))
213 (every (lambda (result) (getf result :match)) *results*)
214 (every (lambda (id) (eq :pass (topic-maps.test-foundation:test-case-status
215 (first (topic-maps.test-foundation:select-cases corpus :id id))))) ids)))
216 (check "registration:all-previous-case-records-and-observations-preserved"
217 (and (eq corpus topic-maps.ctm.tests::*cxtm-corpus*)
218 (equal cases (topic-maps.test-foundation:corpus-cases corpus))
219 (every (lambda (entry)
220 (if (member (topic-maps.test-foundation:test-case-id (car entry)) ids :test #'equal)
221 (equal (first (cdr entry)) (identity-fields (car entry)))
222 (equal (cdr entry) (case-state (car entry))))) previous)
223 (equal old-results topic-maps.ctm.tests::*cxtm-results*)))
224 (let ((after (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses)))
225 (check "registration:only-forty-NOT-RUN-to-PASS-transitions"
226 (and (= (+ 40 (getf *entry-statuses* :pass)) (getf after :pass))
227 (= (- (getf *entry-statuses* :not-run) 40) (getf after :not-run))
228 (= (getf *entry-statuses* :fail) (getf after :fail))
229 (= (getf *entry-statuses* :unsupported) (getf after :unsupported)))))
230 (setf topic-maps.ctm.tests::*cxtm-results*
231 (append topic-maps.ctm.tests::*cxtm-results* *results*)))
232 (check "registration:unique-check-names"
233 (= (length *checks*) (length (remove-duplicates *checks* :test #'equal))))
234 (setf *checks* (nreverse *checks*))
235 (format t "~&REGISTERED-ORIGINALS-PASS: ~D checks; forty actual byte-exact original contracts~%" (length *checks*))
236 t)
237
238 (defun write-reports (prefix)
239 (with-open-file (out (concatenate 'string prefix ".registration.json")
240 :direction :output :if-exists :supersede :if-does-not-exist :create)
241 (topic-maps.test-foundation::emit-json-value
242 (fields "kind" "ctm-registered-original-execution"
243 "passed" (length *checks*) "failed" 0
244 "entryStatuses" (topic-maps.test-foundation::counts-object *entry-statuses*)
245 "checks" (coerce *checks* 'vector) "cases" (coerce *rows* 'vector)) out)
246 (terpri out))
247 (topic-maps.ctm.tests::write-cxtm-report
248 (concatenate 'string prefix ".validation.json")
249 (concatenate 'string prefix ".cxtm.json")
250 :additional-checks *checks*
251 :scope "Slice 20A: forty newly executed original CTM comparisons; prior Case Records, observations and four known DateTime/Include FAILs retained. Full retained regression and XTM integration are evidenced separately."))
252
253
254 ;;; Slice 20B: a selection of seven existing negative Case Records.
255 ;;; Normative review: docs/slice-20b-negative-registration.md. No reader changes.
256 (defparameter *negative-contracts*
257 '(("occurrence-double-positive-infinity" :expected-occurrence-value 15 2 10
258 ("CTM 3.4.1 [28]-[34]" "CTM 3.10 [56]")
259 "Bare INF is neither a CTM number nor another literal alternative.")
260 ("occurrence-double-negative-infinity" :expected-occurrence-value 15 2 10
261 ("CTM 3.4.1 [28]-[34]" "CTM 3.10 [56]")
262 "Bare -INF is neither a CTM number nor another literal alternative.")
263 ("qname-hyphen" :unexpected-token 38 3 3
264 ("CTM 3.3.4 [9]-[11]" "CTM 3.3.6 [14]-[16]")
265 "A QName local part cannot start with a hyphen.")
266 ("string-unbalanced" :unterminated-string 8 1 9
267 ("CTM 3.4.1 [37]-[40]" "CTM 3.4.2 [41]")
268 "The escaped quote is content; the quoted string lacks a closing delimiter.")
269 ("string-unicode" :invalid-unicode-escape 11 1 12
270 ("CTM 3.4.2 [42]")
271 "Uppercase U requires six hexadecimal digits; the closing quote follows four.")
272 ("string-unicode2" :invalid-unicode-escape 15 1 16
273 ("CTM 3.4.2 [42]-[43] and hexadecimal decoding prose")
274 "G cannot be decoded as a hexadecimal digit; see the printed-class qualification.")
275 ("topic-id-illegal" :unexpected-character 0 1 1
276 ("CTM 3.3.6 [14]-[16]")
277 "U+0361 is allowed inside an identifier, but not as its first character.")))
278 (defvar *negative-checks* nil)
279 (defvar *negative-rows* nil)
280 (defvar *negative-entry-statuses* nil)
281
282 (defun captured-read (thunk)
283 (handler-case (values (funcall thunk) nil)
284 (error (condition) (values nil condition))))
285
286 (defun condition-data (condition)
287 (when condition
288 (let ((ctm-p (typep condition 'topic-maps.ctm:ctm-error)))
289 (fields "condition" (string (type-of condition)) "message" (princ-to-string condition)
290 "code" (when ctm-p (topic-maps.test-foundation::word (topic-maps.ctm:error-code condition)))
291 "position" (when ctm-p (topic-maps.ctm:error-position condition))
292 "line" (when ctm-p (topic-maps.ctm:error-line condition))
293 "column" (when ctm-p (topic-maps.ctm:error-column condition))
294 "positionUnit" (when ctm-p (topic-maps.test-foundation::word (topic-maps.ctm:error-position-unit condition)))
295 "source" (when ctm-p (topic-maps.ctm:error-source condition))
296 "cause" (when (and ctm-p (topic-maps.ctm:error-cause condition))
297 (princ-to-string (topic-maps.ctm:error-cause condition)))
298 "context" (when (and ctm-p (topic-maps.ctm:error-context condition))
299 (princ-to-string (topic-maps.ctm:error-context condition)))))))
300
301 (defun condition-signature (condition)
302 (when (typep condition 'topic-maps.ctm:ctm-error)
303 (list (type-of condition) (topic-maps.ctm:error-code condition)
304 (topic-maps.ctm:error-position condition) (topic-maps.ctm:error-line condition)
305 (topic-maps.ctm:error-column condition) (topic-maps.ctm:error-position-unit condition)
306 (topic-maps.ctm:error-source condition) (topic-maps.ctm:error-cause condition)
307 (topic-maps.ctm:error-context condition) (princ-to-string condition))))
308
309 (defun intended-rejection-p (condition spec id)
310 ;; Exact syntax class: unsupported/resource/wrapped model errors never qualify.
311 (and condition (eq (type-of condition) 'topic-maps.ctm:ctm-rejection)
312 (equal (subseq (condition-signature condition) 1 7)
313 (list (second spec) (third spec) (fourth spec) (fifth spec) :character id))
314 (null (topic-maps.ctm:error-cause condition))
315 (null (topic-maps.ctm:error-context condition))))
316
317 (defun replace-fragment (text old new)
318 (let ((position (search old text)))
319 (unless position (error "Missing reviewed source fragment ~S." old))
320 (concatenate 'string (subseq text 0 position) new (subseq text (+ position (length old))))))
321
322 (defun syntax-witness-and-control (name text)
323 "Check the offending source spelling; derive a valid in-memory sibling only."
324 (cond
325 ((equal name "occurrence-double-positive-infinity")
326 (values (equal text (format nil "topic~%occtype: INF."))
327 (replace-fragment text "INF" "1") :occurrence "1"))
328 ((equal name "occurrence-double-negative-infinity")
329 (values (equal text (format nil "topic~%occtype: -INF."))
330 (replace-fragment text "-INF" "-1") :occurrence "-1"))
331 ((equal name "qname-hyphen")
332 (values (equal text (format nil "%prefix ex http://psi.example.org/~%~%ex:-issue.~%"))
333 (replace-fragment text "ex:-issue" "ex:issue") :qname "http://psi.example.org/issue"))
334 ((equal name "string-unbalanced")
335 (values (equal text "topic - \"Foo\\\".")
336 (concatenate 'string (subseq text 0 (1- (length text))) "\".") :name "Foo\""))
337 ((equal name "string-unicode")
338 (values (equal text "a - \"\\U0041\".")
339 (replace-fragment text "\\U0041" "\\u0041") :name "A"))
340 ((equal name "string-unicode2")
341 (values (equal text (format nil "unicode - \"\\U00GG00\".~%"))
342 (replace-fragment text "\\U00GG00" "\\U000041") :name "A"))
343 ((equal name "topic-id-illegal")
344 (let ((mark (code-char #x361)))
345 (values (and (> (length text) 3) (char= mark (char text 0)) (equal (subseq text 1 3) "a."))
346 (concatenate 'string "a" (string mark) (subseq text 2))
347 :identifier (concatenate 'string "a" (string mark)))))
348 (t (error "Unreviewed negative contract ~A." name))))
349
350 (defun control-model-p (map doc name kind value)
351 (and map
352 (ecase kind
353 (:qname (single-identity map :si value))
354 (:identifier (plain-locals-p map doc (list value)))
355 (:occurrence
356 (let* ((owner (local map doc "topic")) (type (local map doc "occtype"))
357 (items (when owner (topic-occurrences owner))) (item (first items)))
358 (and owner type (= 2 (length (topic-map-topics map))) (= 1 (length items))
359 (null (topic-map-associations map))
360 (eq owner (occurrence-parent item)) (eq type (occurrence-type item))
361 (equal value (occurrence-value item)) (equal (integer-datatype) (occurrence-datatype item)))))
362 (:name
363 (let* ((owner (local map doc (cond ((equal name "string-unbalanced") "topic")
364 ((equal name "string-unicode") "a") (t "unicode"))))
365 (names (when owner (topic-names owner))) (item (first names))
366 (type (find-if (lambda (topic) (member (default-name-type-identifier)
367 (topic-subject-identifiers topic) :test #'equal)) (topic-map-topics map))))
368 (and owner type (= 2 (length (topic-map-topics map))) (= 1 (length names))
369 (null (topic-map-associations map)) (eq owner (topic-name-parent item))
370 (eq type (topic-name-type item)) (equal value (topic-name-value item))))))))
371
372 (defun lexical-evidence (text id)
373 ;; Explain the generic QName diagnostic from actual tokens, not its label alone.
374 (let ((topic-maps.ctm::*source-text* text) (topic-maps.ctm::*source-name* id)
375 (lexer (topic-maps.ctm::make-lexer text)) (tokens nil) (failure nil))
376 (handler-case
377 (loop for token = (topic-maps.ctm::next-token lexer) do
378 (push (list (topic-maps.ctm::token-kind token) (topic-maps.ctm::token-value token)
379 (topic-maps.ctm::token-position token)) tokens)
380 until (eq :eof (topic-maps.ctm::token-kind token)))
381 (error (condition) (setf failure condition)))
382 (values (nreverse tokens) failure)))
383
384 (defun exercise-negative-case (case spec)
385 (let* ((id (topic-maps.test-foundation:test-case-id case))
386 (doc (topic-maps.test-foundation:test-case-document-iri case))
387 (name (first spec)) (path (topic-maps.test-foundation:test-case-source-pathname case))
388 (source (binary path)) (text (topic-maps.ctm::decode-utf8 source))
389 (before (case-state case)))
390 (multiple-value-bind (source-witness control-text control-kind control-value)
391 (syntax-witness-and-control name text)
392 (multiple-value-bind (map condition)
393 (captured-read (lambda () (topic-maps.ctm:deserialize-file path :document-iri doc :source-name id)))
394 (multiple-value-bind (repeat-map repeat-condition)
395 (captured-read (lambda () (topic-maps.ctm:deserialize-file path :document-iri doc :source-name id)))
396 (multiple-value-bind (control-map control-condition)
397 (captured-read (lambda () (topic-maps.ctm:deserialize-string control-text :document-iri doc :source-name id)))
398 (multiple-value-bind (tokens lexical-condition) (lexical-evidence text id)
399 (let* ((lexical-witness
400 (if (equal name "qname-hyphen")
401 (and (null lexical-condition)
402 (equal (subseq tokens 3 6)
403 '((:identifier "ex" 36) (:punctuation #\: 38) (:punctuation #\- 39))))
404 t))
405 (control-ok (and (null control-condition)
406 (control-model-p control-map doc name control-kind control-value)))
407 (repeat-ok (and (null map) (null repeat-map)
408 (intended-rejection-p repeat-condition spec id)
409 (equal (condition-signature condition) (condition-signature repeat-condition))))
410 (qualified (and source-witness lexical-witness control-ok repeat-ok
411 (intended-rejection-p condition spec id))))
412 (check (concatenate 'string id ":immutable-original-and-existing-NOT-RUN-record")
413 (and (equal before (case-state case)) (equalp source (binary path))))
414 ;; Observing a different error is evidence, not a conformance PASS.
415 (when qualified
416 (topic-maps.test-foundation:record-result case :ctm-rejection
417 :diagnostics (list (princ-to-string condition))))
418 (check (concatenate 'string id ":only-reviewed-syntax-rejection-can-be-PASS")
419 (if qualified
420 (and (eq :pass (topic-maps.test-foundation:test-case-status case))
421 (eq :ctm-rejection (topic-maps.test-foundation:test-case-actual-outcome case)))
422 (equal before (case-state case))))
423 (format t "~&REGISTERED-NEGATIVE ~A: ~A; ~A~%" id
424 (topic-maps.test-foundation:test-case-status case) (or condition "ACCEPTED"))
425 (fields "id" id "sourcePathname" id "documentIRI" doc
426 "status" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-status case))
427 "actualOutcome" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-actual-outcome case))
428 "condition" (condition-data condition) "repeatCondition" (condition-data repeat-condition)
429 "clauses" (coerce (sixth spec) 'vector) "normativeReason" (seventh spec)
430 "normativeSource" "https://www.isotopicmaps.org/ctm/ctm.html"
431 "qualification" (when (equal name "string-unicode2")
432 "Printed production [43] includes all letters; rejection follows the hexadecimal decoding requirement in 3.4.2 prose, not that character class alone.")
433 "normativeValidated" (if qualified :true :false)
434 "sourceWitness" (if source-witness :true :false)
435 "lexicalWitness" (if lexical-witness :true :false)
436 "reproducibleCondition" (if repeat-ok :true :false)
437 "tokens" (coerce (loop for (kind value position) in tokens collect
438 (fields "kind" (topic-maps.test-foundation::word kind)
439 "value" (if (characterp value) (string value) value) "position" position)) 'vector)
440 "lexicalCondition" (condition-data lexical-condition)
441 "control" (fields "inMemoryOnly" :true "source" control-text
442 "accepted" (if control-ok :true :false) "condition" (condition-data control-condition)
443 "model" (when control-map (topic-maps.template.tests::observe control-map)))
444 "notRegisteredReason" (unless qualified
445 "Observed condition, syntax witness, reproducibility or accepted control did not establish the intended cause; original record remains NOT-RUN."))))))))))
446
447 (defun run-negative-tests ()
448 (unless topic-maps.ctm.tests::*cxtm-corpus* (run-tests))
449 (let* ((*checks* nil) (corpus topic-maps.ctm.tests::*cxtm-corpus*)
450 (cases (topic-maps.test-foundation:corpus-cases corpus))
451 (previous (mapcar (lambda (case) (cons case (case-state case))) cases))
452 (old-results (copy-tree topic-maps.ctm.tests::*cxtm-results*))
453 (root (topic-maps.test-foundation:corpus-root corpus))
454 (base (format nil "https://ctm-tests.example.invalid/~A/"
455 (getf (topic-maps.test-foundation:corpus-metadata corpus) :commit)))
456 (ids (mapcar (lambda (spec) (format nil "ctm/invalid/~A.ctm" (first spec))) *negative-contracts*)))
457 (setf *negative-rows* nil *negative-entry-statuses*
458 (copy-list (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses)))
459 (check "negative-registration:seven-distinct-existing-contracts"
460 (= 7 (length ids) (length (remove-duplicates ids :test #'equal))))
461 (dolist (spec *negative-contracts*)
462 (let* ((id (format nil "ctm/invalid/~A.ctm" (first spec)))
463 (selected (topic-maps.test-foundation:select-cases corpus :id id)) (case (first selected)))
464 (check (concatenate 'string id ":unique-existing-negative-contract-and-document-IRI")
465 (and (= 1 (length selected)) (member case cases :test #'eq)
466 (eq :invalid (topic-maps.test-foundation:test-case-category case))
467 (eq :ctm-rejection (topic-maps.test-foundation:test-case-expected-outcome case))
468 (null (topic-maps.test-foundation:test-case-expected-baseline case))
469 (eq :not-run (topic-maps.test-foundation:test-case-status case))
470 (null (topic-maps.test-foundation:test-case-actual-outcome case))
471 (equal (truename (merge-pathnames id root))
472 (truename (topic-maps.test-foundation:test-case-source-pathname case)))
473 (equal (concatenate 'string base id) (topic-maps.test-foundation:test-case-document-iri case))))
474 (push (exercise-negative-case case spec) *negative-rows*)))
475 (check "negative-registration:old-record-identities-and-CXTM-observations-preserved"
476 (and (eq corpus topic-maps.ctm.tests::*cxtm-corpus*)
477 (equal cases (topic-maps.test-foundation:corpus-cases corpus))
478 (equal old-results topic-maps.ctm.tests::*cxtm-results*)
479 (every (lambda (entry)
480 (if (member (topic-maps.test-foundation:test-case-id (car entry)) ids :test #'equal)
481 (equal (first (cdr entry)) (identity-fields (car entry)))
482 (equal (cdr entry) (case-state (car entry))))) previous)))
483 (let* ((after (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses))
484 (qualified (- (getf after :pass) (getf *negative-entry-statuses* :pass))))
485 (check "negative-registration:only-qualified-NOT-RUN-to-PASS-transitions"
486 (and (<= 0 qualified 7)
487 (= (- (getf *negative-entry-statuses* :not-run) qualified) (getf after :not-run))
488 (= (getf *negative-entry-statuses* :fail) (getf after :fail))
489 (= (getf *negative-entry-statuses* :unsupported) (getf after :unsupported))))
490 (check "negative-registration:unique-check-names"
491 (= (length *checks*) (length (remove-duplicates *checks* :test #'equal))))
492 (setf *negative-rows* (nreverse *negative-rows*) *negative-checks* (nreverse *checks*))
493 (format t "~&REGISTERED-NEGATIVES-PASS: ~D checks; ~D intended original rejections registered; ~D remain NOT-RUN~%"
494 (length *negative-checks*) qualified (- 7 qualified))))
495 t)
496
497 (defun write-negative-reports (prefix)
498 (with-open-file (out (concatenate 'string prefix ".negative.json")
499 :direction :output :if-exists :supersede :if-does-not-exist :create)
500 (topic-maps.test-foundation::emit-json-value
501 (fields "kind" "ctm-negative-original-execution" "passed" (length *negative-checks*) "failed" 0
502 "entryStatuses" (topic-maps.test-foundation::counts-object *negative-entry-statuses*)
503 "checks" (coerce *negative-checks* 'vector) "cases" (coerce *negative-rows* 'vector)) out)
504 (terpri out))
505 (topic-maps.ctm.tests::write-cxtm-report
506 (concatenate 'string prefix ".validation.json") (concatenate 'string prefix ".cxtm.json")
507 :additional-checks *negative-checks*
508 :scope "Slice 20B: seven negative originals actually executed; only intended syntax rejections registered. Earlier positive comparisons and four known DateTime/Include FAILs preserved; full retained regression, Slice 20A and XTM evidence separate."))