]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/ctm-registered-originals.lisp
test(ctm): register forty original Slice 20A contracts
[topic-maps.git] / tests / ctm-registered-originals.lisp
1 (defpackage #:topic-maps.registered-original.tests
2 (:use #:cl #:topic-maps.tmdm)
3 (:export #:run-tests #:write-reports #:candidate-groups))
4 (in-package #:topic-maps.registered-original.tests)
5
6 ;; One selection of the existing discovered Case Records, not another registry.
7 (defparameter *groups*
8 '((:comments "comment-ml" "comment-ml-nested" "comment-ml-nested2" "comment-sl" "comment-sl2"
9 "comment-sl3" "comment-sl4" "comment-sl5" "comment-sl6" "empty")
10 (:identity "association-qname-iid" "item-identifier-INF" "item-identifier-relative" "qname-date" "qname-digits"
11 "rootless-uri" "subject-identifier-dot" "subject-identifier-non-http" "subject-identifier-relative"
12 "subject-locator-dot" "subject-locator-relative" "topic-identifier-INF" "topic-identifier-unicode")
13 (:occurrences "occurrence-decimal-dot-neg2" "occurrence-decimal-dot-pos" "occurrence-decimal-leading-zeros2"
14 "occurrence-duplicate-string" "occurrence-duplicate-uri" "occurrence-qname-value"
15 "occurrence-string-multiline" "occurrence-string-multiline2" "occurrence-uri-semicolon")
16 (:prefixes "prefix-multiline2" "prefix-multiline3" "prefix-multiline4" "prefix-with-qm")
17 (:escapes "string-escape" "string-escape2" "unicode-escape-u4" "unicode-escape-u6")))
18 (defun candidate-groups () (copy-tree *groups*))
19 (defvar *checks* nil)
20 (defvar *rows* nil)
21 (defvar *results* nil)
22 (defvar *entry-statuses* nil)
23 (defun check (label truth)
24 (unless truth (error "Registered original check failed: ~A" label))
25 (push label *checks*))
26 (defun fields (&rest args) (apply #'topic-maps.test-foundation::fields args))
27 (defun binary (path) (topic-maps.ctm.tests::binary-file path))
28 (defun same-set (a b &optional (test #'equal))
29 (and (= (length a) (length b)) (subsetp a b :test test) (subsetp b a :test test)))
30 (defun local (map doc name)
31 (find-if (lambda (topic) (member (concatenate 'string doc "#" name) (topic-item-identifiers topic) :test #'equal))
32 (topic-map-topics map)))
33 (defun single-identity (map kind value)
34 (let ((topics (topic-map-topics map)))
35 (and (= 1 (length topics)) (null (topic-map-associations map))
36 (let ((topic (first topics)))
37 (and (equal (topic-item-identifiers topic) (when (eq kind :ii) (list value)))
38 (equal (topic-subject-identifiers topic) (when (eq kind :si) (list value)))
39 (equal (topic-subject-locators topic) (when (eq kind :sl) (list value)))
40 (null (topic-names topic)) (null (topic-occurrences topic)))))))
41 (defun plain-locals-p (map doc names)
42 (and (= (length names) (length (topic-map-topics map))) (null (topic-map-associations map))
43 (every (lambda (name) (let ((topic (local map doc name)))
44 (and topic (equal (list (concatenate 'string doc "#" name)) (topic-item-identifiers topic))
45 (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic))
46 (null (topic-names topic)) (null (topic-occurrences topic))))) names)))
47 (defun semantic-witness (name group map doc)
48 "Independent concrete object properties, in addition to exact XML equality."
49 (ecase group
50 (:comments
51 (plain-locals-p map doc
52 (cond ((member name '("empty" "comment-sl3") :test #'equal) nil)
53 ((member name '("comment-ml" "comment-ml-nested" "comment-ml-nested2" "comment-sl2") :test #'equal) '("topic1" "topic3"))
54 ((equal name "comment-sl6") '("topic" "topic2")) (t '("topic")))))
55 (:identity
56 (if (equal name "association-qname-iid")
57 (let* ((base "http://www.example.org/my-ctm#")
58 (a (first (topic-map-associations map))) (roles (when a (association-roles a)))
59 (topics (topic-map-topics map)))
60 (flet ((named (suffix) (find-if (lambda (topic) (equal (list (concatenate 'string base suffix)) (topic-item-identifiers topic))) topics)))
61 (and (= 3 (length topics)) (= 1 (length (topic-map-associations map))) (= 1 (length roles))
62 (every (lambda (topic) (and (null (topic-subject-identifiers topic)) (null (topic-subject-locators topic)))) topics)
63 (eq (named "assoctype") (association-type a)) (eq (named "roletype") (association-role-type (first roles)))
64 (eq (named "topic") (association-role-player (first roles))) (eq a (association-role-parent (first roles))))))
65 (let ((expected (cond
66 ((member name '("item-identifier-INF" "topic-identifier-INF") :test #'equal) (list :ii (concatenate 'string doc "#INF")))
67 ((equal name "item-identifier-relative") (list :ii (concatenate 'string doc "#topic")))
68 ((equal name "topic-identifier-unicode") (list :ii (concatenate 'string doc "#ü͡ØΨ㬟͡")))
69 ((equal name "qname-date") '(:si "http://psi.semagia.com/iso8601/1947-01-08"))
70 ((equal name "qname-digits") '(:si "http://psi.example.org/007"))
71 ((equal name "rootless-uri") '(:si "urn:uuid:d84e2777-8928-4bd4-a3e4-8ca835f92304"))
72 ((equal name "subject-identifier-dot") '(:si "http://psi.example.org/fake"))
73 ((equal name "subject-identifier-non-http") '(:si "ftp://psi.example.org/photo.jpg"))
74 ((equal name "subject-identifier-relative") (list :si (concatenate 'string doc "#fake")))
75 ((equal name "subject-locator-dot") '(:sl "http://www.example.org/"))
76 ((equal name "subject-locator-relative") (list :sl (concatenate 'string doc "#fake"))))))
77 (and expected (single-identity map (first expected) (second expected))))))
78 (:prefixes
79 (if (equal name "prefix-with-qm")
80 (single-identity map :sl "http://www.youtube.com/watch?v=fd85SK7dESU")
81 (single-identity map :si "http://psi.example.org/fake")))
82 (:occurrences
83 (let* ((owner (local map doc (if (equal name "occurrence-qname-value") "john" "topic")))
84 (type (local map doc (if (equal name "occurrence-qname-value") "wikipedia" "occtype")))
85 (items (when owner (topic-occurrences owner))) (o (first items))
86 (expected (cond
87 ((equal name "occurrence-decimal-dot-neg2") (list "-0.03" (decimal-datatype)))
88 ((equal name "occurrence-decimal-dot-pos") (list "0.3" (decimal-datatype)))
89 ((equal name "occurrence-decimal-leading-zeros2") (list "2.3" (decimal-datatype)))
90 ((equal name "occurrence-duplicate-string") (list "Occurrence" (string-datatype)))
91 ((equal name "occurrence-duplicate-uri") (list "http://www.example.org/resource" (iri-datatype)))
92 ((equal name "occurrence-qname-value") (list "http://en.wikipedia.org/wiki/John_Lennon" (iri-datatype)))
93 ((equal name "occurrence-string-multiline") (list (format nil "Occurrence~%Value") (string-datatype)))
94 ((equal name "occurrence-string-multiline2") (list (format nil "Occurrence~C~CValue" #\Return #\Newline) (string-datatype)))
95 ((equal name "occurrence-uri-semicolon") (list "http://www.example.org/;something;;somewhere" (iri-datatype))))))
96 (and owner type (= 2 (length (topic-map-topics map))) (= 1 (length items)) (null (topic-map-associations map))
97 (every (lambda (topic) (null (topic-names topic))) (topic-map-topics map))
98 (eq owner (occurrence-parent o)) (eq type (occurrence-type o))
99 (equal expected (list (occurrence-value o) (occurrence-datatype o)))
100 (null (occurrence-scope o)) (null (item-identifiers o)))))
101 (:escapes
102 (let* ((owner (local map doc (cond ((equal name "string-escape") "topic") ((equal name "string-escape2") "A00") (t "zztop"))))
103 (names (when owner (topic-names owner)))
104 (type (find-if (lambda (topic) (member (default-name-type-identifier) (topic-subject-identifiers topic) :test #'equal)) (topic-map-topics map)))
105 (expected (cond ((equal name "string-escape") (list "\\" "Quote (\") character" (format nil "Some whitespace ~C~C~C" #\Newline #\Return #\Tab)))
106 ((equal name "string-escape2") '("A00")) (t '("ZZ Top")))))
107 (and owner type (= 2 (length (topic-map-topics map))) (null (topic-map-associations map))
108 (same-set expected (mapcar #'topic-name-value names))
109 (every (lambda (n) (and (eq owner (topic-name-parent n)) (eq type (topic-name-type n))
110 (null (topic-name-scope n)) (null (topic-name-variants n)))) names))))))
111
112 (defun identity-fields (case)
113 (list (topic-maps.test-foundation:test-case-id case) (topic-maps.test-foundation:test-case-category case)
114 (topic-maps.test-foundation:test-case-source-pathname case) (topic-maps.test-foundation:test-case-document-iri case)
115 (topic-maps.test-foundation:test-case-expected-outcome case) (topic-maps.test-foundation:test-case-expected-baseline case)
116 (topic-maps.test-foundation:test-case-clauses case)))
117 (defun exercise-case (case group)
118 "Record only an actual comparison. Rejections/errors never become empty Maps."
119 (let* ((id (topic-maps.test-foundation:test-case-id case)) (doc (topic-maps.test-foundation:test-case-document-iri case))
120 (name (pathname-name (topic-maps.test-foundation:test-case-source-pathname case)))
121 (identity (copy-tree (identity-fields case))) (source (binary (topic-maps.test-foundation:test-case-source-pathname case)))
122 (evidence nil) (result nil))
123 (handler-case
124 (let* ((map (topic-maps.ctm:deserialize-file (topic-maps.test-foundation:test-case-source-pathname case) :document-iri doc :source-name id))
125 (held (topic-maps.import.tests::handles map)) (before (topic-maps.import.tests::state map held))
126 (actual (topic-maps.cxtm:canonicalize-octets map :base-locator doc))
127 (expected (binary (topic-maps.test-foundation:test-case-expected-baseline case))) (different (mismatch actual expected)))
128 (check (concatenate 'string id ":actual-TMDM-properties-and-EQ-references") (semantic-witness name group map doc))
129 (check (concatenate 'string id ":canonicalization-preserves-raw-objects-and-handles")
130 (equal before (topic-maps.import.tests::state map held)))
131 (check (concatenate 'string id ":deterministic-canonical-octets")
132 (equalp actual (topic-maps.cxtm:canonicalize-octets map :base-locator doc)))
133 (check (concatenate 'string id ":unmodified-source-bytes") (equalp source (binary (topic-maps.test-foundation:test-case-source-pathname case))))
134 (topic-maps.test-foundation:record-result case (if different :cxtm-mismatch :cxtm-match)
135 :diagnostics (when different (list (format nil "First different byte ~D; actual ~D bytes, expected ~D bytes." different (length actual) (length expected)))))
136 (setf evidence (fields "actualBytes" (length actual) "expectedBytes" (length expected)
137 "firstDifferentByte" different "model" (topic-maps.template.tests::observe map))
138 result (list :id id :base doc :actual (topic-maps.ctm::decode-utf8 actual)
139 :expected (topic-maps.ctm::decode-utf8 expected) :match (null different) :bytes (length actual))))
140 ((or topic-maps.ctm:ctm-unsupported topic-maps.cxtm:cxtm-unsupported) (condition)
141 (topic-maps.test-foundation:record-result case :unsupported :diagnostics (list (format nil "~A: ~A" (type-of condition) condition))))
142 (error (condition)
143 (topic-maps.test-foundation:record-result case :execution-error :diagnostics (list (format nil "~A: ~A" (type-of condition) condition)))))
144 (check (concatenate 'string id ":same-contract-identity-after-execution") (equal identity (identity-fields case)))
145 (values
146 (fields "id" id "group" (string-downcase (symbol-name group))
147 "sourcePathname" id "documentIRI" doc
148 "baseline" (format nil "ctm/baseline/~A.ctm.cxtm" name)
149 "status" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-status case))
150 "actualOutcome" (topic-maps.test-foundation::word (topic-maps.test-foundation:test-case-actual-outcome case))
151 "diagnostics" (coerce (topic-maps.test-foundation:test-case-diagnostics case) 'vector)
152 "sourceBytes" (length source) "comparison" evidence)
153 result)))
154
155
156 (defun case-state (case)
157 (copy-tree
158 (list (identity-fields case)
159 (topic-maps.test-foundation:test-case-status case)
160 (topic-maps.test-foundation:test-case-actual-outcome case)
161 (topic-maps.test-foundation:test-case-diagnostics case))))
162
163 (defun run-tests ()
164 ;; Use the same Case Records populated by the retained suites. A direct ASDF
165 ;; invocation may initialize those suites, but must never replace their corpus.
166 (unless topic-maps.ctm.tests::*cxtm-corpus*
167 (topic-maps.include.tests:run-tests))
168 (setf *checks* nil *rows* nil *results* nil)
169 (let* ((corpus topic-maps.ctm.tests::*cxtm-corpus*)
170 (root (topic-maps.test-foundation:corpus-root corpus))
171 (ids (loop for (group . names) in *groups*
172 append (mapcar (lambda (name) (format nil "ctm/in/~A.ctm" name)) names)))
173 (cases (topic-maps.test-foundation:corpus-cases corpus))
174 (previous (mapcar (lambda (case) (cons case (case-state case))) cases))
175 (old-results (copy-tree topic-maps.ctm.tests::*cxtm-results*))
176 (base (format nil "https://ctm-tests.example.invalid/~A/"
177 (getf (topic-maps.test-foundation:corpus-metadata corpus) :commit))))
178 (setf *entry-statuses*
179 (copy-list (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses)))
180 (check "registration:forty-distinct-selected-originals"
181 (and (= 40 (length ids)) (= 40 (length (remove-duplicates ids :test #'equal)))
182 (equal '(10 13 9 4 4) (mapcar (lambda (group) (length (cdr group))) *groups*))))
183 (check "registration:existing-corpus-correspondence"
184 (getf (topic-maps.test-foundation:verify-correspondence corpus) :ok))
185 (dolist (group *groups*)
186 (dolist (name (cdr group))
187 (let* ((id (format nil "ctm/in/~A.ctm" name))
188 (selected (topic-maps.test-foundation:select-cases corpus :id id))
189 (case (first selected)))
190 (check (concatenate 'string id ":unique-existing-positive-contract")
191 (and (= 1 (length selected)) (member case cases :test #'eq)
192 (eq :valid (topic-maps.test-foundation:test-case-category case))
193 (eq :cxtm-equal (topic-maps.test-foundation:test-case-expected-outcome case))
194 (eq :not-run (topic-maps.test-foundation:test-case-status case))
195 (null (topic-maps.test-foundation:test-case-actual-outcome case))))
196 (check (concatenate 'string id ":pinned-source-baseline-and-document-IRI")
197 (and (equal (truename (merge-pathnames id root))
198 (truename (topic-maps.test-foundation:test-case-source-pathname case)))
199 (equal (truename (merge-pathnames (format nil "ctm/baseline/~A.ctm.cxtm" name) root))
200 (truename (topic-maps.test-foundation:test-case-expected-baseline case)))
201 (equal (concatenate 'string base id)
202 (topic-maps.test-foundation:test-case-document-iri case))))
203 (multiple-value-bind (row result) (exercise-case case (car group))
204 (push row *rows*)
205 (when result (push result *results*)))
206 (format t "~&REGISTERED-ORIGINAL ~A: ~A~{ ~A~}~%" id
207 (topic-maps.test-foundation:test-case-status case)
208 (topic-maps.test-foundation:test-case-diagnostics case)))))
209 (setf *rows* (nreverse *rows*) *results* (nreverse *results*))
210 (check "registration:forty-actual-byte-comparisons-pass"
211 (and (= 40 (length *rows*) (length *results*))
212 (every (lambda (result) (getf result :match)) *results*)
213 (every (lambda (id) (eq :pass (topic-maps.test-foundation:test-case-status
214 (first (topic-maps.test-foundation:select-cases corpus :id id))))) ids)))
215 (check "registration:all-previous-case-records-and-observations-preserved"
216 (and (eq corpus topic-maps.ctm.tests::*cxtm-corpus*)
217 (equal cases (topic-maps.test-foundation:corpus-cases corpus))
218 (every (lambda (entry)
219 (if (member (topic-maps.test-foundation:test-case-id (car entry)) ids :test #'equal)
220 (equal (first (cdr entry)) (identity-fields (car entry)))
221 (equal (cdr entry) (case-state (car entry))))) previous)
222 (equal old-results topic-maps.ctm.tests::*cxtm-results*)))
223 (let ((after (getf (topic-maps.test-foundation:summarize-corpus corpus) :statuses)))
224 (check "registration:only-forty-NOT-RUN-to-PASS-transitions"
225 (and (= (+ 40 (getf *entry-statuses* :pass)) (getf after :pass))
226 (= (- (getf *entry-statuses* :not-run) 40) (getf after :not-run))
227 (= (getf *entry-statuses* :fail) (getf after :fail))
228 (= (getf *entry-statuses* :unsupported) (getf after :unsupported)))))
229 (setf topic-maps.ctm.tests::*cxtm-results*
230 (append topic-maps.ctm.tests::*cxtm-results* *results*)))
231 (check "registration:unique-check-names"
232 (= (length *checks*) (length (remove-duplicates *checks* :test #'equal))))
233 (setf *checks* (nreverse *checks*))
234 (format t "~&REGISTERED-ORIGINALS-PASS: ~D checks; forty actual byte-exact original contracts~%" (length *checks*))
235 t)
236
237 (defun write-reports (prefix)
238 (with-open-file (out (concatenate 'string prefix ".registration.json")
239 :direction :output :if-exists :supersede :if-does-not-exist :create)
240 (topic-maps.test-foundation::emit-json-value
241 (fields "kind" "ctm-registered-original-execution"
242 "passed" (length *checks*) "failed" 0
243 "entryStatuses" (topic-maps.test-foundation::counts-object *entry-statuses*)
244 "checks" (coerce *checks* 'vector) "cases" (coerce *rows* 'vector)) out)
245 (terpri out))
246 (topic-maps.ctm.tests::write-cxtm-report
247 (concatenate 'string prefix ".validation.json")
248 (concatenate 'string prefix ".cxtm.json")
249 :additional-checks *checks*
250 :scope "Slice 20A: forty newly executed original CTM comparisons; prior Case Records, observations and four known DateTime/Include FAILs retained. Full retained regression and XTM integration are evidenced separately."))