]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - tests/foundation.lisp
test(ctm): register seven reviewed negative Slice 20B contracts
[topic-maps.git] / tests / foundation.lisp
1 (in-package #:topic-maps.tests)
2
3 (defvar *checks* nil)
4 (defvar *last-checks* nil)
5
6 (defmacro check (label form)
7 `(progn
8 (unless ,form (error "Infrastructure check failed: ~A" ,label))
9 (push ,label *checks*)))
10
11 (defun signals-error-p (thunk)
12 (handler-case (progn (funcall thunk) nil) (error () t)))
13
14 (defun fixture-root ()
15 (asdf:system-relative-pathname "topic-maps/tests" "tests/fixtures/corpus/"))
16
17 (defun status-fixture (expected)
18 ;; This record is not imported from the CTM corpus, and no parser is mocked.
19 (make-test-case :id "infrastructure/outcome-protocol"
20 :category (if (eq expected :cxtm-equal) :valid :invalid)
21 :expected-outcome expected))
22
23 (defun run-tests ()
24 (let* ((*checks* nil)
25 (corpus (discover-corpus))
26 (summary (summarize-corpus corpus))
27 (counts (getf summary :counts))
28 (cases (corpus-cases corpus)))
29 (check "pin-counts-agree" (getf summary :counts-match-pin))
30 (check "input-count-is-valid-plus-invalid"
31 (= (length cases) (+ (getf counts :valid) (getf counts :invalid))))
32 (check "all-imported-cases-not-run"
33 (every (lambda (case) (and (eq :not-run (test-case-status case))
34 (null (test-case-actual-outcome case)))) cases))
35 (check "no-imported-ctm-pass"
36 (equal (getf summary :statuses)
37 (list :pass 0 :fail 0 :unsupported 0 :not-run (length cases))))
38 (check "source-files-exist" (every (lambda (case) (probe-file (test-case-source-pathname case))) cases))
39 (check "valid-baselines-exist"
40 (every (lambda (case) (probe-file (test-case-expected-baseline case)))
41 (select-cases corpus :category :valid)))
42 (check "invalid-contract-is-rejection-without-baseline"
43 (every (lambda (case) (and (eq :ctm-rejection (test-case-expected-outcome case))
44 (null (test-case-expected-baseline case))))
45 (select-cases corpus :category :invalid)))
46 (check "baseline-correspondence" (getf (verify-correspondence corpus) :ok))
47 (check "selection-counts"
48 (and (= (getf counts :valid) (length (select-cases corpus :category :valid)))
49 (= (getf counts :invalid) (length (select-cases corpus :category :invalid)))))
50 (let ((selected (select-cases corpus :id "ctm/in/simple.ctm")))
51 (check "stable-id-selection" (= 1 (length selected)))
52 (check "selected-record-is-original" (member (first selected) cases :test #'eq))
53 (check "combined-selection" (null (select-cases corpus :category :invalid :id "ctm/in/simple.ctm")))
54 (check "expected-outcome-exposed" (eq :cxtm-equal (test-case-expected-outcome (first selected))))
55 (check "document-iri-independent-of-filesystem"
56 (string= (test-case-document-iri (first selected))
57 (format nil "https://ctm-tests.example.invalid/~A/ctm/in/simple.ctm"
58 (getf (corpus-metadata corpus) :commit)))))
59 (check "reviewed-case-clause-exposed"
60 (member "CTM 3.3.6" (test-case-clauses
61 (first (select-cases corpus :id "ctm/in/simple.ctm")))
62 :test #'string=))
63 (check "unreviewed-case-clause-not-invented"
64 (null (test-case-clauses
65 (first (select-cases corpus :id "ctm/in/template-simple.ctm")))))
66 (check "unknown-id-yields-no-cases" (null (select-cases corpus :id "missing")))
67 (check "unknown-category-is-error" (signals-error-p (lambda () (select-cases corpus :category :bogus))))
68 (check "case-order-is-deterministic"
69 (equal (mapcar #'test-case-id cases)
70 (sort (mapcar #'test-case-id cases) #'string<)))
71 (check "fresh-discovery-report-equality"
72 (string= (report-string corpus) (report-string (discover-corpus))))
73 (check "auxiliaries-not-inputs"
74 (notany (lambda (resource) (find (getf resource :id) cases :key #'test-case-id :test #'string=))
75 (corpus-auxiliaries corpus)))
76 (check "baseline-ctm-artifact-not-input"
77 (eq :baseline-non-cxtm-artifact
78 (getf (find "ctm/baseline/name-type.ctm" (corpus-auxiliaries corpus)
79 :key (lambda (r) (getf r :id)) :test #'string=) :role)))
80 (check "auxiliary-resource-registered"
81 (eq :include-or-mergemap-resource
82 (getf (find "ctm/in/include-wildcard2.sub2" (corpus-auxiliaries corpus)
83 :key (lambda (r) (getf r :id)) :test #'string=) :role)))
84 (check "draft-qualification-preserved"
85 (search "2010-03-31" (getf summary :qualification)))
86 (check "canonicalizer-independent-model-dependency"
87 (equal (asdf:system-depends-on (asdf:find-system "topic-maps/cxtm"))
88 '("topic-maps/tmdm")))
89 ;; Incomplete independent fixture: structural problems must be exposed,
90 ;; but must never mark an imported syntax test PASS or FAIL.
91 (let* ((fixture (discover-corpus :root (fixture-root)
92 :origin :infrastructure-fixture
93 :metadata '(:commit "fixture" :qualification "Infrastructure only")
94 :document-base "https://fixtures.example.invalid/corpus/"))
95 (verification (verify-correspondence fixture)))
96 (check "fixture-discovery-counts"
97 (equal (getf (summarize-corpus fixture) :counts)
98 '(:valid 2 :invalid 1 :baselines 2 :auxiliary 2)))
99 (check "missing-baseline-detected"
100 (equal (getf verification :missing-baselines) '("ctm/in/beta.ctm")))
101 (check "unmatched-baseline-detected"
102 (equal (getf verification :unmatched-baselines) '("ctm/baseline/orphan.ctm.cxtm")))
103 (check "correspondence-errors-do-not-pass" (null (getf verification :ok)))
104 (check "fixture-cases-still-not-run" (every (lambda (c) (eq :not-run (test-case-status c))) (corpus-cases fixture)))
105 (check "fixture-iri-policy-explicit"
106 (string= "https://fixtures.example.invalid/corpus/ctm/in/alpha.ctm"
107 (test-case-document-iri (first (corpus-cases fixture)))))
108 (check "iri-does-not-follow-fixture-directory"
109 (string= (report-string fixture)
110 (report-string (discover-corpus :root (fixture-root)
111 :origin :infrastructure-fixture
112 :metadata '(:commit "fixture" :qualification "Infrastructure only")
113 :document-base "https://fixtures.example.invalid/corpus/"))))
114 (let ((case (first (corpus-cases fixture))))
115 (setf (test-case-source-pathname case) (merge-pathnames "absent.ctm" (fixture-root)))
116 (check "missing-source-detected" (equal (getf (verify-correspondence fixture) :missing-inputs)
117 '("ctm/in/alpha.ctm"))))
118 (let ((case (second (corpus-cases fixture))))
119 (setf (test-case-id case) (test-case-id (first (corpus-cases fixture))))
120 (check "duplicate-id-detected" (getf (verify-correspondence fixture) :duplicate-ids))))
121 ;; Result fixtures exercise only state classification; not CTM recognition.
122 (dolist (expected '(:cxtm-equal :ctm-rejection))
123 (let ((case (status-fixture expected)))
124 (check (format nil "~A-initial-not-run" expected) (eq :not-run (test-case-status case)))
125 (record-result case :unsupported :diagnostics '("Fixture capability unavailable"))
126 (check (format nil "~A-unsupported-never-pass" expected) (eq :unsupported (test-case-status case)))
127 (record-result case :execution-error :diagnostics '("Fixture I/O error, not CTM rejection"))
128 (check (format nil "~A-execution-error-fails" expected) (eq :fail (test-case-status case)))
129 (record-result case (if (eq expected :cxtm-equal) :cxtm-match :ctm-rejection))
130 (check (format nil "~A-contract-success" expected) (eq :pass (test-case-status case)))
131 (record-result case (if (eq expected :cxtm-equal) :ctm-rejection :ctm-acceptance))
132 (check (format nil "~A-contract-mismatch" expected) (eq :fail (test-case-status case)))
133 (record-result case :cxtm-mismatch :diagnostics '("Independent fixture mismatch"))
134 (check (format nil "~A-diagnostics-retained" expected)
135 (equal '("Independent fixture mismatch") (test-case-diagnostics case)))
136 (reset-result case)
137 (check (format nil "~A-reset-not-run" expected)
138 (and (eq :not-run (test-case-status case)) (null (test-case-actual-outcome case))
139 (null (test-case-diagnostics case))))))
140 (check "unknown-outcome-is-error"
141 (signals-error-p (lambda () (record-result (status-fixture :cxtm-equal) :bogus))))
142 (check "relative-document-base-rejected"
143 (signals-error-p (lambda () (discover-corpus :document-base "relative/"))))
144 (check "fragment-document-base-rejected"
145 (signals-error-p (lambda () (discover-corpus :document-base "https://example.invalid/#fragment/"))))
146 (check "report-no-machine-path"
147 (and (not (search (namestring (corpus-root corpus)) (report-string corpus)))
148 (not (search "file://" (report-string corpus)))))
149 (check "report-no-implementation-claim"
150 (search "\"ctmParsing\":false" (report-string corpus)))
151 (check "report-keeps-all-imported-not-run" (= (length cases) (getf (getf summary :statuses) :not-run)))
152 (setf *last-checks* (nreverse *checks*))
153 (format t "~&FOUNDATION-PASS: ~D infrastructure checks; ~D valid, ~D invalid, ~D baselines, ~D auxiliaries; ~D CTM cases NOT-RUN, zero CTM passes.~%"
154 (length *last-checks*) (getf counts :valid) (getf counts :invalid)
155 (getf counts :baselines) (getf counts :auxiliary) (length cases))
156 t))
157
158 (defun write-tested-report (pathname)
159 (run-tests)
160 (write-validation-report (discover-corpus) pathname
161 :infrastructure-checks *last-checks*))