]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/corpus.lisp
Implement XTM 2.0 identity and instanceOf reader with offline evidence
[topic-maps.git] / src / corpus.lisp
1 (in-package #:topic-maps.test-foundation)
2
3 (defstruct test-case
4 "A test contract and its recorded result. Creating it executes no CTM."
5 (id "" :type string)
6 category
7 source-pathname
8 (document-iri "" :type string)
9 expected-outcome
10 expected-baseline
11 actual-outcome
12 (status :not-run)
13 diagnostics
14 ;; NIL means unreviewed, not that no normative clause applies.
15 clauses)
16
17 (defstruct corpus root origin metadata cases baselines auxiliaries)
18
19 (defun auxiliary-role (relative)
20 (cond ((equal relative "ctm/STATUS.txt") :draft-qualification)
21 ((equal relative "ctm/MISSING.txt") :upstream-coverage-notes)
22 ((uiop:string-prefix-p "ctm/baseline/" relative) :baseline-non-cxtm-artifact)
23 ((or (equal (pathname-type relative) "sub")
24 (equal (pathname-type relative) "sub2")) :include-or-mergemap-resource)
25 (t :unclassified-resource)))
26
27 (defun discover-corpus (&key (root (default-corpus-root))
28 (metadata (read-corpus-metadata))
29 (origin :upstream-ctm) document-base)
30 "Discover files by directory and extension; do not read CTM syntax."
31 (let* ((root (uiop:ensure-directory-pathname (truename root)))
32 (ctm (merge-pathnames "ctm/" root))
33 (base (or document-base
34 (format nil "https://ctm-tests.example.invalid/~A/"
35 (getf metadata :commit))))
36 (cases nil)
37 (baselines (remove-if-not #'(lambda (p) (file-type-p p "cxtm"))
38 (sorted-files (merge-pathnames "baseline/" ctm) root)))
39 (inputs nil)
40 (clauses (when (eq origin :upstream-ctm) (read-clause-map))))
41 (dolist (directory '("in/" "invalid/"))
42 (unless (uiop:directory-exists-p (merge-pathnames directory ctm))
43 (error "Missing CTM input directory ~A" directory))
44 (let* ((valid (equal directory "in/"))
45 (input-root (merge-pathnames directory ctm)))
46 (dolist (path (sorted-files input-root root))
47 (when (file-type-p path "ctm")
48 (push path inputs)
49 (push (make-test-case
50 :id (relative-file-name path root)
51 :clauses (copy-list (cdr (assoc (relative-file-name path root) clauses :test #'string=)))
52 :category (if valid :valid :invalid)
53 :source-pathname path
54 :document-iri (document-iri base (relative-file-name path root))
55 :expected-outcome (if valid :cxtm-equal :ctm-rejection)
56 :expected-baseline
57 (when valid
58 (merge-pathnames
59 (concatenate 'string
60 (relative-file-name path input-root) ".cxtm")
61 (merge-pathnames "baseline/" ctm))))
62 cases)))))
63 (let ((auxiliary
64 (loop for path in (sorted-files ctm root)
65 unless (or (member path inputs :test #'equal)
66 (member path baselines :test #'equal))
67 collect (let ((id (relative-file-name path root)))
68 (list :id id :source-pathname path
69 :document-iri (document-iri base id)
70 :role (auxiliary-role id))))))
71 (make-corpus :root root :origin origin :metadata (copy-tree metadata)
72 :cases (sort cases #'string< :key #'test-case-id)
73 :baselines baselines :auxiliaries auxiliary))))
74
75 (defun select-cases (corpus &key category id)
76 "Return cases in stable ID order. An unknown ID yields NIL."
77 (unless (member category '(nil :valid :invalid))
78 (error "Unknown case category ~S." category))
79 (remove-if-not (lambda (case)
80 (and (or (null category) (eq category (test-case-category case)))
81 (or (null id) (string= id (test-case-id case)))))
82 (corpus-cases corpus)))
83
84 (defun verify-correspondence (corpus)
85 "Return structural evidence without changing execution statuses."
86 (let* ((cases (corpus-cases corpus))
87 (expected (remove nil (mapcar #'test-case-expected-baseline cases)))
88 (missing-inputs
89 (loop for case in cases
90 unless (probe-file (test-case-source-pathname case))
91 collect (test-case-id case)))
92 (missing-baselines
93 (loop for case in cases
94 when (and (test-case-expected-baseline case)
95 (not (probe-file (test-case-expected-baseline case))))
96 collect (test-case-id case)))
97 (unmatched
98 (loop for path in (corpus-baselines corpus)
99 unless (member path expected :test #'equal)
100 collect (relative-file-name path (corpus-root corpus))))
101 (duplicate-ids
102 (loop with seen = (make-hash-table :test #'equal)
103 for case in cases for id = (test-case-id case)
104 if (gethash id seen) collect id
105 else do (setf (gethash id seen) t))))
106 (list :ok (not (or missing-inputs missing-baselines unmatched duplicate-ids))
107 :missing-inputs missing-inputs :missing-baselines missing-baselines
108 :unmatched-baselines unmatched :duplicate-ids duplicate-ids)))
109
110 (defun reset-result (case)
111 (setf (test-case-status case) :not-run
112 (test-case-actual-outcome case) nil
113 (test-case-diagnostics case) nil)
114 case)
115
116 (defun record-result (case outcome &key diagnostics)
117 "Classify an outcome supplied by an executor. This function executes nothing.
118 CTM rejection means an identified syntax/semantic rejection, never an I/O or
119 unsupported-feature exception. Infrastructure fixtures exercise this protocol."
120 (unless (and (member outcome '(:cxtm-match :cxtm-mismatch :ctm-rejection
121 :ctm-acceptance :unsupported :execution-error))
122 (member (test-case-expected-outcome case) '(:cxtm-equal :ctm-rejection))
123 (listp diagnostics) (every #'stringp diagnostics))
124 (error "Invalid outcome contract ~S / ~S." outcome diagnostics))
125 (setf (test-case-actual-outcome case) outcome
126 (test-case-diagnostics case) (copy-list diagnostics)
127 (test-case-status case)
128 (cond ((eq outcome :unsupported) :unsupported)
129 ((or (and (eq (test-case-expected-outcome case) :cxtm-equal)
130 (eq outcome :cxtm-match))
131 (and (eq (test-case-expected-outcome case) :ctm-rejection)
132 (eq outcome :ctm-rejection))) :pass)
133 (t :fail)))
134 case)
135
136 (defun summarize-corpus (corpus)
137 (let* ((cases (corpus-cases corpus))
138 (counts (list :valid (length (select-cases corpus :category :valid))
139 :invalid (length (select-cases corpus :category :invalid))
140 :baselines (length (corpus-baselines corpus))
141 :auxiliary (length (corpus-auxiliaries corpus))))
142 (statuses
143 (loop for status in '(:pass :fail :unsupported :not-run)
144 append (list status (count status cases :key #'test-case-status))))
145 (expected (getf (corpus-metadata corpus) :expected-counts)))
146 (list :origin (corpus-origin corpus) :repository (getf (corpus-metadata corpus) :repository)
147 :commit (getf (corpus-metadata corpus) :commit)
148 :qualification (getf (corpus-metadata corpus) :qualification)
149 :counts counts :counts-match-pin (or (null expected) (equal counts expected))
150 :statuses statuses :correspondence (verify-correspondence corpus))))