]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20b.py
Register five original XTM mergeMap contracts (Slice 22H)
[topic-maps.git] / scripts / validate-slice-20b.py
1 #!/usr/bin/env python3
2 """Validate seven intended original CTM rejections and every retained suite offline."""
3 import argparse
4 import datetime
5 import hashlib
6 import importlib.util
7 import json
8 import os
9 from pathlib import Path
10 import re
11 import shutil
12 import subprocess
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '9044114749dedc72f190f4aa9ba0443dfe7860b5'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 EXPECTED = {
18 'occurrence-double-positive-infinity': ('expected-occurrence-value', 15, 2, 10, 'I'),
19 'occurrence-double-negative-infinity': ('expected-occurrence-value', 15, 2, 10, '-'),
20 'qname-hyphen': ('unexpected-token', 38, 3, 3, ':'),
21 'string-unbalanced': ('unterminated-string', 8, 1, 9, '"'),
22 'string-unicode': ('invalid-unicode-escape', 11, 1, 12, '"'),
23 'string-unicode2': ('invalid-unicode-escape', 15, 1, 16, 'G'),
24 'topic-id-illegal': ('unexpected-character', 0, 1, 1, '\u0361'),
25 }
26
27
28 def main():
29 # Reuse the existing validation utilities and full 20A/19D2 chain.
30 spec = importlib.util.spec_from_file_location('slice20a', ROOT / 'scripts/validate-slice-20a.py')
31 previous = importlib.util.module_from_spec(spec)
32 spec.loader.exec_module(previous)
33 read, save, sha, command, snapshot = previous.read, previous.save, previous.sha, previous.command, previous.snapshot
34 parser = argparse.ArgumentParser(description=__doc__)
35 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
36 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20b')
37 args = parser.parse_args()
38 runtime = shutil.which(args.sbcl)
39 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
40 output = args.output.resolve()
41 output.mkdir(parents=True, exist_ok=False)
42 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
43 test_path = 'tests/ctm-registered-originals.lisp'
44 tracked_tests = command(['git', 'ls-tree', '-r', '--name-only', BASE, '--', 'tests']).splitlines()
45 protected_paths = ['src', 'vendor', 'references', 'patches', 'flake.nix', 'flake.lock',
46 'scripts/test.sh', 'scripts/prepare-cxml.py', 'scripts/validate-slice-19d2.py',
47 'scripts/validate-slice-20a.py', 'docs/slice-20a-registration.md'] + [
48 p for p in tracked_tests if p != test_path]
49 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
50 # Existing positive tests and execution order are retained verbatim; only
51 # the exports of new entry points and appended negative suite are allowed.
52 prior_test = subprocess.check_output(['git', 'show', BASE + ':' + test_path], cwd=ROOT, text=True)
53 prior_test = prior_test.replace('(:export #:run-tests #:write-reports #:candidate-groups))',
54 '(:export #:run-tests #:write-reports #:candidate-groups\n #:run-negative-tests #:write-negative-reports))', 1)
55 assert (ROOT / test_path).read_text().startswith(prior_test)
56 for name in ['topic-maps.asd', 'scripts/fresh-sbcl.lisp']:
57 prior = subprocess.check_output(['git', 'show', BASE + ':' + name], cwd=ROOT, text=True)
58 assert (ROOT / name).read_text().startswith(prior)
59 protected = snapshot(p for directory in ['src', 'vendor', 'references', 'patches']
60 for p in (ROOT / directory).rglob('*'))
61 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
62 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
63 for p in (ROOT / directory).rglob('*')
64 if p.suffix in ['.lisp', '.py', '.sh', '.md'])
65 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
66 prefix = output / 'current'
67 env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(prefix))
68 invocation = ['python3', 'scripts/validate-slice-20a.py', '--sbcl', runtime,
69 '--output', str(output / 'retained-20a')]
70 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
71 stderr=subprocess.STDOUT, timeout=600)
72 (output / 'retained-driver.log').write_bytes(result.stdout)
73 assert result.returncode == 0, 'Retained regression failed; see retained-driver.log and retained-20a/retained/*.log'
74 retained = read(output / 'retained-20a/execution.json')
75 before = read(output / 'retained-20a/current.validation.json')
76 assert before == read(ROOT / 'reports/slice-20a/current.validation.json')
77 for suffix in ['cxtm', 'registration']:
78 assert read(output / f'retained-20a/current.{suffix}.json') == read(ROOT / f'reports/slice-20a/current.{suffix}.json')
79 final = read(Path(str(prefix) + '.validation.json'))
80 cxtm = read(Path(str(prefix) + '.cxtm.json'))
81 negative = read(Path(str(prefix) + '.negative.json'))
82 assert negative['entryStatuses'] == before['statuses'] == {
83 'pass': 245, 'fail': 4, 'unsupported': 0, 'not-run': 24}
84 ids = ['ctm/invalid/' + name + '.ctm' for name in EXPECTED]
85 assert [c['id'] for c in negative['cases']] == ids
86 assert negative['failed'] == 0
87 assert negative['passed'] == len(negative['checks']) == len(set(negative['checks']))
88 log = (output / 'retained-20a/retained/regression.log').read_text()
89 marker = re.search(r'(?m)^REGISTERED-NEGATIVES-PASS: (\d+) checks; (\d+) intended original rejections registered; (\d+) remain NOT-RUN', log)
90 assert marker and int(marker.group(1)) == negative['passed']
91 observed = re.findall(r'^REGISTERED-NEGATIVE (ctm/invalid/[^:]+): (PASS|NOT-RUN);', log, re.M)
92 assert [i for i, _ in observed] == ids
93 assert len(final['cases']) == len(before['cases']) == 273
94 assert [c['id'] for c in final['cases']] == [c['id'] for c in before['cases']]
95 assert {k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope']} == {
96 k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope']}
97 old_cxtm = read(output / 'retained-20a/current.cxtm.json')
98 assert cxtm['cases'] == old_cxtm['cases']
99 assert cxtm['contractCounts'] == old_cxtm['contractCounts'] == {'pass': 217, 'fail': 4}
100 rows = {c['id']: c for c in negative['cases']}
101 candidates, changes = [], []
102 for a, b in zip(before['cases'], final['cases']):
103 if a['id'] not in rows:
104 assert a == b, a['id']
105 continue
106 row = rows[a['id']]
107 name = Path(a['id']).stem
108 code, position, line, column, character = EXPECTED[name]
109 source = ROOT / 'vendor/cxtm' / a['sourcePathname']
110 text = source.read_bytes().decode('utf-8')
111 assert a['status'] == 'not-run' and a['actualOutcome'] is None
112 assert a['category'] == 'invalid' and a['expectedOutcome'] == 'ctm-rejection' and a['expectedCXTM'] is None
113 assert row['sourcePathname'] == a['sourcePathname'] and row['documentIRI'] == a['documentIRI']
114 assert a['documentIRI'] == 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + a['id']
115 condition = row['condition']
116 qualified = row['normativeValidated']
117 if qualified:
118 assert condition == row['repeatCondition']
119 assert condition['condition'] == 'CTM-REJECTION'
120 assert (condition['code'], condition['position'], condition['line'], condition['column']) == (code, position, line, column)
121 assert condition['source'] == a['id'] and condition['positionUnit'] == 'character'
122 assert condition['cause'] is condition['context'] is None
123 assert text[position] == character
124 # Independent character-index/line/column calculation from original bytes.
125 assert text[:position].count('\n') + 1 == line
126 assert len(text[:position].rsplit('\n', 1)[-1]) + 1 == column
127 assert row['sourceWitness'] and row['lexicalWitness'] and row['reproducibleCondition']
128 assert row['control']['inMemoryOnly'] and row['control']['accepted'] and row['control']['condition'] is None
129 assert row['notRegisteredReason'] is None and row['clauses']
130 control = row['control']['source']
131 model = row['control']['model']
132 assert control != text and model['associationCount'] == 0
133 if name.startswith('occurrence-double-'):
134 value = '-1' if 'negative' in name else '1'
135 assert control == text.replace('-INF' if 'negative' in name else 'INF', value)
136 occurrences = [o for t in model['topics'] for o in t['occurrences']]
137 assert model['topicCount'] == 2 and len(occurrences) == 1
138 assert occurrences[0]['value'] == value and occurrences[0]['datatype'] == 'http://www.w3.org/2001/XMLSchema#integer'
139 assert occurrences[0]['parent'] == 1 and occurrences[0]['type'] == 2
140 elif name == 'qname-hyphen':
141 assert control == text.replace('ex:-issue', 'ex:issue')
142 assert [(t['kind'], t['value'], t['position']) for t in row['tokens'][3:6]] == [
143 ('identifier', 'ex', 36), ('punctuation', ':', 38), ('punctuation', '-', 39)]
144 assert model['topicCount'] == 1 and model['topics'][0]['subjectIdentifiers'] == ['http://psi.example.org/issue']
145 elif name == 'topic-id-illegal':
146 assert control == 'a\u0361' + text[2:]
147 assert model['topicCount'] == 1 and model['topics'][0]['itemIdentifiers'] == [a['documentIRI'] + '#a\u0361']
148 else:
149 expected_control = {
150 'string-unbalanced': text[:-1] + '".',
151 'string-unicode': text.replace('\\U0041', '\\u0041'),
152 'string-unicode2': text.replace('\\U00GG00', '\\U000041'),
153 }[name]
154 assert control == expected_control
155 names = [n for t in model['topics'] for n in t['names']]
156 assert model['topicCount'] == 2 and len(names) == 1
157 assert names[0]['value'] == ('Foo"' if name == 'string-unbalanced' else 'A')
158 assert names[0]['parent'] == 1 and names[0]['type'] == 2
159 assert row['lexicalCondition'] == condition
160 if name == 'string-unicode2':
161 assert 'Printed production [43]' in row['qualification']
162 assert b == dict(a, status='pass', actualOutcome='ctm-rejection', diagnostics=[condition['message']])
163 changes.append({'id': a['id'], 'before': a['status'], 'after': b['status']})
164 else:
165 assert b == a and row['status'] == 'not-run' and row['actualOutcome'] is None
166 assert row['notRegisteredReason']
167 candidates.append({'id': a['id'], 'status': b['status'], 'sourceSHA256': sha(source),
168 'documentIRI': a['documentIRI'], 'normativeReason': row['normativeReason'],
169 'clauses': row['clauses'], 'condition': condition,
170 'normativeValidated': qualified, 'qualification': row['qualification']})
171 qualified = len(changes)
172 assert int(marker.group(2)) == qualified and int(marker.group(3)) == 7 - qualified
173 assert final['statuses'] == {'pass': 245 + qualified, 'fail': 4, 'unsupported': 0, 'not-run': 24 - qualified}
174 xtm = read(output / 'retained-20a/retained/xtm.json')
175 assert xtm == read(ROOT / 'reports/slice-20a/retained/xtm.json')
176 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
177 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
178 save(output / 'candidates.json', candidates)
179 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
180 (output / 'contracts.md').write_text(
181 '# Slice 20B: actual original rejections\n\n'
182 'Positions count Unicode characters from zero; lines and columns start at one.\n\n'
183 '| Original CTM | Status | Condition | Code | Position | Line:column |\n'
184 '| --- | --- | --- | --- | ---: | --- |\n' +
185 ''.join(f"| `{Path(c['id']).name}` | {c['status'].upper()} | `{c['condition']['condition']}` | "
186 f"`{c['condition']['code']}` | {c['condition']['position']} | "
187 f"{c['condition']['line']}:{c['condition']['column']} |\n" for c in candidates) +
188 '\nReviewed causes, the QName diagnostic trace and the Unicode printed-class qualification '
189 'are documented in `../../docs/slice-20b-negative-registration.md`. Full actual conditions, '
190 'repeated calls and accepted in-memory control models are in `current.negative.json`.\n')
191 assert protected == {p: sha(ROOT / p) for p in protected}
192 assert historical == {p: sha(ROOT / p) for p in historical}
193 assert sources == {p: sha(ROOT / p) for p in sources}
194 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
195 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
196 evidence = {
197 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
198 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
199 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
200 'retainedValidationCommand': invocation, 'retainedExecutionEvidence': 'retained-20a/execution.json',
201 'retainedLispChecks': retained['retainedLispChecks'], 'xmlQualificationChecks': retained['xmlQualificationChecks'],
202 'standaloneXTMChecks': retained['standaloneXTMChecks'], 'xtmMergemapChecks': retained['xtmMergemapChecks'],
203 'slice20aChecks': retained['registeredOriginalChecks'], 'slice20aActualByteExactContracts': 40,
204 'registeredNegativeChecks': negative['passed'], 'newIntendedOriginalRejections': qualified,
205 'pythonUnitTests': retained['pythonUnitTests'], 'nfcOracleComparisons': retained['nfcOracleComparisons'],
206 'network': retained['network'], 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'],
207 'ctmStatusChanges': changes, 'xtmCorpusStatuses': retained['xtmCorpusStatuses'],
208 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
209 'oldOriginalCXTMComparisonsUnchanged': len(old_cxtm['cases']),
210 'allHistoricalReportBytesUnchanged': True, 'allProtectedFileBytesUnchanged': True,
211 'positiveRegistrationImplementationRetained': True,
212 'normativeReview': {'source': 'https://www.isotopicmaps.org/ctm/ctm.html',
213 'consultedDate': '2026-10-19', 'description': 'docs/slice-20b-negative-registration.md',
214 'qualification': 'Printed [43] permits more letters than hexadecimal decoding prose; string-unicode2 is justified by the latter.'},
215 'persistentLispImagesTouched': False, 'fullOfflineNixRebuild': retained['fullOfflineNixRebuild'],
216 'acceptancePassed': qualified == 7,
217 }
218 save(output / 'execution.json', evidence)
219 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
220 'standaloneXTMChecks', 'xtmMergemapChecks', 'slice20aChecks', 'registeredNegativeChecks',
221 'newIntendedOriginalRejections', 'ctmStatusesAfter', 'xtmCorpusStatuses', 'acceptancePassed']}, indent=2))
222 assert evidence['acceptancePassed'], 'Some intended causes remain unproved; those records stay NOT-RUN. See current.negative.json.'
223
224
225 if __name__ == '__main__':
226 main()