]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20g.py
Verify CTM/XTM semantic equivalence through TMDM and CXTM (Slice 22F)
[topic-maps.git] / scripts / validate-slice-20g.py
1 #!/usr/bin/env python3
2 """Validate four Triple String originals and the complete retained offline suite."""
3 import argparse
4 import datetime
5 import importlib.util
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '1ef5768ce6913f0c04da111d5f525b6d50a27e0b'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 CONTRACTS = {
18 'occurrence-string-triple': (41, 367, 'Occurrence " Value'),
19 'occurrence-string-triple2': (48, 370, 'Occurrence"" " Value"'),
20 'unicode-escape-u4-triple-string': (33, 331, 'ZZ \nTop'),
21 'unicode-escape-u6-triple-string': (37, 331, 'ZZ \nTop'),
22 }
23
24
25 def main():
26 spec = importlib.util.spec_from_file_location('slice20a', ROOT / 'scripts/validate-slice-20a.py')
27 common = importlib.util.module_from_spec(spec)
28 spec.loader.exec_module(common)
29 read, save, sha, command, snapshot = common.read, common.save, common.sha, common.command, common.snapshot
30 parser = argparse.ArgumentParser(description=__doc__)
31 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
32 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20g')
33 parser.add_argument('--preimplementation', type=Path, default=ROOT / 'reports/slice-20g/preimplementation.json')
34 args = parser.parse_args()
35 pre = read(args.preimplementation)
36 assert pre['baseCommit'] == BASE and len(pre['cases']) == 12 and len(pre['originals']) == 4
37 runtime = shutil.which(args.sbcl)
38 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
39 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
40 mutable = {'src/ctm/lexer.lisp', 'tests/ctm-identity.lisp', 'tests/ctm-templates.lisp',
41 'tests/ctm-prolog.lisp', 'tests/ctm-integer-literal.lisp',
42 'tests/ctm-registered-originals.lisp', 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
43 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
44 protected_paths = [p for p in tracked if p not in mutable]
45 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
46 output = args.output.resolve()
47 output.mkdir(parents=True, exist_ok=False)
48 retained = output / 'retained'
49 retained.mkdir()
50 save(output / 'preimplementation.json', pre)
51 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
52 protected = snapshot(ROOT / p for p in protected_paths)
53 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
54 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
55 for p in (ROOT / directory).rglob('*') if p.suffix in ['.lisp', '.py', '.sh', '.md'])
56 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
57 runs = []
58 with tempfile.TemporaryDirectory(prefix='topic-maps-20g-') as temp_name:
59 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(Path(temp_name) / 'sources')]
60 provenance = json.loads(command(preparation)) # Existing pinned local sources, nix --offline.
61 save(output / 'source-preparation.json', provenance)
62 env = dict(os.environ, SBCL=runtime, GIT_OPTIONAL_LOCKS='0', PYTHONDONTWRITEBYTECODE='1',
63 CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
64 TOPIC_MAPS_REPORT=str(retained / 'core.json'),
65 TOPIC_MAPS_XTM_REPORT=str(retained / 'xtm.json'),
66 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
67 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
68 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
69 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'),
70 TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
71 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'),
72 TOPIC_MAPS_TRIPLE_REPORT=str(output / 'current'))
73 for label, invocation in [
74 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
75 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
76 ('regression', [str(ROOT / 'scripts/test.sh')]),
77 ]:
78 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
79 run = {'label': label, 'command': invocation, 'freshProcess': True, 'network': 'denied by sandbox-exec',
80 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
81 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
82 stderr=subprocess.STDOUT, timeout=300)
83 (output / (label + '.log')).write_bytes(result.stdout)
84 run.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
85 runs.append(run)
86 assert result.returncode == 0, f'{label} failed; see {label}.log'
87 if label in ['xml', 'xtm']:
88 marker = 'CXML' if label == 'xml' else 'XTM'
89 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
90 assert len(labels) == len(set(labels)) == (146 if label == 'xml' else 327)
91 assert f'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result.stdout.decode()
92 run['passedChecks'] = len(labels)
93 # Permit only named obsolete Unsupported assertions in regenerated reports.
94 # Archived report bytes are independently hashed and remain unchanged.
95 labels = {
96 'diagnostic:unchanged-excluded-feature:def f() a - """N""". end': 'diagnostic:triple-string-body-supported',
97 'known-triple-string-unsupported:specific-condition-code-position-source': 'known-string-unclosed:specific-condition-code-position-source',
98 'known-triple-string-unsupported:position-units': 'known-string-unclosed:position-units',
99 'negative:triple-still-unsupported:class-code-exact-position': 'negative:triple-unclosed:class-code-exact-position',
100 }
101 old_prolog = read(ROOT / 'reports/slice-20f/retained/slice20c.prolog.json')
102 old_prolog_row = next(r for r in old_prolog['negativeControls'] if r['label'] == 'known-triple-string-unsupported')
103 old_integer = read(ROOT / 'reports/slice-20f/current.literal.json')
104 old_integer_row = next(r for r in old_integer['negativeControls'] if r['label'] == 'triple-still-unsupported')
105 def updated_condition(prior, source):
106 return dict(prior, condition='CTM-REJECTION', code='unterminated-string', source=source,
107 message=f'UNTERMINATED-STRING in {source} at CHARACTER {prior["position"]} (line {prior["line"]}, column {prior["column"]}): Unterminated triple-quoted string.')
108 def qualified_retained(value):
109 if isinstance(value, dict):
110 if value.get('label') == 'known-triple-string-unsupported':
111 assert value == old_prolog_row
112 return dict(value, label='known-string-unclosed',
113 condition=updated_condition(value['condition'], 'known-string-unclosed'))
114 if value.get('label') == 'triple-still-unsupported':
115 assert value == old_integer_row
116 return dict(value, label='triple-unclosed', sourceText='topic t: """x.',
117 condition=updated_condition(value['condition'], 'triple-unclosed'))
118 return {k: qualified_retained(v) for k, v in value.items()}
119 if isinstance(value, list):
120 return [qualified_retained(v) for v in value]
121 if isinstance(value, str):
122 return labels.get(value, value)
123 return value
124 verified = []
125 for old in sorted((ROOT / 'reports/slice-20f/retained').glob('*.json')):
126 assert qualified_retained(read(old)) == read(retained / old.name), str(old)
127 verified.append(str(old.relative_to(ROOT)))
128 for suffix in ['validation', 'cxtm', 'literal']:
129 old = ROOT / f'reports/slice-20f/current.{suffix}.json'
130 assert qualified_retained(read(old)) == read(retained / f'slice20f.{suffix}.json'), str(old)
131 verified.append(str(old.relative_to(ROOT)))
132 before = read(retained / 'slice20f.validation.json')
133 final = read(output / 'current.validation.json')
134 triple = read(output / 'current.triple.json')
135 cxtm = read(output / 'current.cxtm.json')
136 assert before['statuses'] == triple['entryStatuses'] == {'pass': 257, 'fail': 4, 'unsupported': 0, 'not-run': 12}
137 assert final['statuses'] == {'pass': 261, 'fail': 4, 'unsupported': 0, 'not-run': 8}
138 assert triple['passed'] == len(triple['checks']) == len(set(triple['checks'])) and triple['failed'] == 0
139 rows = {r['id']: r for r in triple['originals']}
140 ids = ['ctm/in/' + n + '.ctm' for n in CONTRACTS]
141 assert list(rows) == ids
142 changes, comparisons = [], []
143 positives = {p['label']: p for p in triple['positives']}
144 assert len(positives) == len(triple['positives']) == 77
145 assert all(p['initialCounter'] == p['finalCounter'] == 37 for p in positives.values())
146 negatives = {r['label']: r for r in triple['negativeControls']}
147 assert len(negatives) == len(triple['negativeControls']) == 31
148 for prior in pre['cases']:
149 current = negatives[prior['label']]
150 assert prior['sourceText'] == current['sourceText']
151 assert prior['result'] == {k: current['condition'][k] for k in prior['result']}, prior['label']
152 assert len(before['cases']) == len(final['cases']) == 273
153 for a, b in zip(before['cases'], final['cases']):
154 assert a['id'] == b['id']
155 if a['id'] not in rows:
156 assert a == b, a['id']
157 continue
158 row = rows[a['id']]
159 name = Path(a['id']).stem
160 source_size, expected_size, expected_value = CONTRACTS[name]
161 source = ROOT / 'vendor/cxtm' / a['id']
162 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (name + '.ctm.cxtm')
163 assert a['status'] == 'not-run' and a['actualOutcome'] is None and a['category'] == 'valid'
164 assert a['documentIRI'] == row['documentIRI'] == 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + a['id']
165 assert b == dict(a, status='pass', actualOutcome='cxtm-match')
166 assert row['status'] == 'pass' and row['actualOutcome'] == 'cxtm-match' and row['diagnostics'] == []
167 assert source.stat().st_size == row['sourceBytes'] == source_size
168 assert row['comparison']['actualBytes'] == row['comparison']['expectedBytes'] == expected_size
169 assert row['comparison']['firstDifferentByte'] is None
170 model = row['comparison']['model']
171 assert model['topicCount'] == 2 and model['associationCount'] == 0
172 constructs = [item for t in model['topics'] for item in t['occurrences' if name.startswith('occurrence-') else 'names']]
173 assert len(constructs) == 1 and constructs[0]['value'] == expected_value
174 if name.startswith('occurrence-'):
175 assert constructs[0]['datatype'] == 'http://www.w3.org/2001/XMLSchema#string'
176 actual_case = next(c for c in cxtm['cases'] if c['id'] == a['id'])
177 actual = actual_case['actualCXTM'].encode('utf-8')
178 assert actual == actual_case['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
179 control = positives[a['id'] + ':ordinary-control']['actualCXTM'].encode('utf-8')
180 assert control == actual
181 prior = next(r for r in pre['originals'] if r['original'] == a['id'])
182 assert prior['documentIRI'] == row['documentIRI']
183 assert prior['result']['condition'] == 'CTM-UNSUPPORTED' and prior['result']['code'] == 'triple-quoted-string'
184 comparisons.append({'id': a['id'], 'documentIRI': row['documentIRI'], 'sourceBytes': source_size,
185 'sourceSHA256': sha(source), 'expectedBytes': expected_size, 'expectedSHA256': sha(baseline),
186 'actualBytes': len(actual), 'actualSHA256': common.digest(actual), 'firstDifferentByte': None,
187 'ordinaryControlBytes': len(control), 'ordinaryControlSHA256': common.digest(control),
188 'value': expected_value, 'datatype': 'http://www.w3.org/2001/XMLSchema#string',
189 'canonicalizationPreservesMap': True, 'executedInFullRegression': True})
190 changes.append({'id': a['id'], 'before': a['status'], 'after': b['status']})
191 assert len(changes) == 4
192 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
193 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
194 old_cxtm = read(retained / 'slice20f.cxtm.json')
195 assert [c for c in cxtm['cases'] if c['id'] not in rows] == old_cxtm['cases']
196 assert cxtm['contractCounts'] == {'pass': 223, 'fail': 4}
197 for case in cxtm['cases']:
198 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (Path(case['id']).name + '.cxtm')
199 actual = case['actualCXTM'].encode('utf-8')
200 assert (actual == baseline.read_bytes()) == (case['status'] == 'pass')
201 assert case['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
202 log = (output / 'regression.log').read_text()
203 prior_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
204 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in prior_totals}
205 assert totals == prior_totals and sum(totals.values()) == 12100
206 for marker, count in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25),
207 ('PROLOG', 171), ('EMPTY2', 13), ('CTM-INTEGER', 112), ('TRIPLE', triple['passed'])]:
208 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
209 assert re.findall(r'^TRIPLE-ORIGINAL (ctm/in/[^:]+): PASS;', log, re.M) == ids
210 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
211 xtm = read(retained / 'xtm.json')
212 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
213 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
214 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
215 lock = read(ROOT / 'references/corpus-lock.json')
216 for entry in lock['files']:
217 path = ROOT / 'vendor/cxtm' / entry['path']
218 assert path.stat().st_size == entry['size'] and sha(path) == entry['sha256'], str(path)
219 assert protected == {p: sha(ROOT / p) for p in protected}
220 assert historical == {p: sha(ROOT / p) for p in historical}
221 assert sources == {p: sha(ROOT / p) for p in sources}
222 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
223 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
224 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
225 evidence = {
226 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
227 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
228 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
229 'sourcePreparationCommand': preparation, 'qualifiedSourceProvenance': provenance, 'runs': runs,
230 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
231 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
232 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': 13, 'ctmIntegerChecks': 112,
233 'tripleChecks': triple['passed'], 'positiveControls': len(positives), 'negativeControls': len(negatives),
234 'ordinaryStringDiagnosticSignaturesUnchanged': len(pre['cases']), 'originalComparisons': comparisons,
235 'retainedReportLabelChanges': labels, 'updatedAssertions': [
236 'Identity Triple String boundary now checks its actual Name value.',
237 'Existing Template body String path checks Name N after invocation.',
238 'Prolog and CTM-integer old valid-triple Unsupported probes now reject unclosed Triple Strings.'],
239 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
240 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': changes,
241 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
242 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
243 'retainedReportsVerified': verified, 'allHistoricalReportBytesUnchanged': True,
244 'allProtectedFileBytesUnchanged': True, 'pinnedCorpusFilesVerified': len(lock['files']),
245 'persistentLispImagesTouched': False,
246 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
247 'executionAuthorization': 'Regular require_escalated approval; no symlink or sandbox configuration changes.',
248 'acceptancePassed': True,
249 }
250 save(output / 'execution.json', evidence)
251 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
252 'standaloneXTMChecks', 'xtmMergemapChecks', 'tripleChecks', 'positiveControls', 'negativeControls',
253 'ctmStatusesAfter', 'xtmCorpusStatuses', 'originalComparisons', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent=2))
254
255
256 if __name__ == '__main__':
257 main()