]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20c.py
Add XTM Variant reification and qualify Slice 21O originals
[topic-maps.git] / scripts / validate-slice-20c.py
1 #!/usr/bin/env python3
2 """Validate the minimal CTM prolog implementation and every retained suite offline."""
3 import argparse
4 import datetime
5 import importlib.util
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = 'd024dc96ce091db06b9d09ea7c1ccb5f90b2886d'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 EXPECTED = {
18 'BOM-contradicts-encoding': ('encoding-conflict', 14, 'byte', None, None),
19 'unknown-directive': ('unknown-directive', 0, 'character', 1, 1),
20 'wrong-version': ('invalid-version', 9, 'character', 1, 10),
21 }
22 # Five prior 19D2 labels plus the one obsolete version-unimplemented assertion.
23 RENAMED = {
24 'diagnostic:known-XTM-unimplemented:specific-condition': 'diagnostic:XTM-explicit-resolver-required:specific-condition',
25 'diagnostic:known-XTM-unimplemented:source-position': 'diagnostic:XTM-explicit-resolver-required:source-position',
26 'policy:XTM-recognized-without-retrieval:specific-condition': 'policy:XTM-explicit-resolver-before-retrieval:specific-condition',
27 'policy:XTM-recognized-without-retrieval:source-position': 'policy:XTM-explicit-resolver-before-retrieval:source-position',
28 'policy:XTM-does-not-call-resolver': 'policy:XTM-without-resolver-does-not-fetch',
29 'remaining-construct-unsupported:%version 1.0': 'prefix:version-1.0-prolog-supported',
30 }
31
32
33 def main():
34 spec = importlib.util.spec_from_file_location('slice20a', ROOT / 'scripts/validate-slice-20a.py')
35 common = importlib.util.module_from_spec(spec)
36 spec.loader.exec_module(common)
37 read, save, sha, command, snapshot = common.read, common.save, common.sha, common.command, common.snapshot
38 parser = argparse.ArgumentParser(description=__doc__)
39 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
40 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20c')
41 parser.add_argument('--preimplementation-log', type=Path)
42 args = parser.parse_args()
43 runtime = shutil.which(args.sbcl)
44 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
45 output = args.output.resolve()
46 output.mkdir(parents=True, exist_ok=False)
47 retained_dir = output / 'retained'
48 retained_dir.mkdir()
49 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
50 protected_paths = ['src/tmdm', 'src/cxtm', 'src/xml', 'src/xtm', 'patches', 'vendor', 'references',
51 'flake.nix', 'flake.lock', 'scripts/test.sh', 'scripts/prepare-cxml.py',
52 'scripts/validate-slice-19d2.py', 'scripts/validate-slice-20a.py', 'scripts/validate-slice-20b.py']
53 tracked_tests = command(['git', 'ls-tree', '-r', '--name-only', BASE, '--', 'tests']).splitlines()
54 protected_paths += [p for p in tracked_tests if p != 'tests/ctm-prefixes.lisp']
55 protected_paths += ['src/ctm/' + p.name for p in (ROOT / 'src/ctm').glob('*.lisp')
56 if p.name not in ['unicode.lisp', 'lexer.lisp', 'reader.lisp', 'resources.lisp']]
57 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
58 protected = snapshot(p for directory in ['src/tmdm', 'src/cxtm', 'src/xml', 'src/xtm', 'vendor', 'references', 'patches']
59 for p in (ROOT / directory).rglob('*'))
60 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
61 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
62 for p in (ROOT / directory).rglob('*') if p.suffix in ['.lisp', '.py', '.sh', '.md'])
63 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
64 if args.preimplementation_log:
65 preimplementation = [json.loads(line) for line in args.preimplementation_log.read_text().splitlines() if line.startswith('{')]
66 assert [Path(r['id']).stem for r in preimplementation] == list(EXPECTED)
67 assert all(r['condition']['condition'] == 'CTM-UNSUPPORTED' and r['condition']['code'] == 'unsupported-directive'
68 for r in preimplementation)
69 save(output / 'preimplementation.json', {'baseCommit': BASE, 'runtime': 'SBCL 2.5.4',
70 'freshProcess': True, 'network': 'denied by sandbox-exec', 'cases': preimplementation})
71 runs = []
72 with tempfile.TemporaryDirectory(prefix='topic-maps-20c-') as tmp_name:
73 tmp = Path(tmp_name)
74 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(tmp / 'sources')]
75 provenance = json.loads(command(preparation))
76 save(output / 'source-preparation.json', provenance)
77 env = dict(os.environ, SBCL=runtime, PYTHONDONTWRITEBYTECODE='1', CL_SOURCE_REGISTRY=provenance['sourceRegistry'])
78 env.update(TOPIC_MAPS_REPORT=str(retained_dir / 'core.json'),
79 TOPIC_MAPS_XTM_REPORT=str(retained_dir / 'xtm.json'),
80 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained_dir / 'xtm-mergemap'),
81 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained_dir / 'slice20a'),
82 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained_dir / 'slice20b'),
83 TOPIC_MAPS_PROLOG_REPORT=str(output / 'current'))
84 invocations = [
85 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
86 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
87 ('regression', [str(ROOT / 'scripts/test.sh')]),
88 ]
89 for label, invocation in invocations:
90 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
91 context = {'label': label, 'command': invocation, 'freshProcess': True, 'network': 'denied by sandbox-exec',
92 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
93 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
94 (output / (label + '.log')).write_bytes(result.stdout)
95 context.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
96 runs.append(context)
97 assert result.returncode == 0, f'{label} failed; see {label}.log'
98 if label in ['xml', 'xtm']:
99 marker = 'CXML' if label == 'xml' else 'XTM'
100 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
101 assert len(labels) == len(set(labels)) == (146 if label == 'xml' else 327)
102 assert f'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result.stdout.decode()
103 context['passedChecks'] = len(labels)
104 # Keep the two retained documented inspection examples executable too.
105 for label, document in [('xtm-inspection', 'docs/slice-19d2-xtm-mergemap.md'),
106 ('ctm-inspection', 'docs/slice-20a-registration.md')]:
107 forms = (ROOT / document).read_text().split('```lisp\n')[-1].split('\n```', 1)[0]
108 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', '/dev/stdin']
109 result = subprocess.run(invocation, cwd=ROOT, env=env, input=forms.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
110 (output / (label + '.log')).write_bytes(result.stdout)
111 assert result.returncode == 0, label
112 def retained_equal(prior, current):
113 if prior == current:
114 return
115 assert 'checks' in prior and 'checks' in current
116 assert current['checks'] == [RENAMED.get(c, c) for c in prior['checks']]
117 assert {k: v for k, v in prior.items() if k != 'checks'} == {k: v for k, v in current.items() if k != 'checks'}
118 baseline = ROOT / 'reports/slice-19c'
119 verified_reports = []
120 for old in sorted(baseline.glob('*.json')):
121 if old.name == 'execution.json':
122 continue
123 current = retained_dir / ('core.json' if old.name == 'validation.json' else 'core.json.' + old.name)
124 retained_equal(read(old), read(current))
125 verified_reports.append(str(old.relative_to(ROOT)))
126 for suffix in ['validation', 'cxtm', 'integration']:
127 retained_equal(read(ROOT / f'reports/slice-19d2/{suffix}.json'), read(retained_dir / f'xtm-mergemap.{suffix}.json'))
128 for stage, directory, suffixes in [('slice20a', 'slice-20a', ['validation', 'cxtm', 'registration']),
129 ('slice20b', 'slice-20b', ['validation', 'cxtm', 'negative'])]:
130 for suffix in suffixes:
131 retained_equal(read(ROOT / f'reports/{directory}/current.{suffix}.json'), read(retained_dir / f'{stage}.{suffix}.json'))
132 before = read(retained_dir / 'slice20b.validation.json')
133 final = read(output / 'current.validation.json')
134 cxtm = read(output / 'current.cxtm.json')
135 prolog = read(output / 'current.prolog.json')
136 assert prolog['entryStatuses'] == before['statuses'] == {'pass': 252, 'fail': 4, 'unsupported': 0, 'not-run': 17}
137 assert final['statuses'] == {'pass': 255, 'fail': 4, 'unsupported': 0, 'not-run': 14}
138 assert prolog['failed'] == 0 and prolog['passed'] == len(prolog['checks']) == len(set(prolog['checks']))
139 rows = {r['id']: r for r in prolog['originals']}
140 assert list(rows) == ['ctm/invalid/' + name + '.ctm' for name in EXPECTED]
141 assert len(before['cases']) == len(final['cases']) == 273
142 changes, candidates = [], []
143 for a, b in zip(before['cases'], final['cases']):
144 assert a['id'] == b['id']
145 if a['id'] not in rows:
146 assert a == b, a['id']
147 continue
148 row = rows[a['id']]
149 assert row['qualified'] and row['status'] == 'pass' and row['control']['accepted'] and row['control']['condition'] is None
150 assert row['documentIRI'] == a['documentIRI']
151 assert a['documentIRI'] == 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + a['id']
152 assert row['condition'] == row['repeatCondition']
153 c = row['condition']
154 expected = EXPECTED[Path(a['id']).stem]
155 assert c['condition'] == 'CTM-REJECTION' and (c['code'], c['position'], c['positionUnit'], c['line'], c['column']) == expected
156 assert c['source'] == a['id'] and c['cause'] is c['context'] is None
157 data = (ROOT / 'vendor/cxtm' / a['sourcePathname']).read_bytes()
158 assert row['sourceHex'] == data.hex(' ').upper()
159 assert row['sourceTextAfterBOM'] == data.decode('utf-8-sig')
160 if c['positionUnit'] == 'byte':
161 assert data[:3] == bytes.fromhex('ef bb bf') and data[c['position']:].startswith(b'iso-8859-1')
162 else:
163 text = data.decode()
164 assert text[c['position']:].startswith('10.0' if Path(a['id']).stem == 'wrong-version' else '%ctm')
165 assert text[:c['position']].count('\n') + 1 == c['line']
166 assert len(text[:c['position']].rsplit('\n', 1)[-1]) + 1 == c['column']
167 assert a['status'] == 'not-run' and a['actualOutcome'] is None
168 assert b == dict(a, status='pass', actualOutcome='ctm-rejection', diagnostics=[c['message']])
169 changes.append({'id': a['id'], 'before': a['status'], 'after': b['status']})
170 candidates.append({'id': a['id'], 'status': b['status'], 'sourceSHA256': sha(ROOT / 'vendor/cxtm' / a['id']),
171 'documentIRI': a['documentIRI'], 'clauses': row['clauses'], 'condition': c})
172 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope']} == {
173 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope']}
174 assert cxtm['cases'] == read(retained_dir / 'slice20b.cxtm.json')['cases']
175 assert cxtm['contractCounts'] == {'pass': 217, 'fail': 4}
176 # Independently recheck every actual positive CXTM stream against its pin.
177 for case in cxtm['cases']:
178 name = Path(case['id']).name
179 baseline_bytes = (ROOT / 'vendor/cxtm/ctm/baseline' / (name + '.cxtm')).read_bytes()
180 actual = case['actualCXTM'].encode('utf-8')
181 assert (actual == baseline_bytes) == (case['status'] == 'pass')
182 assert case['expectedCXTM'].encode('utf-8') == baseline_bytes
183 positives = prolog['positives']
184 assert len([p for p in positives if p['label'].startswith('utf8:provided=')]) == 18
185 assert all(p['accepted'] for p in positives)
186 assert 'encoding-selection-precedes-decoder:zero-calls' in prolog['checks']
187 negative_labels = {r['label']: r['condition'] for r in prolog['negativeControls']}
188 for label, c in negative_labels.items():
189 assert c['condition'] in ['CTM-REJECTION', 'CTM-UNSUPPORTED']
190 if c['positionUnit'] == 'byte':
191 assert c['line'] is c['column'] is None
192 assert negative_labels['conflict-before-invalid-body-decoding']['code'] == 'encoding-conflict'
193 assert negative_labels['unsupported-before-invalid-body-decoding']['condition'] == 'CTM-UNSUPPORTED'
194 assert negative_labels['known-literal-unsupported']['code'] == 'occurrence-literal-datatype'
195 log = (output / 'regression.log').read_text()
196 previous_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
197 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in previous_totals}
198 assert totals == previous_totals and sum(totals.values()) == 12100
199 for marker, expected in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25), ('PROLOG', prolog['passed'])]:
200 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == expected
201 assert len(re.findall(r'^PROLOG-ORIGINAL ctm/invalid/[^:]+: PASS;', log, re.M)) == 3
202 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
203 xtm = read(retained_dir / 'xtm.json')
204 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
205 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
206 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
207 assert protected == {p: sha(ROOT / p) for p in protected}
208 assert historical == {p: sha(ROOT / p) for p in historical}
209 assert sources == {p: sha(ROOT / p) for p in sources}
210 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
211 save(output / 'candidates.json', candidates)
212 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
213 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
214 evidence = {
215 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
216 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
217 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
218 'sourcePreparationCommand': preparation, 'qualifiedSourceProvenance': provenance, 'runs': runs,
219 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
220 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
221 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': prolog['passed'],
222 'positivePrologAndResourceControls': len(positives), 'negativeControls': len(prolog['negativeControls']),
223 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
224 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': changes,
225 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
226 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
227 'retainedReportsVerified': verified_reports, 'retainedReportLabelChanges': RENAMED,
228 'retainedPositiveCXTMComparisons': len(cxtm['cases']),
229 'allHistoricalReportBytesUnchanged': True, 'allProtectedFileBytesUnchanged': True,
230 'encodingConflictBeforeDecoderCalls': 0, 'optionalEncodingAPI': 'deserialize-file :encoding',
231 'encodingScope': 'UTF-8 only; existing strict decoder unchanged; no aliases or new charset decoder; resolver protocol unchanged.',
232 'persistentLispImagesTouched': False, 'documentedRetainedInspectionsPassed': True,
233 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
234 'acceptancePassed': True,
235 }
236 save(output / 'execution.json', evidence)
237 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks', 'standaloneXTMChecks',
238 'xtmMergemapChecks', 'slice20aChecks', 'slice20bChecks', 'prologChecks', 'ctmStatusesAfter', 'xtmCorpusStatuses', 'acceptancePassed']}, indent=2))
239
240
241 if __name__ == '__main__':
242 main()