]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20h.py
Document qualified disposition of remaining XTM corpus cases
[topic-maps.git] / scripts / validate-slice-20h.py
1 #!/usr/bin/env python3
2 """Validate ISO-8859-5 and the complete retained offline suite in fresh SBCL."""
3 import argparse
4 import datetime
5 import importlib.util
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '7f0e40d1558255773357c5ed4fa44bb5184d85e0'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 ORIGINAL = 'ctm/in/encoding-8859-5.ctm'
18 ISO_NAMES = ['ISO-8859-5', 'ISO_8859-5:1988', 'ISO_8859-5', 'iso-ir-144',
19 'cyrillic', 'csISOLatinCyrillic']
20
21
22 def main():
23 spec = importlib.util.spec_from_file_location('slice20a', ROOT / 'scripts/validate-slice-20a.py')
24 common = importlib.util.module_from_spec(spec)
25 spec.loader.exec_module(common)
26 read, save, sha, command, snapshot = common.read, common.save, common.sha, common.command, common.snapshot
27 parser = argparse.ArgumentParser(description=__doc__)
28 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
29 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20h')
30 parser.add_argument('--preimplementation', type=Path, default=ROOT / 'reports/slice-20h/preimplementation.json')
31 parser.add_argument('--runtime-qualification', type=Path, default=ROOT / 'reports/slice-20h/runtime-qualification.json')
32 args = parser.parse_args()
33 pre, qualification = read(args.preimplementation), read(args.runtime_qualification)
34 oracle = [ord(c) for c in bytes(range(256)).decode('iso8859_5')]
35 assert pre['baseCommit'] == qualification['baseCommit'] == BASE
36 assert pre['runtime'] == qualification['runtime'] == 'SBCL 2.5.4'
37 assert pre['network'] == qualification['network'] == 'denied'
38 assert pre['original']['id'] == ORIGINAL and pre['original']['code'] == 'unsupported-encoding'
39 assert pre['original']['condition'] == 'CTM-UNSUPPORTED' and pre['original']['position'] == 11
40 assert pre['original']['positionUnit'] == 'byte'
41 assert qualification['all256DecodedWithoutReplacement'] and qualification['all256EncodeRoundTrip']
42 assert qualification['pythonCodecOracleEqual'] and qualification['codepoints'] == oracle
43 runtime = shutil.which(args.sbcl)
44 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
45 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
46 mutable = {'src/ctm/unicode.lisp', 'src/ctm/reader.lisp', 'tests/ctm-prolog.lisp',
47 'tests/ctm-integer-literal.lisp', 'tests/ctm-registered-originals.lisp',
48 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
49 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
50 protected_paths = [p for p in tracked if p not in mutable]
51 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
52 old_unicode = subprocess.check_output(['git', 'show', BASE + ':src/ctm/unicode.lisp'], cwd=ROOT).decode()
53 new_unicode = (ROOT / 'src/ctm/unicode.lisp').read_text()
54 # Includes both strict UTF-8 and IRI unescaping, byte for byte.
55 assert old_unicode[:old_unicode.index(';; Slice 20C:')] == new_unicode[:new_unicode.index(';; Inspect')]
56 output = args.output.resolve()
57 output.mkdir(parents=True, exist_ok=False)
58 retained = output / 'retained'
59 retained.mkdir()
60 save(output / 'preimplementation.json', pre)
61 save(output / 'runtime-qualification.json', qualification)
62 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
63 protected = snapshot(ROOT / p for p in protected_paths)
64 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
65 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
66 for p in (ROOT / directory).rglob('*') if p.suffix in ['.lisp', '.py', '.sh', '.md'])
67 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
68 runs = []
69 with tempfile.TemporaryDirectory(prefix='topic-maps-20h-') as temp_name:
70 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(Path(temp_name) / 'sources')]
71 provenance = json.loads(command(preparation)) # Pinned local Store sources, nix --offline.
72 save(output / 'source-preparation.json', provenance)
73 env = dict(os.environ, SBCL=runtime, GIT_OPTIONAL_LOCKS='0', PYTHONDONTWRITEBYTECODE='1',
74 CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
75 TOPIC_MAPS_REPORT=str(retained / 'core.json'),
76 TOPIC_MAPS_XTM_REPORT=str(retained / 'xtm.json'),
77 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
78 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
79 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
80 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'),
81 TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
82 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'),
83 TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
84 TOPIC_MAPS_ENCODING_REPORT=str(output / 'current'))
85 for label, invocation in [
86 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
87 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
88 ('regression', [str(ROOT / 'scripts/test.sh')]),
89 ]:
90 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
91 run = {'label': label, 'command': invocation, 'freshProcess': True, 'network': 'denied by sandbox-exec',
92 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
93 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
94 stderr=subprocess.STDOUT, timeout=300)
95 (output / (label + '.log')).write_bytes(result.stdout)
96 run.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
97 runs.append(run)
98 assert result.returncode == 0, f'{label} failed; see {label}.log'
99 if label in ['xml', 'xtm']:
100 marker = 'CXML' if label == 'xml' else 'XTM'
101 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
102 assert len(labels) == len(set(labels)) == (146 if label == 'xml' else 327)
103 assert f'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result.stdout.decode()
104 run['passedChecks'] = len(labels)
105
106 # Only the seven obsolete Unsupported fixtures and their check label may
107 # change. Their complete prior rows are matched before this transformation.
108 old_prolog = read(ROOT / 'reports/slice-20g/retained/slice20c.prolog.json')
109 old_integer = read(ROOT / 'reports/slice-20g/retained/slice20f.literal.json')
110 obsolete = {r['label']: r for r in old_prolog['negativeControls'] + old_integer['negativeControls']
111 if r['condition']['code'] == 'unsupported-encoding'}
112 assert len(obsolete) == 7
113 def qualified_retained(value):
114 if isinstance(value, dict):
115 if value.get('label') in obsolete:
116 assert value == obsolete[value['label']]
117 current = dict(value)
118 condition = dict(value['condition'])
119 condition['message'] = condition['message'].replace(
120 'Only UTF-8 CTM input is implemented.',
121 'Only UTF-8 and ISO-8859-5 CTM input are implemented.').replace(
122 'Charset "iso-8859-5" is recognized but only UTF-8 decoding is implemented.',
123 'Charset "iso-8859-1" is not implemented; supported CTM encodings are UTF-8 and ISO-8859-5.')
124 if value['label'] == 'iso88595-still-unsupported':
125 current.update(label='iso88591-still-unsupported',
126 sourceText=value['sourceText'].replace('iso-8859-5', 'iso-8859-1'))
127 condition['source'] = current['label']
128 condition['message'] = condition['message'].replace(value['label'], current['label'])
129 current['condition'] = condition
130 return current
131 return {k: qualified_retained(v) for k, v in value.items()}
132 if isinstance(value, list):
133 return [qualified_retained(v) for v in value]
134 if value == 'negative:iso88595-still-unsupported:class-code-exact-position':
135 return 'negative:iso88591-still-unsupported:class-code-exact-position'
136 return value
137 verified = []
138 for old in sorted((ROOT / 'reports/slice-20g/retained').glob('*.json')):
139 assert qualified_retained(read(old)) == read(retained / old.name), str(old)
140 verified.append(str(old.relative_to(ROOT)))
141 for suffix in ['validation', 'cxtm', 'triple']:
142 old = ROOT / f'reports/slice-20g/current.{suffix}.json'
143 assert qualified_retained(read(old)) == read(retained / f'slice20g.{suffix}.json'), str(old)
144 verified.append(str(old.relative_to(ROOT)))
145 before = read(retained / 'slice20g.validation.json')
146 final, encoding, cxtm = [read(output / f'current.{name}.json') for name in ['validation', 'encoding', 'cxtm']]
147 assert before['statuses'] == encoding['entryStatuses'] == {'pass': 261, 'fail': 4, 'unsupported': 0, 'not-run': 8}
148 assert final['statuses'] == {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
149 assert encoding['passed'] == len(encoding['checks']) == len(set(encoding['checks'])) and encoding['failed'] == 0
150 assert encoding['decoderCodepoints'] == oracle == qualification['codepoints']
151 assert encoding['ISO88595Names'] == ISO_NAMES and encoding['UTF8Names'] == ['UTF-8', 'csUTF8']
152 assert encoding['selectionEvidence'] == {'conflictAndUnsupportedUTF8Calls': 0, 'conflictAndUnsupportedISOCalls': 0}
153 positives = {p['label']: p for p in encoding['positives']}
154 negatives = {r['label']: r for r in encoding['negativeControls']}
155 assert len(positives) == len(encoding['positives']) == 74
156 assert len(negatives) == len(encoding['negativeControls']) == 33
157 for charset in ['ISO-8859-5', 'csUTF8']:
158 row = negatives['source-position:' + charset]
159 condition = row['condition']
160 assert condition['code'] == 'unexpected-character' and condition['positionUnit'] == 'character'
161 assert condition['line'] == 3 and condition['column'] == 1
162 assert (row['positionInOriginalBytes'] == condition['position']) == (charset == 'ISO-8859-5')
163 for purpose in ['INCLUDE', 'MERGEMAP']:
164 for generic in ['NIL', 'T']:
165 row = positives[f'resource:{purpose}:generic={generic}']
166 assert row['finalCounter'] == (20 if purpose == 'INCLUDE' else 18)
167 assert 'C0 DE E1 E1 D8 EF' in row['inputHex']
168 row = encoding['original']
169 assert row['id'] == ORIGINAL and row['status'] == 'pass' and row['actualOutcome'] == 'cxtm-match'
170 assert row['diagnostics'] == [] and row['sourceBytes'] == 42
171 source = ROOT / 'vendor/cxtm' / ORIGINAL
172 baseline = ROOT / 'vendor/cxtm/ctm/baseline/encoding-8859-5.ctm.cxtm'
173 assert source.read_bytes() == b'%encoding "iso-8859-5"\nrussia - "\xc0\xde\xe1\xe1\xd8\xef" .'
174 expected = baseline.read_bytes() # Read the actual pinned byte count; never prescribe it.
175 comparison = row['comparison']
176 assert comparison['firstDifferentByte'] is None
177 assert comparison['actualBytes'] == comparison['expectedBytes'] == len(expected)
178 model = comparison['model']
179 assert model['topicCount'] == 2 and model['associationCount'] == 0
180 names = [n for topic in model['topics'] for n in topic['names']]
181 assert len(names) == 1 and names[0]['value'] == 'Россия'
182 assert len(before['cases']) == len(final['cases']) == 273
183 changes = []
184 for a, b in zip(before['cases'], final['cases']):
185 assert a['id'] == b['id']
186 if a['id'] != ORIGINAL:
187 assert a == b, a['id']
188 continue
189 assert a['status'] == 'not-run' and a['actualOutcome'] is None and a['category'] == 'valid'
190 assert a['documentIRI'] == row['documentIRI'] == pre['original']['documentIRI']
191 assert a['documentIRI'] == 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + ORIGINAL
192 assert b == dict(a, status='pass', actualOutcome='cxtm-match')
193 changes.append({'id': ORIGINAL, 'before': a['status'], 'after': b['status']})
194 assert len(changes) == 1
195 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
196 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
197 old_cxtm = read(retained / 'slice20g.cxtm.json')
198 assert [c for c in cxtm['cases'] if c['id'] != ORIGINAL] == old_cxtm['cases']
199 assert cxtm['contractCounts'] == {'pass': 224, 'fail': 4}
200 for case in cxtm['cases']:
201 pinned = ROOT / 'vendor/cxtm/ctm/baseline' / (Path(case['id']).name + '.cxtm')
202 actual = case['actualCXTM'].encode('utf-8')
203 assert (actual == pinned.read_bytes()) == (case['status'] == 'pass')
204 assert case['expectedCXTM'].encode('utf-8') == pinned.read_bytes()
205 original_case = next(c for c in cxtm['cases'] if c['id'] == ORIGINAL)
206 actual = original_case['actualCXTM'].encode('utf-8')
207 assert actual == original_case['expectedCXTM'].encode('utf-8') == expected
208 (output / 'actual').mkdir()
209 (output / 'actual/encoding-8859-5.ctm.cxtm').write_bytes(actual)
210 log = (output / 'regression.log').read_text()
211 prior_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
212 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in prior_totals}
213 assert totals == prior_totals and sum(totals.values()) == 12100
214 for marker, count in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25),
215 ('PROLOG', 171), ('EMPTY2', 13), ('CTM-INTEGER', 112), ('TRIPLE', 226),
216 ('ENCODING', encoding['passed'])]:
217 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
218 assert re.findall(r'^ENCODING-ORIGINAL (ctm/in/[^:]+): PASS;', log, re.M) == [ORIGINAL]
219 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
220 xtm = read(retained / 'xtm.json')
221 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
222 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
223 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
224 lock = read(ROOT / 'references/corpus-lock.json')
225 for entry in lock['files']:
226 path = ROOT / 'vendor/cxtm' / entry['path']
227 assert path.stat().st_size == entry['size'] and sha(path) == entry['sha256'], str(path)
228 assert protected == {p: sha(ROOT / p) for p in protected}
229 assert historical == {p: sha(ROOT / p) for p in historical}
230 assert sources == {p: sha(ROOT / p) for p in sources}
231 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
232 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
233 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
234 evidence = {
235 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
236 'initialRepositoryState': {'head': BASE, 'branch': 'codex/ctm-slice-20g', 'indexClean': True, 'worktreeClean': True},
237 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
238 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
239 'sourcePreparationCommand': preparation, 'qualifiedSourceProvenance': provenance, 'runs': runs,
240 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
241 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
242 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': 13,
243 'ctmIntegerChecks': 112, 'tripleChecks': 226, 'encodingChecks': encoding['passed'],
244 'positiveControls': len(positives), 'negativeControls': len(negatives),
245 'all256ByteMappingsMatchIndependentOracle': True, 'runtimeQualificationBeforeImplementation': qualification,
246 'supportedISO88595Names': ISO_NAMES, 'supportedUTF8Names': ['UTF-8', 'csUTF8'],
247 'strictUTF8AndIRIUnescapingByteIdenticalToBase': True,
248 'originalComparison': {'id': ORIGINAL, 'documentIRI': row['documentIRI'], 'sourceBytes': source.stat().st_size,
249 'sourceSHA256': sha(source), 'expectedBytes': len(expected), 'expectedSHA256': sha(baseline),
250 'actualBytes': len(actual), 'actualSHA256': common.digest(actual), 'firstDifferentByte': None,
251 'originalNameBytesHex': 'C0 DE E1 E1 D8 EF', 'value': 'Россия',
252 'canonicalizationPreservesMap': True, 'executedInFullRegression': True},
253 'updatedUnsupportedAssertions': sorted(obsolete),
254 'updatedAssertionReason': 'ISO-8859-5 is now supported; ISO-8859-1 remains unsupported. Only messages and one named fixture/check label change in regenerated reports.',
255 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
256 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': changes,
257 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
258 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
259 'retainedReportsVerified': verified, 'allHistoricalReportBytesUnchanged': True,
260 'historicalReportFilesVerified': len(historical), 'allProtectedFileBytesUnchanged': True,
261 'protectedFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
262 'persistentLispImagesTouched': False,
263 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
264 'sandboxStartupError': 'writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.',
265 'executionAuthorization': 'Regular require_escalated approval, canonical SBCL/Nix Store paths; no symlink or sandbox configuration changes. All regression subprocesses deny network.',
266 'acceptancePassed': True,
267 }
268 save(output / 'execution.json', evidence)
269 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
270 'standaloneXTMChecks', 'xtmMergemapChecks', 'encodingChecks', 'positiveControls', 'negativeControls',
271 'ctmStatusesAfter', 'xtmCorpusStatuses', 'originalComparison', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent=2))
272
273
274 if __name__ == '__main__':
275 main()