]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20e.py
Qualify closed XML/C14N 1.0 infrastructure for XTM resourceData
[topic-maps.git] / scripts / validate-slice-20e.py
1 #!/usr/bin/env python3
2 """Validate the empty2 original registration and all retained suites offline."""
3 import argparse
4 import datetime
5 import importlib.util
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '9866a0e94a2a43c6aa73025f1c8d06b0f093a34b'
16 ID = 'ctm/in/empty2.ctm'
17 POLICY = '(version 1)(allow default)(deny network*)'
18 SOURCE = b'%version 1.0\n'
19 EXPECTED = b'<topicMap>\n</topicMap>\n'
20
21
22 def main():
23 # Reuse existing report/hash utilities and historical label qualification.
24 def previous(name):
25 spec = importlib.util.spec_from_file_location(name, ROOT / ('scripts/validate-' + name + '.py'))
26 module = importlib.util.module_from_spec(spec)
27 spec.loader.exec_module(module)
28 return module
29 common, prolog_driver = previous('slice-20a'), previous('slice-20c')
30 read, save, sha, command, snapshot = common.read, common.save, common.sha, common.command, common.snapshot
31 parser = argparse.ArgumentParser(description=__doc__)
32 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
33 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20e')
34 args = parser.parse_args()
35 runtime = shutil.which(args.sbcl)
36 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
37 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
38 mutable = {'tests/ctm-registered-originals.lisp', 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
39 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
40 protected_paths = [p for p in tracked if p not in mutable]
41 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
42 output = args.output.resolve()
43 output.mkdir(parents=True, exist_ok=False)
44 retained = output / 'retained'
45 retained.mkdir()
46 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
47 protected = snapshot(ROOT / p for p in protected_paths)
48 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
49 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
50 for p in (ROOT / directory).rglob('*') if p.suffix in ['.lisp', '.py', '.sh', '.md'])
51 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
52 source = ROOT / 'vendor/cxtm' / ID
53 baseline = ROOT / 'vendor/cxtm/ctm/baseline/empty2.ctm.cxtm'
54 assert source.read_bytes() == SOURCE and baseline.read_bytes() == EXPECTED
55 runs = []
56 with tempfile.TemporaryDirectory(prefix='topic-maps-20e-') as temp_name:
57 temp = Path(temp_name)
58 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(temp / 'sources')]
59 provenance = json.loads(command(preparation)) # nix eval --offline; verifies local Store source hashes.
60 save(output / 'source-preparation.json', provenance)
61 env = dict(os.environ, SBCL=runtime, GIT_OPTIONAL_LOCKS='0', PYTHONDONTWRITEBYTECODE='1',
62 CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
63 TOPIC_MAPS_REPORT=str(retained / 'core.json'),
64 TOPIC_MAPS_XTM_REPORT=str(retained / 'xtm.json'),
65 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
66 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
67 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
68 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'),
69 TOPIC_MAPS_EMPTY2_REPORT=str(output / 'current'))
70 invocations = [
71 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
72 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
73 ('regression', [str(ROOT / 'scripts/test.sh')]),
74 ]
75 for label, invocation in invocations:
76 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
77 run = {'label': label, 'command': invocation, 'freshProcess': True,
78 'network': 'denied by sandbox-exec',
79 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
80 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
81 stderr=subprocess.STDOUT, timeout=300)
82 (output / (label + '.log')).write_bytes(result.stdout)
83 run.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
84 runs.append(run)
85 assert result.returncode == 0, f'{label} failed; see {label}.log'
86 if label in ['xml', 'xtm']:
87 marker = 'CXML' if label == 'xml' else 'XTM'
88 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
89 assert len(labels) == len(set(labels)) == (146 if label == 'xml' else 327)
90 assert f'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result.stdout.decode()
91 run['passedChecks'] = len(labels)
92 # Compare regenerated retained stages with their historical provenance.
93 def retained_equal(prior, current):
94 if prior == current:
95 return
96 assert current['checks'] == [prolog_driver.RENAMED.get(c, c) for c in prior['checks']]
97 assert {k: v for k, v in prior.items() if k != 'checks'} == {
98 k: v for k, v in current.items() if k != 'checks'}
99 verified = []
100 for old in sorted((ROOT / 'reports/slice-19c').glob('*.json')):
101 if old.name == 'execution.json':
102 continue
103 current = retained / ('core.json' if old.name == 'validation.json' else 'core.json.' + old.name)
104 retained_equal(read(old), read(current))
105 verified.append(str(old.relative_to(ROOT)))
106 for suffix in ['validation', 'cxtm', 'integration']:
107 old = ROOT / f'reports/slice-19d2/{suffix}.json'
108 retained_equal(read(old), read(retained / f'xtm-mergemap.{suffix}.json'))
109 verified.append(str(old.relative_to(ROOT)))
110 for stage, directory, suffixes in [
111 ('slice20a', 'slice-20a', ['validation', 'cxtm', 'registration']),
112 ('slice20b', 'slice-20b', ['validation', 'cxtm', 'negative']),
113 ('slice20c', 'slice-20c', ['validation', 'cxtm', 'prolog']),
114 ]:
115 for suffix in suffixes:
116 old = ROOT / f'reports/{directory}/current.{suffix}.json'
117 assert read(old) == read(retained / f'{stage}.{suffix}.json'), str(old)
118 verified.append(str(old.relative_to(ROOT)))
119 before = read(retained / 'slice20c.validation.json')
120 final = read(output / 'current.validation.json')
121 registration = read(output / 'current.registration.json')
122 cxtm = read(output / 'current.cxtm.json')
123 assert before['statuses'] == registration['entryStatuses'] == {
124 'pass': 255, 'fail': 4, 'unsupported': 0, 'not-run': 14}
125 assert final['statuses'] == {'pass': 256, 'fail': 4, 'unsupported': 0, 'not-run': 13}
126 assert registration['passed'] == len(registration['checks']) == len(set(registration['checks']))
127 assert registration['failed'] == 0 and len(registration['cases']) == 1
128 row = registration['cases'][0]
129 assert row['id'] == ID and row['sourceBytes'] == 13
130 assert row['status'] == 'pass' and row['actualOutcome'] == 'cxtm-match' and row['diagnostics'] == []
131 assert row['comparison']['actualBytes'] == row['comparison']['expectedBytes'] == 23
132 assert row['comparison']['firstDifferentByte'] is None
133 assert row['comparison']['model'] == {'topicCount': 0, 'associationCount': 0, 'topics': []}
134 changes = []
135 assert len(before['cases']) == len(final['cases']) == 273
136 for a, b in zip(before['cases'], final['cases']):
137 assert a['id'] == b['id']
138 if a['id'] != ID:
139 assert a == b, a['id']
140 continue
141 assert a['category'] == 'valid' and a['expectedOutcome'] == 'cxtm-equal'
142 assert a['status'] == 'not-run' and a['actualOutcome'] is None
143 assert a['documentIRI'] == row['documentIRI'] == (
144 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + ID)
145 assert b == dict(a, status='pass', actualOutcome='cxtm-match')
146 changes.append({'id': ID, 'before': a['status'], 'after': b['status']})
147 assert len(changes) == 1
148 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
149 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
150 old_cxtm = read(retained / 'slice20c.cxtm.json')
151 assert [c for c in cxtm['cases'] if c['id'] != ID] == old_cxtm['cases']
152 assert cxtm['contractCounts'] == {'pass': 218, 'fail': 4}
153 for case in cxtm['cases']:
154 pinned = ROOT / 'vendor/cxtm/ctm/baseline' / (Path(case['id']).name + '.cxtm')
155 actual = case['actualCXTM'].encode('utf-8')
156 assert (actual == pinned.read_bytes()) == (case['status'] == 'pass')
157 assert case['expectedCXTM'].encode('utf-8') == pinned.read_bytes()
158 comparison = next(c for c in cxtm['cases'] if c['id'] == ID)
159 assert comparison['actualCXTM'].encode() == comparison['expectedCXTM'].encode() == EXPECTED
160 log = (output / 'regression.log').read_text()
161 prior_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
162 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in prior_totals}
163 assert totals == prior_totals and sum(totals.values()) == 12100
164 for marker, count in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286),
165 ('REGISTERED-NEGATIVES', 25), ('PROLOG', 171), ('EMPTY2', registration['passed'])]:
166 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
167 assert log.count('EMPTY2-ORIGINAL ctm/in/empty2.ctm: PASS; 23 byte-exact CXTM bytes') == 1
168 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
169 xtm = read(retained / 'xtm.json')
170 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
171 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
172 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
173 lock = read(ROOT / 'references/corpus-lock.json')
174 for entry in lock['files']:
175 path = ROOT / 'vendor/cxtm' / entry['path']
176 assert path.stat().st_size == entry['size'] and sha(path) == entry['sha256'], str(path)
177 assert protected == {p: sha(ROOT / p) for p in protected}
178 assert historical == {p: sha(ROOT / p) for p in historical}
179 assert sources == {p: sha(ROOT / p) for p in sources}
180 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
181 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
182 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
183 evidence = {
184 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
185 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
186 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
187 'sourcePreparationCommand': preparation, 'qualifiedSourceProvenance': provenance, 'runs': runs,
188 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
189 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
190 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': registration['passed'],
191 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
192 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': changes,
193 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
194 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
195 'retainedReportsVerified': verified, 'allHistoricalReportBytesUnchanged': True,
196 'allProtectedFileBytesUnchanged': True, 'pinnedCorpusFilesVerified': len(lock['files']),
197 'originalComparison': {
198 'id': ID, 'documentIRI': row['documentIRI'], 'sourceBytes': len(SOURCE), 'sourceSHA256': sha(source),
199 'expectedBytes': len(EXPECTED), 'expectedSHA256': sha(baseline),
200 'actualBytes': len(EXPECTED), 'actualSHA256': common.digest(comparison['actualCXTM'].encode()),
201 'firstDifferentByte': None, 'topics': 0, 'associations': 0,
202 'canonicalizationPreservesMap': True, 'executedInFullRegression': True,
203 },
204 'persistentLispImagesTouched': False,
205 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
206 'sandboxStartError': 'writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.',
207 'executionAuthorization': 'Regular require_escalated auto-review approval; no symlink or sandbox configuration changes.',
208 'acceptancePassed': True,
209 }
210 save(output / 'execution.json', evidence)
211 print(json.dumps({k: evidence[k] for k in [
212 'runtime', 'retainedLispChecks', 'xmlQualificationChecks', 'standaloneXTMChecks', 'xtmMergemapChecks',
213 'slice20aChecks', 'slice20bChecks', 'prologChecks', 'empty2Checks', 'ctmStatusesAfter',
214 'xtmCorpusStatuses', 'originalComparison', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent=2))
215
216
217 if __name__ == '__main__':
218 main()