]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20a.py
feat(tmdm): readers for the TMDM 7.2/7.3 relationships; tests for both default name...
[topic-maps.git] / scripts / validate-slice-20a.py
1 #!/usr/bin/env python3
2 """Register forty original CTM contracts after the complete retained offline suite."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12
13 ROOT = Path(__file__).resolve().parent.parent
14 BASE = '9c196ddc16cf5cd727fcde81ad7835edefa0f787'
15 POLICY = '(version 1)(allow default)(deny network*)'
16 GROUPS = {
17 'comments': ['comment-ml', 'comment-ml-nested', 'comment-ml-nested2', 'comment-sl',
18 'comment-sl2', 'comment-sl3', 'comment-sl4', 'comment-sl5', 'comment-sl6', 'empty'],
19 'identity': ['association-qname-iid', 'item-identifier-INF', 'item-identifier-relative',
20 'qname-date', 'qname-digits', 'rootless-uri', 'subject-identifier-dot',
21 'subject-identifier-non-http', 'subject-identifier-relative', 'subject-locator-dot',
22 'subject-locator-relative', 'topic-identifier-INF', 'topic-identifier-unicode'],
23 'occurrences': ['occurrence-decimal-dot-neg2', 'occurrence-decimal-dot-pos',
24 'occurrence-decimal-leading-zeros2', 'occurrence-duplicate-string',
25 'occurrence-duplicate-uri', 'occurrence-qname-value', 'occurrence-string-multiline',
26 'occurrence-string-multiline2', 'occurrence-uri-semicolon'],
27 'prefixes': ['prefix-multiline2', 'prefix-multiline3', 'prefix-multiline4', 'prefix-with-qm'],
28 'escapes': ['string-escape', 'string-escape2', 'unicode-escape-u4', 'unicode-escape-u6'],
29 }
30 BEGUN = {
31 'topic-maps.asd': '9013f4ace885dc049e98f674e8f59378d108b391b2aeac73fb54ee45119f8842',
32 'tests/ctm-registered-originals.lisp': '13eb134dde1620114e838f7595d1b1a0cfbd06624b1a0d23f5971441648e42e4',
33 }
34
35
36 def read(path):
37 return json.loads(path.read_text())
38
39
40 def digest(data):
41 return hashlib.sha256(data).hexdigest()
42
43
44 def sha(path):
45 return digest(path.read_bytes())
46
47
48 def command(args):
49 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
50
51
52 def save(path, value):
53 path.write_text(json.dumps(value, indent=2) + '\n')
54
55
56 def snapshot(paths):
57 return {str(p.relative_to(ROOT)): sha(p) for p in sorted(paths) if p.is_file()}
58
59
60 def main():
61 parser = argparse.ArgumentParser(description=__doc__)
62 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
63 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20a')
64 args = parser.parse_args()
65 runtime = shutil.which(args.sbcl)
66 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
67 output = args.output.resolve()
68 output.mkdir(parents=True, exist_ok=False)
69 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
70 tracked_tests = command(['git', 'ls-tree', '-r', '--name-only', BASE, '--', 'tests']).splitlines()
71 protected_paths = ['src', 'vendor', 'references', 'patches', 'flake.nix', 'flake.lock',
72 'scripts/test.sh', 'scripts/prepare-cxml.py', 'scripts/validate-slice-19d2.py'] + tracked_tests
73 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
74 protected = snapshot(p for directory in ['src', 'vendor', 'references', 'patches']
75 for p in (ROOT / directory).rglob('*'))
76 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
77 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
78 for p in (ROOT / directory).rglob('*')
79 if p.suffix in ['.lisp', '.py', '.sh', '.md'])
80 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
81 prefix = output / 'current'
82 environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1',
83 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(prefix))
84 # Reuse the unchanged qualification/regression driver, including its byte
85 # checks against historical reports. Its integration snapshot precedes 20A.
86 invocation = ['python3', 'scripts/validate-slice-19d2.py', '--sbcl', runtime,
87 '--output', str(output / 'retained')]
88 result = subprocess.run(invocation, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
89 stderr=subprocess.STDOUT, timeout=600)
90 (output / 'retained-driver.log').write_bytes(result.stdout)
91 assert result.returncode == 0, 'Retained regression failed; see retained-driver.log and retained/*.log'
92 retained = read(output / 'retained/execution.json')
93 old = read(ROOT / 'reports/slice-19d2/validation.json')
94 before = read(output / 'retained/validation.json')
95 old_cxtm = read(ROOT / 'reports/slice-19d2/cxtm.json')
96 before_cxtm = read(output / 'retained/cxtm.json')
97 assert before == old
98 assert before_cxtm == old_cxtm
99 assert read(output / 'retained/integration.json') == read(ROOT / 'reports/slice-19d2/integration.json')
100 final = read(Path(str(prefix) + '.validation.json'))
101 cxtm = read(Path(str(prefix) + '.cxtm.json'))
102 registration = read(Path(str(prefix) + '.registration.json'))
103 ids = ['ctm/in/' + name + '.ctm' for names in GROUPS.values() for name in names]
104 assert len(ids) == len(set(ids)) == 40
105 assert registration['entryStatuses'] == before['statuses'] == {
106 'pass': 205, 'fail': 4, 'unsupported': 0, 'not-run': 64}
107 assert final['statuses'] == {'pass': 245, 'fail': 4, 'unsupported': 0, 'not-run': 24}
108 assert len(before['cases']) == len(final['cases']) == 273
109 assert [c['id'] for c in before['cases']] == [c['id'] for c in final['cases']]
110 changes = []
111 for a, b in zip(before['cases'], final['cases']):
112 if a['id'] in ids:
113 assert a['status'] == 'not-run' and a['actualOutcome'] is None
114 expected = dict(a, status='pass', actualOutcome='cxtm-match', diagnostics=[])
115 assert b == expected, a['id']
116 changes.append({'id': a['id'], 'before': a['status'], 'after': b['status']})
117 else:
118 assert a == b, a['id']
119 assert {c['id'] for c in changes} == set(ids)
120 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope']} == {
121 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope']}
122 assert cxtm['cases'][:len(before_cxtm['cases'])] == before_cxtm['cases']
123 assert cxtm['contractCounts'] == {'pass': 217, 'fail': 4}
124 added = cxtm['cases'][len(before_cxtm['cases']):]
125 assert [c['id'] for c in added] == ids == [c['id'] for c in registration['cases']]
126 assert registration['failed'] == 0
127 assert registration['passed'] == len(registration['checks']) == len(set(registration['checks']))
128 log = (output / 'retained/regression.log').read_text()
129 assert int(re.search(r'(?m)^REGISTERED-ORIGINALS-PASS: (\d+)', log).group(1)) == registration['passed']
130 candidates = []
131 (output / 'actual').mkdir()
132 for case, row in zip(added, registration['cases']):
133 name = Path(case['id']).stem
134 group = next(group for group, names in GROUPS.items() if name in names)
135 source = ROOT / 'vendor/cxtm' / case['id']
136 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (name + '.ctm.cxtm')
137 expected = baseline.read_bytes()
138 actual = case['actualCXTM'].encode('utf-8')
139 assert case['status'] == row['status'] == 'pass'
140 assert row['group'] == group and row['actualOutcome'] == 'cxtm-match' and row['diagnostics'] == []
141 assert row['sourcePathname'] == case['id'] and row['documentIRI'] == case['baseLocator']
142 assert row['baseline'] == str(baseline.relative_to(ROOT / 'vendor/cxtm'))
143 assert actual == expected == case['expectedCXTM'].encode('utf-8'), case['id']
144 assert case['byteCount'] == row['comparison']['actualBytes'] == row['comparison']['expectedBytes'] == len(expected)
145 assert row['comparison']['firstDifferentByte'] is None and row['sourceBytes'] == len(source.read_bytes())
146 (output / 'actual' / (name + '.ctm.cxtm')).write_bytes(actual)
147 candidates.append({'id': case['id'], 'group': group, 'status': 'pass',
148 'documentIRI': case['baseLocator'], 'bytes': len(actual),
149 'sourceSHA256': sha(source), 'baselineSHA256': sha(baseline),
150 'actualSHA256': digest(actual), 'byteExact': True})
151 save(output / 'candidates.json', candidates)
152 (output / 'contracts.md').write_text(
153 '# Slice 20A: actual original comparisons\n\n'
154 '| Original CTM | Group | Status | CXTM bytes |\n| --- | --- | --- | ---: |\n' +
155 ''.join(f"| `{Path(c['id']).name}` | {c['group']} | PASS | {c['bytes']} |\n" for c in candidates) +
156 '\nAll forty actual outputs equal the unchanged pinned baselines byte for byte. '
157 'Individual source, baseline and actual SHA-256 digests are in `candidates.json`.\n')
158 xtm = read(output / 'retained/xtm.json')
159 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
160 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
161 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
162 remaining = [c for c in final['cases'] if c['status'] == 'not-run']
163 assert len(remaining) == 24
164 assert sum(c['category'] == 'valid' for c in remaining) == 10
165 save(output / 'remaining.json', remaining)
166 forms = (ROOT / 'docs/slice-20a-registration.md').read_text().split('```lisp\n')[-1].split('\n```', 1)[0]
167 inspection_env = dict(environment, CL_SOURCE_REGISTRY=retained['qualifiedSourceProvenance']['sourceRegistry'])
168 inspection_command = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform',
169 '--no-sysinit', '--no-userinit', '--script', '/dev/stdin']
170 inspection = subprocess.run(inspection_command, cwd=ROOT, env=inspection_env, input=forms.encode(),
171 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
172 (output / 'inspection.log').write_bytes(inspection.stdout)
173 assert inspection.returncode == 0, 'Documented inspection failed; see inspection.log'
174 assert protected == {p: sha(ROOT / p) for p in protected}
175 assert historical == {p: sha(ROOT / p) for p in historical}
176 assert sources == {p: sha(ROOT / p) for p in sources}
177 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
178 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
179 evidence = {
180 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
181 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
182 'resumedUncommittedFileSHA256': BEGUN, 'sourceSHA256': sources,
183 'runtime': 'SBCL 2.5.4', 'newDependencyDownloads': 0,
184 'retainedValidationCommand': invocation, 'retainedExecutionEvidence': 'retained/execution.json',
185 'retainedLispChecks': retained['retainedLispChecks'],
186 'xmlQualificationChecks': next(r['passedChecks'] for r in retained['runs'] if r['label'] == 'xml'),
187 'standaloneXTMChecks': next(r['passedChecks'] for r in retained['runs'] if r['label'] == 'xtm'),
188 'xtmMergemapChecks': retained['newXTMIntegrationChecks'],
189 'registeredOriginalChecks': registration['passed'], 'newActualByteExactContracts': len(candidates),
190 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
191 'network': 'All Lisp qualification, regression and inspection processes deny network* with sandbox-exec; Nix preparation uses --offline and verified existing Store sources.',
192 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'],
193 'ctmStatusChanges': changes,
194 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
195 'remainingCases': 'remaining.json; ten positive and fourteen negative cases not selected or executed by Slice 20A',
196 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
197 'oldOriginalComparisonsUnchanged': len(before_cxtm['cases']),
198 'allHistoricalReportBytesUnchanged': True, 'allProtectedFileBytesUnchanged': True,
199 'documentedInspectionPassed': True, 'persistentLispImagesTouched': False,
200 'fullOfflineNixRebuild': 'NOT-RUN; reused verified local Store sources, unchanged patches and fresh SBCL loads.',
201 }
202 save(output / 'execution.json', evidence)
203 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
204 'standaloneXTMChecks', 'xtmMergemapChecks', 'registeredOriginalChecks', 'newActualByteExactContracts',
205 'ctmStatusesAfter', 'xtmCorpusStatuses']}, indent=2))
206
207
208 if __name__ == '__main__':
209 main()