]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-15.py
Support textual XTM name variants with qualified original contracts and offline evidence
[topic-maps.git] / scripts / validate-slice-15.py
1 #!/usr/bin/env python3
2 """Record one full offline fresh-SBCL run. No relocation or cache management."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 import pathlib
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = pathlib.Path(__file__).resolve().parent.parent
15 BASE = '4f7b4d26d40f8f5566594aa2e3ad6296d44fc4c3'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones', 'typed-names', 'scopes', 'variants', 'references']
19
20
21 def sha(path):
22 return hashlib.sha256(path.read_bytes()).hexdigest()
23
24
25 def main():
26 parser = argparse.ArgumentParser(description=__doc__)
27 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
28 parser.add_argument('--output', default=str(ROOT / 'reports/slice-15'))
29 args = parser.parse_args()
30 runtime = shutil.which(args.sbcl)
31 if runtime is None:
32 raise RuntimeError('SBCL executable not found: ' + args.sbcl)
33 output = pathlib.Path(args.output).resolve()
34 output.mkdir(parents=True, exist_ok=True)
35 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references']
36 for p in (ROOT / directory).rglob('*') if p.is_file()}
37 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts']
38 for p in (ROOT / directory).rglob('*') if p.is_file() and
39 '__pycache__' not in p.parts and p.suffix in ['.lisp', '.py', '.sh']}
40 sources['topic-maps.asd'] = sha(ROOT / 'topic-maps.asd')
41 version = subprocess.check_output([runtime, '--version'], text=True).strip()
42 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
43 with tempfile.TemporaryDirectory(prefix='ctm-slice-15-validation-') as temp:
44 report = pathlib.Path(temp) / 'validation.json'
45 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
46 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(ROOT / 'scripts/test.sh')]
47 result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
48 (output / 'tests.log').write_bytes(result.stdout)
49 if result.returncode:
50 raise RuntimeError('Full regression failed: see ' + str(output / 'tests.log'))
51 log = result.stdout.decode('utf-8')
52 for name in NAMES:
53 source = report if name == 'validation' else pathlib.Path(str(report) + '.' + name + '.json')
54 (output / (name + '.json')).write_bytes(source.read_bytes())
55 old = ROOT / 'reports/slice-14'
56 for name in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones', 'typed-names', 'scopes', 'variants']:
57 key = 'lexicalProbes' if name == 'timezones' else 'cases'
58 assert json.loads((old / (name + '.json')).read_text())[key] == json.loads((output / (name + '.json')).read_text())[key], name
59 previous = json.loads((old / 'cxtm.json').read_text())
60 current = json.loads((output / 'cxtm.json').read_text())
61 assert previous['cases'] == current['cases'][:64]
62 original = json.loads((old / 'validation.json').read_text())
63 now = json.loads((output / 'validation.json').read_text())
64 transitions = [(a['id'], a['status'], b['status']) for a, b in zip(original['cases'], now['cases']) if a['status'] != b['status']]
65 selected = json.loads((output / 'references.json').read_text())['cases']
66 selected_ids = [c['id'] for c in selected] + ['ctm/invalid/embedded-topic-global-scope.ctm']
67 assert transitions == [(id, 'not-run', 'pass') for id in sorted(selected_ids)], transitions
68 assert now['statuses'] == {'pass': 93, 'fail': 2, 'unsupported': 0, 'not-run': 178}
69 assert current['contractCounts'] == {'pass': 83, 'fail': 2}
70 actual_dir = output / 'actual'
71 actual_dir.mkdir(exist_ok=True)
72 for case in current['cases']:
73 if case['id'] in selected_ids:
74 (actual_dir / (pathlib.Path(case['id']).name + '.cxtm')).write_bytes(case['actualCXTM'].encode('utf-8'))
75 document = (ROOT / 'docs/reading-topic-references.md').read_text()
76 example = document.split('```lisp\n', 1)[1].split('\n```', 1)[0]
77 inspection = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
78 '--no-userinit', '--script', '/dev/stdin'], input=example.encode(), cwd=ROOT,
79 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
80 (output / 'describe.log').write_bytes(inspection.stdout)
81 assert inspection.returncode == 0
82 for path, value in protected.items():
83 assert sha(ROOT / path) == value, path
84 for path, value in sources.items():
85 assert sha(ROOT / path) == value, path
86 markers = ['FOUNDATION', 'CTM-IDENTITY-SEMANTICS', 'CTM-MERGING-SEMANTICS', 'CTM-ASSOCIATIONS-SEMANTICS',
87 'CXTM', 'OCCURRENCES', 'PREFIX-DATATYPES', 'INTEGERS', 'DECIMALS', 'DATES', 'DATETIMES', 'TIMEZONES', 'TYPED-NAMES', 'SCOPES', 'VARIANTS', 'REFERENCES']
88 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in markers}
89 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-15', runtime=version,
90 startedAtUTC=started, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
91 command=command, exitCode=result.returncode, freshSBCL=True, network='denied by sandbox-exec',
92 regressionChecks=totals, totalLispChecks=sum(totals.values()), importedStatuses=now['statuses'],
93 transitions=transitions, previous64CXTMOutputsAndOutcomesUnchanged=True, originalCorpusAndReferencesUnchanged=True,
94 sourceSHA256=sources, documentedInspectionPassed=True,
95 qualification='Pinned CTM draft-era corpus; regression assertion PASS does not erase the two normative baseline FAILs.',
96 artifactsSHA256={str(p.relative_to(output)): sha(p) for p in sorted(output.rglob('*'))
97 if p.is_file() and p.name != 'execution.json'})
98 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
99 print(json.dumps({k: evidence[k] for k in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses', 'transitions']}, sort_keys=True))
100 print('SLICE-15-VALIDATION-PASS: fresh offline SBCL; retained outputs unchanged; 21 original reference comparisons and one specific rejection')
101
102
103 if __name__ == '__main__':
104 main()