]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-16b.py
Add XTM name scope with five original contracts and offline evidence
[topic-maps.git] / scripts / validate-slice-16b.py
1 #!/usr/bin/env python3
2 """Record Slice 16B's complete offline fresh-SBCL regression and source evidence."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 import pathlib
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = pathlib.Path(__file__).resolve().parent.parent
15 BASE = 'f24ac27e1e1cc0118411b058d40af3c1c87f3ffb'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification']
20
21
22 def sha(path):
23 return hashlib.sha256(path.read_bytes()).hexdigest()
24
25
26 def main():
27 parser = argparse.ArgumentParser(description=__doc__)
28 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
29 parser.add_argument('--output', default=str(ROOT / 'reports/slice-16b'))
30 args = parser.parse_args()
31 runtime = shutil.which(args.sbcl)
32 if runtime is None:
33 raise RuntimeError('SBCL executable not found: ' + args.sbcl)
34 output = pathlib.Path(args.output).resolve()
35 output.mkdir(parents=True, exist_ok=True)
36 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references']
37 for p in (ROOT / directory).rglob('*') if p.is_file()}
38 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
39 for p in (ROOT / directory).rglob('*') if p.is_file() and
40 '__pycache__' not in p.parts and p.suffix in ['.lisp', '.py', '.sh', '.md']}
41 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md']})
42 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
43 version = subprocess.check_output([runtime, '--version'], text=True).strip()
44 with tempfile.TemporaryDirectory(prefix='ctm-slice-16b-validation-') as temp:
45 report = pathlib.Path(temp) / 'validation.json'
46 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
47 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(ROOT / 'scripts/test.sh')]
48 result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
49 (output / 'tests.log').write_bytes(result.stdout)
50 if result.returncode:
51 raise RuntimeError('Full regression failed: see ' + str(output / 'tests.log'))
52 log = result.stdout.decode('utf-8')
53 for name in NAMES:
54 source = report if name == 'validation' else pathlib.Path(str(report) + '.' + name + '.json')
55 (output / (name + '.json')).write_bytes(source.read_bytes())
56 old = ROOT / 'reports/slice-16a'
57 previous = json.loads((old / 'validation.json').read_text())
58 now = json.loads((output / 'validation.json').read_text())
59 assert now['statuses'] == {'pass': 106, 'fail': 2, 'unsupported': 0, 'not-run': 165}
60 reification = json.loads((output / 'reification.json').read_text())
61 selected = sorted([c['id'] for c in reification['cases']] + [c['id'] for c in reification['negativeCases']])
62 changes = [(a['id'], a['status'], b['status']) for a, b in zip(previous['cases'], now['cases']) if a['status'] != b['status']]
63 assert changes == [(id, 'not-run', 'pass') for id in selected]
64 for a, b in zip(previous['cases'], now['cases']):
65 if a['id'] not in selected:
66 assert a == b, a['id']
67 for name in NAMES[1:-1]:
68 key = 'lexicalProbes' if name == 'timezones' else 'examples' if name == 'identifiers' else 'cases'
69 prior = json.loads((old / (name + '.json')).read_text())[key]
70 current = json.loads((output / (name + '.json')).read_text())[key]
71 assert prior == (current[:len(prior)] if name == 'cxtm' else current), name
72 actual_reifiers = output / 'reification-actual'
73 actual_reifiers.mkdir(exist_ok=True)
74 for case in json.loads((output / 'cxtm.json').read_text())['cases']:
75 if case['id'] in selected:
76 actual = case['actualCXTM'].encode('utf-8')
77 assert actual == (ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')).read_bytes()
78 (actual_reifiers / (pathlib.Path(case['id']).name + '.cxtm')).write_bytes(actual)
79 identifiers = json.loads((output / 'identifiers.json').read_text())
80 actual_dir = output / 'manual-model-actual'
81 actual_dir.mkdir(exist_ok=True)
82 for example in identifiers['examples']:
83 actual = example['actualCXTM'].encode('utf-8')
84 assert actual == (ROOT / 'vendor/cxtm' / example['baseline']).read_bytes()
85 (actual_dir / (pathlib.Path(example['source']).name + '.cxtm')).write_bytes(actual)
86 document = (ROOT / 'docs/reading-reification.md').read_text()
87 example = document.split('```lisp\n', 2)[2].split('\n```', 1)[0]
88 inspection = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
89 '--no-userinit', '--script', '/dev/stdin'], input=example.encode(), cwd=ROOT,
90 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
91 (output / 'describe.log').write_bytes(inspection.stdout)
92 assert inspection.returncode == 0, 'Documented inspection forms failed'
93 for path, value in protected.items():
94 assert sha(ROOT / path) == value, path
95 for path, value in sources.items():
96 assert sha(ROOT / path) == value, path
97 previous_totals = json.loads((old / 'execution.json').read_text())['regressionChecks']
98 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1))
99 for marker in [*previous_totals, 'REIFICATION']}
100 assert {k: totals[k] for k in previous_totals} == previous_totals
101 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-16b', runtime=version,
102 startedAtUTC=started, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
103 command=command, exitCode=result.returncode, freshSBCL=True, network='denied by sandbox-exec',
104 regressionChecks=totals, totalLispChecks=sum(totals.values()), importedStatuses=now['statuses'],
105 importedStatusChanges=changes, previous83CXTMOutputsAndAllModelObservationsUnchanged=True,
106 reificationOriginalPositives=len(reification['cases']), reificationOriginalNegatives=len(reification['negativeCases']),
107 manualXTM2ModelBaselineComparisons=len(identifiers['examples']), xtmDeserializationImplemented=False,
108 originalCorpusAndReferencesUnchanged=True, sourceSHA256=sources, documentedInspectionPassed=True,
109 qualification='Same-map Reification; manual TMDM models are not XTM deserialization contracts. The two DateTime baseline FAILs remain unchanged.',
110 artifactsSHA256={str(p.relative_to(output)): sha(p) for p in sorted(output.rglob('*'))
111 if p.is_file() and p.name != 'execution.json'})
112 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
113 print(json.dumps({k: evidence[k] for k in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys=True))
114 print('SLICE-16B-VALIDATION-PASS: fresh offline SBCL; 11 exact original Reification comparisons and 2 specific rejections; retained results unchanged')
115
116
117 if __name__ == '__main__':
118 main()