]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-19c.py
feat: support basic XTM topic names (slice 21D)
[topic-maps.git] / scripts / validate-slice-19c.py
1 #!/usr/bin/env python3
2 """Validate Slice 19C in a fresh SBCL 2.5.4 process with network access denied."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 import pathlib
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = pathlib.Path(__file__).resolve().parent.parent
15 BASE = '5e716c078b885e97ebe858230ce7d8578e65ae4b'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity',
20 'relationships', 'templates', 'map-import', 'mergemap', 'include']
21
22
23 def sha(path):
24 return hashlib.sha256(path.read_bytes()).hexdigest()
25
26
27 def main():
28 parser = argparse.ArgumentParser(description=__doc__)
29 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
30 parser.add_argument('--output', default=str(ROOT / 'reports/slice-19c'))
31 args = parser.parse_args()
32 runtime = shutil.which(args.sbcl)
33 assert runtime is not None, 'SBCL executable not found'
34 output = pathlib.Path(args.output).resolve()
35 output.mkdir(parents=True, exist_ok=True)
36 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references']
37 for p in (ROOT / directory).rglob('*') if p.is_file()}
38 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
39 for p in (ROOT / directory).rglob('*') if p.is_file() and
40 '__pycache__' not in p.parts and p.suffix in ['.lisp', '.py', '.sh', '.md']}
41 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md']})
42 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
43 version = subprocess.check_output([runtime, '--version'], text=True).strip()
44 assert version == 'SBCL 2.5.4', version
45 with tempfile.TemporaryDirectory(prefix='tmdm-slice-19c-validation-') as temp:
46 report = pathlib.Path(temp) / 'validation.json'
47 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
48 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(ROOT / 'scripts/test.sh')]
49 result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
50 (output / 'tests.log').write_bytes(result.stdout)
51 if result.returncode:
52 raise RuntimeError('Full regression failed: see ' + str(output / 'tests.log'))
53 log = result.stdout.decode('utf-8')
54 for name in NAMES:
55 source = report if name == 'validation' else pathlib.Path(str(report) + '.' + name + '.json')
56 (output / (name + '.json')).write_bytes(source.read_bytes())
57 old = ROOT / 'reports/slice-19b'
58 previous = json.loads((old / 'validation.json').read_text())
59 now = json.loads((output / 'validation.json').read_text())
60 assert now['statuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
61 selected = sorted('ctm/in/' + n + '.ctm' for n in [
62 'include', 'include-prefix', 'include-prefix2', 'include-template',
63 'include-template-wildcard', 'include-tm-reifier', 'include-topicref',
64 'include-wildcard', 'include-wildcard2'])
65 selected += sorted('ctm/invalid/' + n + '.ctm' for n in ['include-prefix', 'include-prefix2', 'include-xtm'])
66 discrepancies = {'ctm/in/include-template.ctm', 'ctm/in/include-wildcard2.ctm'}
67 changes = [(a['id'], a['status'], b['status']) for a, b in zip(previous['cases'], now['cases']) if a['status'] != b['status']]
68 assert changes == [(id, 'not-run', 'fail' if id in discrepancies else 'pass') for id in selected]
69 assert [c['id'] for c in previous['cases']] == [c['id'] for c in now['cases']]
70 for a, b in zip(previous['cases'], now['cases']):
71 if a['id'] not in selected:
72 assert a == b, a['id']
73 renamed = {
74 'semantics': ('identity-feature-boundary:%include <other.ctm>', 'identity:Include-requires-explicit-resolver'),
75 'prefixes': ('remaining-construct-unsupported:%include <other.ctm>', 'prefix:Include-requires-explicit-resolver'),
76 'references': ('references:feature-boundary:%include <file.ctm>', 'references:Include-requires-explicit-resolver'),
77 'templates': ('diagnostic:unchanged-excluded-feature:%include <other>', 'diagnostic:Include-requires-explicit-resolver')}
78 for name in NAMES[1:-1]:
79 prior = json.loads((old / (name + '.json')).read_text())
80 current = json.loads((output / (name + '.json')).read_text())
81 if name == 'cxtm':
82 assert prior['cases'] == current['cases'][:len(prior['cases'])]
83 assert current['contractCounts'] == {'pass': 176, 'fail': 4}
84 actual = output / 'actual'
85 actual.mkdir(exist_ok=True)
86 for case in current['cases'][len(prior['cases']):]:
87 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
88 raw = case['actualCXTM'].encode('utf-8')
89 assert (raw == baseline.read_bytes()) == (case['id'] not in discrepancies)
90 (actual / baseline.name).write_bytes(raw)
91 elif name in renamed:
92 old_check, new_check = renamed[name]
93 assert [new_check if c == old_check else c for c in prior['checks']] == current['checks']
94 for key in prior.keys() - {'checks', 'rejections'}:
95 assert prior[key] == current[key], name + ':' + key
96 if name == 'templates':
97 # Original rejection class/code/source/positions stay identical.
98 # Contexts now also preserve definition/invocation provenance.
99 for a, b in zip(prior['rejections'], current['rejections']):
100 assert {k: v for k, v in a.items() if k != 'context'} == {k: v for k, v in b.items() if k != 'context'}
101 assert a['context'] == b['context'] or ':DEFINITION-SOURCE' in b['context']
102 else:
103 assert prior == current, name
104 example = (ROOT / 'docs/reading-ctm-include.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
105 inspection = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
106 '--no-userinit', '--script', '/dev/stdin'], input=example.encode(), cwd=ROOT,
107 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
108 (output / 'describe.log').write_bytes(inspection.stdout)
109 assert inspection.returncode == 0, 'Documented inspection forms failed'
110 control = """(require :asdf)
111 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
112 (asdf:load-asd (truename "topic-maps.asd"))
113 (asdf:load-system "topic-maps/ctm")
114 (load (make-string-input-stream (uiop:run-program
115 (list "git" "show" "BASE:src/ctm/lexer.lisp") :output :string)))
116 (handler-case
117 (progn (topic-maps.ctm:deserialize-string
118 "%include <child>"
119 :document-iri "urn:base-control")
120 (error "Base unexpectedly accepted Include"))
121 (topic-maps.ctm:ctm-unsupported (c)
122 (assert (eq :unsupported-directive (topic-maps.ctm:error-code c)))
123 (format t "BASE-CONTROL-PASS: committed lexer rejected Include as ~A~%" c)))
124 """.replace('BASE:', BASE + ':')
125 probe = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
126 '--no-userinit', '--script', '/dev/stdin'], input=control.encode(), cwd=ROOT,
127 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
128 (output / 'base-control.log').write_bytes(probe.stdout)
129 assert probe.returncode == 0, 'Base lexer control failed'
130 for path, value in protected.items():
131 assert sha(ROOT / path) == value, path
132 assert not subprocess.check_output(['git', 'diff', BASE, '--', 'vendor', 'references', 'src/tmdm', 'src/cxtm'], cwd=ROOT)
133 for path, value in sources.items():
134 assert sha(ROOT / path) == value, path
135 previous_totals = json.loads((old / 'execution.json').read_text())['regressionChecks']
136 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1))
137 for marker in [*previous_totals, 'INCLUDE']}
138 assert {k: totals[k] for k in previous_totals} == previous_totals
139 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-19c', runtime=version,
140 startedAtUTC=started, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
141 command=command, exitCode=result.returncode, freshSBCL=True, network='denied by sandbox-exec',
142 regressionChecks=totals, totalLispChecks=sum(totals.values()), importedStatuses=now['statuses'],
143 importedStatusChanges=changes, retainedOriginalCXTMOutputsAndModelObservationsUnchanged=True,
144 originalCorpusAndReferencesUnchanged=True, tmdmAndCxtmSourcesUnchanged=True,
145 documentedInspectionPassed=True, baseLexerCounterprobePassed=True, sourceSHA256=sources,
146 qualification='CTM Include with immediate-parent Document/Wildcard environments, per-level aliases and source-preserving Templates. All twelve originals execute: seven byte-equal positives, three rejections, two explicit mismatches. Two retained DateTime FAILs unchanged; XTM remains NOT-RUN.',
147 artifactsSHA256={str(p.relative_to(output)): sha(p) for p in sorted(output.rglob('*'))
148 if p.is_file() and p.name != 'execution.json'})
149 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
150 print(json.dumps({k: evidence[k] for k in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys=True))
151 print('SLICE-19C-VALIDATION-PASS: fresh offline regression; seven byte-exact Include originals, three rejections, two original mismatches; retained contracts unchanged')
152
153
154 if __name__ == '__main__':
155 main()