]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-17a.py
Add XTM Variant reification and qualify Slice 21O originals
[topic-maps.git] / scripts / validate-slice-17a.py
1 #!/usr/bin/env python3
2 """Record Slice 17A's full offline regression and unchanged retained evidence."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 import pathlib
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = pathlib.Path(__file__).resolve().parent.parent
15 BASE = 'f1270a43720ab1454bebfb3d0a9e3555554a7ab9'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity']
20 SELECTED = 'ctm/in/association-duplicate-role.ctm'
21
22
23 def sha(path):
24 return hashlib.sha256(path.read_bytes()).hexdigest()
25
26
27 def main():
28 parser = argparse.ArgumentParser(description=__doc__)
29 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
30 parser.add_argument('--output', default=str(ROOT / 'reports/slice-17a'))
31 args = parser.parse_args()
32 runtime = shutil.which(args.sbcl)
33 if runtime is None:
34 raise RuntimeError('SBCL executable not found: ' + args.sbcl)
35 output = pathlib.Path(args.output).resolve()
36 output.mkdir(parents=True, exist_ok=True)
37 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references', 'src/tmdm', 'src/cxtm']
38 for p in (ROOT / directory).rglob('*') if p.is_file()}
39 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
40 for p in (ROOT / directory).rglob('*') if p.is_file() and
41 '__pycache__' not in p.parts and p.suffix in ['.lisp', '.py', '.sh', '.md']}
42 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md']})
43 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
44 version = subprocess.check_output([runtime, '--version'], text=True).strip()
45 with tempfile.TemporaryDirectory(prefix='ctm-slice-17a-validation-') as temp:
46 report = pathlib.Path(temp) / 'validation.json'
47 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
48 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(ROOT / 'scripts/test.sh')]
49 result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
50 (output / 'tests.log').write_bytes(result.stdout)
51 if result.returncode:
52 raise RuntimeError('Full regression failed: see ' + str(output / 'tests.log'))
53 log = result.stdout.decode('utf-8')
54 for name in NAMES:
55 source = report if name == 'validation' else pathlib.Path(str(report) + '.' + name + '.json')
56 (output / (name + '.json')).write_bytes(source.read_bytes())
57 old = ROOT / 'reports/slice-16b'
58 previous = json.loads((old / 'validation.json').read_text())
59 now = json.loads((output / 'validation.json').read_text())
60 assert now['statuses'] == {'pass': 107, 'fail': 2, 'unsupported': 0, 'not-run': 164}
61 assert [c['id'] for c in previous['cases']] == [c['id'] for c in now['cases']]
62 changes = [(a['id'], a['status'], b['status']) for a, b in zip(previous['cases'], now['cases']) if a['status'] != b['status']]
63 assert changes == [(SELECTED, 'not-run', 'pass')]
64 for a, b in zip(previous['cases'], now['cases']):
65 if a['id'] != SELECTED:
66 assert a == b, a['id']
67 for name in NAMES[1:-1]:
68 key = 'lexicalProbes' if name == 'timezones' else 'examples' if name == 'identifiers' else 'cases'
69 prior = json.loads((old / (name + '.json')).read_text())[key]
70 current = json.loads((output / (name + '.json')).read_text())[key]
71 assert prior == (current[:len(prior)] if name == 'cxtm' else current), name
72 if name == 'reification':
73 assert json.loads((old / (name + '.json')).read_text())['negativeCases'] == json.loads((output / (name + '.json')).read_text())['negativeCases']
74 cxtm = json.loads((output / 'cxtm.json').read_text())
75 assert cxtm['contractCounts'] == {'pass': 95, 'fail': 2}
76 selected = [c for c in cxtm['cases'] if c['id'] == SELECTED]
77 assert len(selected) == 1
78 actual = selected[0]['actualCXTM'].encode('utf-8')
79 baseline = ROOT / 'vendor/cxtm/ctm/baseline/association-duplicate-role.ctm.cxtm'
80 assert actual == baseline.read_bytes() and len(actual) == 1157
81 (output / 'association-duplicate-role.ctm.cxtm').write_bytes(actual)
82 example = (ROOT / 'docs/reading-association-arity.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
83 inspection = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
84 '--no-userinit', '--script', '/dev/stdin'], input=example.encode(), cwd=ROOT,
85 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
86 (output / 'describe.log').write_bytes(inspection.stdout)
87 assert inspection.returncode == 0, 'Documented inspection forms failed'
88 # Demonstrate that the original reader rejects this exact source. It is
89 # loaded only in another disposable process; the checkout is never reverted.
90 control = '''(require :asdf)
91 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
92 (asdf:load-asd (truename "topic-maps.asd"))
93 (asdf:load-system "topic-maps/ctm")
94 (let ((source (uiop:run-program '("git" "show" "BASE:src/ctm/reader.lisp") :output :string)))
95 (load (make-string-input-stream source)))
96 (handler-case
97 (progn (topic-maps.ctm:deserialize-file "vendor/cxtm/ctm/in/association-duplicate-role.ctm"
98 :document-iri "https://example.invalid/original.ctm")
99 (error "Original reader unexpectedly accepted n-ary source"))
100 (topic-maps.ctm:ctm-unsupported (c)
101 (assert (eq :association-arity (topic-maps.ctm:error-code c)))
102 (assert (= 48 (topic-maps.ctm:error-position c)))
103 (format t "ORIGINAL-READER-CONTROL-PASS: CTM-UNSUPPORTED :ASSOCIATION-ARITY at 48~%")))
104 '''.replace('BASE:', BASE + ':')
105 negative = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
106 '--no-userinit', '--script', '/dev/stdin'], input=control.encode(), cwd=ROOT,
107 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
108 (output / 'original-reader-control.log').write_bytes(negative.stdout)
109 assert negative.returncode == 0, 'Original reader control failed'
110 for path, value in protected.items():
111 assert sha(ROOT / path) == value, path
112 # The independent model and serializer must also match the accepted base.
113 assert not subprocess.check_output(['git', 'diff', BASE, '--', 'src/tmdm', 'src/cxtm', 'vendor', 'references'], cwd=ROOT)
114 for path, value in sources.items():
115 assert sha(ROOT / path) == value, path
116 previous_totals = json.loads((old / 'execution.json').read_text())['regressionChecks']
117 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1))
118 for marker in [*previous_totals, 'ASSOCIATION-ARITY']}
119 assert {k: totals[k] for k in previous_totals} == previous_totals
120 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-17a', runtime=version,
121 startedAtUTC=started, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
122 command=command, exitCode=result.returncode, freshSBCL=True, network='denied by sandbox-exec',
123 regressionChecks=totals, totalLispChecks=sum(totals.values()), importedStatuses=now['statuses'],
124 importedStatusChanges=changes, previous94PassingCXTMAndTwoKnownFailuresUnchanged=True,
125 allRetainedModelObservationsUnchanged=True, modelAndCanonicalizerUnchangedFromBase=True,
126 originalCorpusAndReferencesUnchanged=True, documentedInspectionPassed=True,
127 originalReaderControlPassed=True, sourceSHA256=sources,
128 qualification='Unbounded CTM Role declarations; core-subject identity protection and isa/ako remain open for 17B. The two DateTime baseline FAILs remain unchanged.',
129 artifactsSHA256={str(p.relative_to(output)): sha(p) for p in sorted(output.rglob('*'))
130 if p.is_file() and p.name != 'execution.json'})
131 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
132 print(json.dumps({k: evidence[k] for k in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses', 'importedStatusChanges']}, sort_keys=True))
133 print('SLICE-17A-VALIDATION-PASS: complete fresh offline SBCL regression; 1157 original bytes; all retained contracts unchanged')
134
135
136 if __name__ == '__main__':
137 main()