]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-17b.py
Implement noncyclic XTM mergeMap content import (Slice 22E)
[topic-maps.git] / scripts / validate-slice-17b.py
1 #!/usr/bin/env python3
2 """Record Slice 17B's full fresh offline regression and unchanged prior contracts."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 import pathlib
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = pathlib.Path(__file__).resolve().parent.parent
15 BASE = 'bdc56a8236c6e6180df217d37e82846025026096'
16 POLICY = '(version 1)(allow default)(deny network*)'
17 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity', 'relationships']
20 SELECTED = ['ctm/in/' + name + '.ctm' for name in [
21 'isa-relationship', 'isa-relationship-duplicate', 'isa-relationship-duplicate2',
22 'isa-relationship-embedded', 'isa-relationship-type-qname', 'ako-relationship',
23 'ako-relationship-duplicate', 'ako-relationship-duplicate2', 'ako-relationship-embedded',
24 'ako-relationship-supertype-qname', 'ako-merge-away', 'subject-identifier-topicref', 'item-identifier-topicref']]
25
26
27 def sha(path):
28 return hashlib.sha256(path.read_bytes()).hexdigest()
29
30
31 def main():
32 parser = argparse.ArgumentParser(description=__doc__)
33 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
34 parser.add_argument('--output', default=str(ROOT / 'reports/slice-17b'))
35 args = parser.parse_args()
36 runtime = shutil.which(args.sbcl)
37 if runtime is None:
38 raise RuntimeError('SBCL executable not found: ' + args.sbcl)
39 output = pathlib.Path(args.output).resolve()
40 output.mkdir(parents=True, exist_ok=True)
41 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references']
42 for p in (ROOT / directory).rglob('*') if p.is_file()}
43 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
44 for p in (ROOT / directory).rglob('*') if p.is_file() and
45 '__pycache__' not in p.parts and p.suffix in ['.lisp', '.py', '.sh', '.md']}
46 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md']})
47 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
48 version = subprocess.check_output([runtime, '--version'], text=True).strip()
49 with tempfile.TemporaryDirectory(prefix='ctm-slice-17b-validation-') as temp:
50 report = pathlib.Path(temp) / 'validation.json'
51 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
52 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(ROOT / 'scripts/test.sh')]
53 result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
54 (output / 'tests.log').write_bytes(result.stdout)
55 if result.returncode:
56 raise RuntimeError('Full regression failed: see ' + str(output / 'tests.log'))
57 log = result.stdout.decode('utf-8')
58 for name in NAMES:
59 source = report if name == 'validation' else pathlib.Path(str(report) + '.' + name + '.json')
60 (output / (name + '.json')).write_bytes(source.read_bytes())
61 old = ROOT / 'reports/slice-17a'
62 previous = json.loads((old / 'validation.json').read_text())
63 now = json.loads((output / 'validation.json').read_text())
64 assert now['statuses'] == {'pass': 120, 'fail': 2, 'unsupported': 0, 'not-run': 151}
65 assert [c['id'] for c in previous['cases']] == [c['id'] for c in now['cases']]
66 changes = [(a['id'], a['status'], b['status']) for a, b in zip(previous['cases'], now['cases']) if a['status'] != b['status']]
67 assert changes == [(id, 'not-run', 'pass') for id in sorted(SELECTED)]
68 for a, b in zip(previous['cases'], now['cases']):
69 if a['id'] not in SELECTED:
70 assert a == b, a['id']
71 for name in NAMES[1:-1]:
72 key = 'lexicalProbes' if name == 'timezones' else 'examples' if name == 'identifiers' else 'original' if name == 'arity' else 'cases'
73 prior = json.loads((old / (name + '.json')).read_text())[key]
74 current = json.loads((output / (name + '.json')).read_text())[key]
75 assert prior == (current[:len(prior)] if name == 'cxtm' else current), name
76 extra = 'negativeCases' if name == 'reification' else 'witnesses' if name == 'arity' else None
77 if extra:
78 assert json.loads((old / (name + '.json')).read_text())[extra] == json.loads((output / (name + '.json')).read_text())[extra], name
79 cxtm = json.loads((output / 'cxtm.json').read_text())
80 assert cxtm['contractCounts'] == {'pass': 108, 'fail': 2}
81 actual_dir = output / 'actual'
82 actual_dir.mkdir(exist_ok=True)
83 selected = [c for c in cxtm['cases'] if c['id'] in SELECTED]
84 assert len(selected) == len(SELECTED)
85 for case in selected:
86 actual = case['actualCXTM'].encode('utf-8')
87 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
88 assert actual == baseline.read_bytes()
89 (actual_dir / baseline.name).write_bytes(actual)
90 example = (ROOT / 'docs/reading-type-relationships.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
91 inspection = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
92 '--no-userinit', '--script', '/dev/stdin'], input=example.encode(), cwd=ROOT,
93 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
94 (output / 'describe.log').write_bytes(inspection.stdout)
95 assert inspection.returncode == 0, 'Documented inspection forms failed'
96 # Restore definitions only inside another fresh disposable process. No
97 # checkout reversal, persistent image changes or alternate object model.
98 control = '''(require :asdf)
99 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
100 (asdf:load-asd (truename "topic-maps.asd"))
101 (asdf:load-system "topic-maps/ctm")
102 (dolist (path '("src/tmdm/reification.lisp" "src/tmdm/merging.lisp" "src/ctm/reader.lisp"))
103 (load (make-string-input-stream
104 (uiop:run-program (list "git" "show" (concatenate 'string "BASE:" path)) :output :string))))
105 (let* ((m (topic-maps.tmdm:make-topic-map))
106 (a (topic-maps.tmdm:intern-topic m :subject-identifier "http://psi.topicmaps.org/iso13250/model/type"))
107 (b (topic-maps.tmdm:intern-topic m :subject-identifier "http://psi.topicmaps.org/iso13250/model/instance"))
108 (c (topic-maps.tmdm:merge-topics a b)))
109 (assert (= 1 (length (topic-maps.tmdm:topic-map-topics m))))
110 (assert (= 2 (length (topic-maps.tmdm:topic-subject-identifiers c))))
111 (format t "BASE-CORE-GAP-CONFIRMED: different type/instance Core Subjects merged~%"))
112 (dolist (text '("a isa b." "a ako b."))
113 (handler-case
114 (progn (topic-maps.ctm:deserialize-string text :document-iri "https://example.invalid/base.ctm")
115 (error "Base reader unexpectedly accepted relationship syntax"))
116 (topic-maps.ctm:ctm-unsupported (c)
117 (assert (eq :unsupported-keyword (topic-maps.ctm:error-code c)))
118 (format t "BASE-READER-CONTROL-PASS: ~S rejected as ~A~%" text (topic-maps.ctm:error-code c)))))
119 '''.replace('BASE:', BASE + ':')
120 negative = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit',
121 '--no-userinit', '--script', '/dev/stdin'], input=control.encode(), cwd=ROOT,
122 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
123 (output / 'base-control.log').write_bytes(negative.stdout)
124 assert negative.returncode == 0, 'Base control failed'
125 for path, value in protected.items():
126 assert sha(ROOT / path) == value, path
127 assert not subprocess.check_output(['git', 'diff', BASE, '--', 'vendor', 'references'], cwd=ROOT)
128 for path, value in sources.items():
129 assert sha(ROOT / path) == value, path
130 previous_totals = json.loads((old / 'execution.json').read_text())['regressionChecks']
131 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1))
132 for marker in [*previous_totals, 'TYPE-RELATIONSHIPS']}
133 assert {k: totals[k] for k in previous_totals} == previous_totals
134 relationships = json.loads((output / 'relationships.json').read_text())
135 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-17b', runtime=version,
136 startedAtUTC=started, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
137 command=command, exitCode=result.returncode, freshSBCL=True, network='denied by sandbox-exec',
138 regressionChecks=totals, totalLispChecks=sum(totals.values()), importedStatuses=now['statuses'],
139 importedStatusChanges=changes, previous95PassingCXTMAndTwoKnownFailuresUnchanged=True,
140 allRetainedModelObservationsUnchanged=True, originalCorpusAndReferencesUnchanged=True,
141 documentedInspectionPassed=True, baseControlPassed=True,
142 protectedCoreSubjectIdentifiers=relationships['coreSubjectIdentifiers'],
143 atomicConflictObservations=len(relationships['rejections']), sourceSHA256=sources,
144 qualification='CTM isa/ako and TMDM 2006 Core Subject locators. Atomic semantic rejection is API policy. No inferred Associations are materialized. The two DateTime baseline FAILs remain unchanged.',
145 artifactsSHA256={str(p.relative_to(output)): sha(p) for p in sorted(output.rglob('*'))
146 if p.is_file() and p.name != 'execution.json'})
147 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
148 print(json.dumps({k: evidence[k] for k in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses', 'atomicConflictObservations']}, sort_keys=True))
149 print('SLICE-17B-VALIDATION-PASS: complete fresh offline SBCL regression; 13 original byte comparisons; all retained contracts unchanged')
150
151
152 if __name__ == '__main__':
153 main()