]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-19d2.py
Add XTM Variant reification and qualify Slice 21O originals
[topic-maps.git] / scripts / validate-slice-19d2.py
1 #!/usr/bin/env python3
2 """Validate controlled XTM Mergemap and all retained suites, offline and fresh."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = 'f6a86668fce782a2e77d3bd763d58b0489079d75'
16 POLICY = '(version 1)(allow default)(deny network*)'
17
18
19 def command(args):
20 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
21
22
23 def sha(path):
24 return hashlib.sha256(path.read_bytes()).hexdigest()
25
26
27 def read(path):
28 return json.loads(path.read_text())
29
30
31 def main():
32 parser = argparse.ArgumentParser(description=__doc__)
33 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
34 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-19d2')
35 args = parser.parse_args()
36 runtime = shutil.which(args.sbcl)
37 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
38 output = args.output.resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 protected_paths = ['src/tmdm', 'src/cxtm', 'src/xtm', 'src/xml', 'patches', 'flake.nix',
41 'flake.lock', 'vendor', 'references', 'scripts/prepare-cxml.py',
42 'tests/cxml-probe.lisp', 'tests/cxml-encoding-regressions.lisp', 'tests/xtm-identity.lisp', 'scripts/test.sh']
43 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
44 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references', 'patches']
45 for p in (ROOT / directory).rglob('*') if p.is_file()}
46 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
47 for p in (ROOT / directory).rglob('*') if p.is_file() and p.suffix in ['.lisp', '.py', '.sh', '.md']}
48 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md', 'flake.nix', 'flake.lock']})
49 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
50 xtm_source = '\n'.join(p.read_text() for p in sorted((ROOT / 'src/xtm').glob('*.lisp')))
51 assert xtm_source.count('(topic-maps.xml:parse-octets ') == 1
52 assert not re.search(r'\(cxml:parse[-\w]*\s', xtm_source, re.I)
53 runs = []
54 with tempfile.TemporaryDirectory(prefix='topic-maps-19d2-') as tmp:
55 tmp = Path(tmp)
56 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(tmp / 'sources')]
57 provenance = json.loads(command(preparation))
58 environment = dict(os.environ, CL_SOURCE_REGISTRY=provenance['sourceRegistry'], PYTHONDONTWRITEBYTECODE='1')
59 base_report = tmp / 'core.json'
60 final_prefix = tmp / 'xtm-mergemap'
61 for label, invocation in [
62 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
63 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
64 ('regression', [str(ROOT / 'scripts/test.sh')])]:
65 env = dict(environment, SBCL=runtime)
66 if label == 'xtm':
67 env['TOPIC_MAPS_XTM_REPORT'] = str(output / 'xtm.json')
68 if label == 'regression':
69 env['TOPIC_MAPS_REPORT'] = str(base_report)
70 env['TOPIC_MAPS_XTM_MERGEMAP_REPORT'] = str(final_prefix)
71 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
72 context = {'label': label, 'command': invocation, 'freshProcess': True,
73 'network': 'denied by sandbox-exec', 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
74 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
75 stderr=subprocess.STDOUT, timeout=240)
76 (output / (label + '.log')).write_bytes(result.stdout)
77 context.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
78 assert result.returncode == 0, label + ' failed; see saved log'
79 if label in ['xml', 'xtm']:
80 prefix = 'CXML' if label == 'xml' else 'XTM'
81 passed = re.findall(r'^' + prefix + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
82 assert len(passed) == len(set(passed))
83 assert prefix + '-SUMMARY ' + str(len(passed)) + ' PASS 0 FAIL' in result.stdout.decode()
84 context['passedChecks'] = len(passed)
85 assert len(passed) == (146 if label == 'xml' else 327)
86 runs.append(context)
87 base = read(base_report)
88 baseline = ROOT / 'reports/slice-19c'
89 assert base['statuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
90 assert base == read(baseline / 'validation.json')
91 retained = []
92 for old in sorted(baseline.glob('*.json')):
93 if old.name == 'execution.json':
94 continue
95 path = base_report if old.name == 'validation.json' else Path(str(base_report) + '.' + old.name)
96 prior, current = read(old), read(path)
97 if old.name in ['mergemap.json', 'cxtm.json']:
98 renamed = {'diagnostic:known-XTM-unimplemented:specific-condition': 'diagnostic:XTM-explicit-resolver-required:specific-condition',
99 'diagnostic:known-XTM-unimplemented:source-position': 'diagnostic:XTM-explicit-resolver-required:source-position',
100 'policy:XTM-recognized-without-retrieval:specific-condition': 'policy:XTM-explicit-resolver-before-retrieval:specific-condition',
101 'policy:XTM-recognized-without-retrieval:source-position': 'policy:XTM-explicit-resolver-before-retrieval:source-position',
102 'policy:XTM-does-not-call-resolver': 'policy:XTM-without-resolver-does-not-fetch'}
103 assert current['checks'] == [renamed.get(c, c) for c in prior['checks']]
104 assert {k: v for k, v in prior.items() if k != 'checks'} == {k: v for k, v in current.items() if k != 'checks'}
105 else:
106 assert old.read_bytes() == path.read_bytes(), old.name
107 retained.append(old.name)
108 final = read(Path(str(final_prefix) + '.validation.json'))
109 cxtm = read(Path(str(final_prefix) + '.cxtm.json'))
110 integration = read(Path(str(final_prefix) + '.integration.json'))
111 assert final['statuses'] == {'pass': 205, 'fail': 4, 'unsupported': 0, 'not-run': 64}
112 changes = [(a['id'], a['status'], b['status']) for a, b in zip(base['cases'], final['cases']) if a != b]
113 assert changes == [('ctm/in/mergemap-xtm.ctm', 'not-run', 'pass')]
114 assert len(final['cases']) == len(base['cases']) == 273
115 old_cxtm = read(Path(str(base_report) + '.cxtm.json'))
116 assert cxtm['cases'][:-1] == old_cxtm['cases']
117 assert cxtm['contractCounts'] == {'pass': 177, 'fail': 4}
118 original = cxtm['cases'][-1]
119 expected = (ROOT / 'vendor/cxtm/ctm/baseline/mergemap-xtm.ctm.cxtm').read_bytes()
120 actual = original['actualCXTM'].encode()
121 assert len(actual) == len(expected) == 1519
122 assert actual == expected == original['expectedCXTM'].encode()
123 (output / 'mergemap-xtm.actual.cxtm').write_bytes(actual)
124 for name in ['validation', 'cxtm', 'integration']:
125 shutil.copyfile(Path(str(final_prefix) + '.' + name + '.json'), output / (name + '.json'))
126 assert integration['failed'] == 0 and integration['passed'] == len(integration['checks']) == len(set(integration['checks']))
127 log = (output / 'regression.log').read_text()
128 previous_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
129 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in previous_totals}
130 assert totals == previous_totals and sum(totals.values()) == 12100
131 assert int(re.search(r'XTM-MERGEMAP-PASS: (\d+)', log).group(1)) == integration['passed']
132 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log
133 assert 'XTM-ADAPTER-BOUNDARY-PASS:' in log
134 xtm = read(output / 'xtm.json')
135 assert xtm == read(ROOT / 'reports/slice-19d1/xtm-1.json')
136 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
137 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
138 forms = (ROOT / 'docs/slice-19d2-xtm-mergemap.md').read_text().split('```lisp\n')[-1].split('\n```', 1)[0]
139 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', '/dev/stdin']
140 inspection = subprocess.run(invocation, cwd=ROOT, env=environment, input=forms.encode(),
141 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
142 (output / 'inspection.log').write_bytes(inspection.stdout)
143 assert inspection.returncode == 0, 'Documented inspection forms failed'
144 assert protected == {p: sha(ROOT / p) for p in protected}
145 assert sources == {p: sha(ROOT / p) for p in sources}
146 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
147 evidence = {'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
148 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
149 'runtime': 'SBCL 2.5.4', 'newDependencyDownloads': 0, 'sourcePreparationCommand': preparation,
150 'qualifiedSourceProvenance': provenance, 'sourceSHA256': sources,
151 'pinnedCorpusXMLPatchesAndTMDMUnchanged': True, 'runs': runs,
152 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
153 'newXTMIntegrationChecks': integration['passed'], 'retainedReportsVerified': retained,
154 'retainedReportChange': 'Five obsolete XTM-unimplemented check labels now test missing explicit resolver, in mergemap.json and its checks embedded in cxtm.json; original cases/observations unchanged',
155 'ctmStatuses': final['statuses'], 'ctmStatusChanges': changes,
156 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
157 'xtmPositiveAndNegativeRegistration': 'Six original positives executed; 107 other positives and 11 negative inputs remain unexecuted.',
158 'allFourKnownDiscrepanciesUnchanged': True, 'originalBaselineByteCount': 1519,
159 'documentedInspectionPassed': True, 'rawUTF16AtXMLBoundaryEQ': True, 'unchangedChildModelAndEQImportReferences': True,
160 'persistentLispImagesTouched': False,
161 'fullOfflineNixRebuild': 'NOT-RUN; existing verified Store sources and unchanged local patches, fresh offline ASDF loads.'}
162 (output / 'execution.json').write_text(json.dumps(evidence, indent=2) + '\n')
163 print(json.dumps({k: evidence[k] for k in ['runtime', 'retainedLispChecks', 'newXTMIntegrationChecks', 'ctmStatuses', 'xtmCorpusStatuses']}, indent=2))
164
165
166 if __name__ == '__main__':
167 main()