]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-cxml.py
Implement XTM occurrence reification with native conformance evidence
[topic-maps.git] / scripts / validate-cxml.py
1 #!/usr/bin/env python3
2 """Qualify pinned CXML sources in fresh, network-denied SBCL 2.5.4 processes."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = 'bc4fcf0f126bab6567b37a21fc8c8eab2944c152'
16 PIN = '50ab793786d9de88ee30ec4e4c24fb4236fc2674'
17 POLICY = '(version 1)(allow default)(deny network*)'
18 # Avoid a local Nix evaluator-cache dynamic-derivations error; no cache or
19 # global Nix setting is changed, and no derivation is built here.
20 NIX_EVAL = ['nix', 'eval', '--offline', '--option', 'eval-cache', 'false']
21
22
23 def command(args, **kwargs):
24 return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
25
26
27 def sha(path):
28 return hashlib.sha256(path.read_bytes()).hexdigest()
29
30
31 def main():
32 parser = argparse.ArgumentParser(description=__doc__)
33 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
34 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-19d0')
35 args = parser.parse_args()
36 runtime = shutil.which(args.sbcl)
37 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
38 output = args.output.resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
41 lock = json.loads((ROOT / 'flake.lock').read_text())
42 assert lock['nodes']['nixpkgs']['locked']['rev'] == PIN
43 system = command(NIX_EVAL + ['--impure', '--raw',
44 '--expr', 'builtins.currentSystem'])
45 # Evaluate the same source registry exported by the DevShell. This does
46 # not realize the (separate, not fully cached) default toolchain shell.
47 environment = json.loads(command(NIX_EVAL + [
48 '--json', '.#lib.cxmlEnvironment',
49 '--apply', 'f: f "' + system + '"']))
50 registry = command(NIX_EVAL + ['--raw',
51 '.#devShells.' + system + '.default.CL_SOURCE_REGISTRY'])
52 assert registry == environment['sourceRegistry']
53 expected = {'cxml': '20200610-git', 'closure-common': '20181018-git',
54 'babel': '20241012-git', 'alexandria': '20241012-git',
55 'trivial-features': '20230618-git',
56 'trivial-gray-streams': '20241012-git', 'puri': '20201016-git'}
57 assert {p['name']: p['version'] for p in environment['manifest']} == expected
58 for package in environment['manifest']:
59 assert Path(package['source']).is_dir(), 'Source not provisioned: ' + package['name']
60 actual = command(['nix', 'hash', 'path', '--type', 'sha256', '--base32', package['source']])
61 assert actual == package['sha256'], 'SOURCE HASH MISMATCH: ' + package['name']
62 package['verifiedNarSHA256'] = actual
63 protected = {str(p.relative_to(ROOT)): sha(p)
64 for directory in ['vendor/cxtm', 'references']
65 for p in (ROOT / directory).rglob('*') if p.is_file()}
66 files = ['flake.nix', 'flake.lock', 'tests/cxml-probe.lisp', 'scripts/validate-cxml.py']
67 source_hashes = {p: sha(ROOT / p) for p in files}
68 env = dict(os.environ, CL_SOURCE_REGISTRY=registry, PYTHONDONTWRITEBYTECODE='1')
69 runs = []
70 for number in [1, 2]:
71 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime,
72 '--noinform', '--no-sysinit', '--no-userinit', '--script',
73 str(ROOT / 'tests/cxml-probe.lisp')]
74 result = subprocess.run(invocation, cwd=ROOT, env=env,
75 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
76 log_name = 'cxml-run-' + str(number) + '.log'
77 (output / log_name).write_bytes(result.stdout)
78 log = result.stdout.decode('utf-8')
79 checks = re.findall(r'^CXML-CHECK-PASS (.+)$', log, re.M)
80 failed = re.findall(r'^CXML-CHECK-FAIL (.+)$', log, re.M)
81 assert checks and len(checks + failed) == len(set(checks + failed))
82 assert 'CXML-SUMMARY ' + str(len(checks)) + ' PASS ' + str(len(failed)) + ' FAIL' in log
83 assert (result.returncode == 0) == (not failed), 'Unclassified probe execution failure'
84 runs.append({'run': number, 'exitCode': result.returncode,
85 'freshProcess': True, 'network': 'denied by sandbox-exec',
86 'checks': checks, 'failedChecks': failed,
87 'command': invocation, 'log': log_name})
88 assert runs[0]['checks'] == runs[1]['checks']
89 assert runs[0]['failedChecks'] == runs[1]['failedChecks']
90 # Retain the established full regression command/harness unchanged.
91 with tempfile.TemporaryDirectory(prefix='topic-maps-19d0-regression-') as temp:
92 invocation = ['python3', str(ROOT / 'scripts/validate-slice-19c.py'),
93 '--sbcl', runtime, '--output', temp]
94 result = subprocess.run(invocation, cwd=ROOT, env=env,
95 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
96 (output / 'regression-driver.log').write_bytes(result.stdout)
97 assert result.returncode == 0, 'Retained regression failed'
98 temporary = Path(temp)
99 regression = json.loads((temporary / 'execution.json').read_text())
100 assert regression['importedStatuses'] == {
101 'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
102 assert regression['totalLispChecks'] == 12100
103 identical_reports = []
104 for previous in sorted((ROOT / 'reports/slice-19c').glob('*.json')):
105 if previous.name == 'execution.json':
106 continue
107 assert (temporary / previous.name).read_bytes() == previous.read_bytes(), previous.name
108 identical_reports.append(previous.name)
109 for name in ['execution.json', 'tests.log']:
110 shutil.copyfile(temporary / name, output / ('regression-' + name))
111 assert not command(['git', 'diff', BASE, '--', 'src', 'topic-maps.asd',
112 'vendor', 'references', 'flake.lock', 'scripts/test.sh'])
113 assert protected == {p: sha(ROOT / p) for p in protected}
114 assert source_hashes == {p: sha(ROOT / p) for p in source_hashes}
115 evidence = {
116 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
117 'startedAtUTC': started,
118 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
119 'runtime': 'SBCL 2.5.4', 'nixSystem': system, 'nixpkgsCommit': PIN,
120 'sourceEnvironment': environment, 'sourceSHA256': source_hashes,
121 'sourceHashesVerified': True, 'devShellRegistryEqualsTestRegistry': True,
122 'xmlRuns': runs,
123 'xmlChecksPerRun': len(runs[0]['checks']) + len(runs[0]['failedChecks']),
124 'xmlPassedChecksPerRun': len(runs[0]['checks']),
125 'xmlFailedChecksPerRun': len(runs[0]['failedChecks']),
126 'xmlQualification': 'FAIL' if runs[0]['failedChecks'] else 'PASS',
127 'encodingCounterexamples': {
128 'utf16leAstral': {'character': '\U0001f98a', 'bytesHex': '\U0001f98a'.encode('utf-16le').hex(),
129 'strictPythonRoundTrip': bytes.fromhex('3ed88add').decode('utf-16le') == '\U0001f98a'},
130 'invalidUtf8': {'bytesHex': 'c328', 'strictPythonRejection': False},
131 },
132 'securityPolicy': 'Reject every DOCTYPE; disallow internal subsets; '
133 'resolver signals external-resource-forbidden; no NIL fallback. '
134 'Catalog disabled and DTD cache fresh for each parse.',
135 'securityCounterprobe': 'NIL resolver reaches trapped CXML file-opening fallback; '
136 'rejecting resolver never reaches it.',
137 'sourceProvisioning': 'Only CXML and closure-common downloaded with authorized '
138 'hash-checked nix store prefetch-file; five sources already present.',
139 'defaultDevShellBuild': 'NOT-RUN: existing pinned toolchain not fully provisioned',
140 'fullOfflineRebuild': 'NOT-RUN: existing Nix SBCL and source artifacts used',
141 'regressionChecks': regression['regressionChecks'],
142 'totalLispChecks': regression['totalLispChecks'],
143 'importedStatuses': regression['importedStatuses'],
144 'byteIdenticalRetainedReports': identical_reports,
145 'newXTMContracts': 0, 'pinnedCorpusAndNormativeReferencesUnchanged': True,
146 'tmdmCtmCxtmAndExistingHarnessUnchanged': True,
147 }
148 try:
149 bytes.fromhex('c328').decode('utf-8')
150 except UnicodeDecodeError:
151 evidence['encodingCounterexamples']['invalidUtf8']['strictPythonRejection'] = True
152 assert evidence['encodingCounterexamples']['utf16leAstral']['strictPythonRoundTrip']
153 assert evidence['encodingCounterexamples']['invalidUtf8']['strictPythonRejection']
154 (output / 'execution.json').write_text(json.dumps(evidence, indent=2) + '\n')
155 print(json.dumps({k: evidence[k] for k in ['runtime', 'xmlChecksPerRun', 'xmlPassedChecksPerRun',
156 'xmlFailedChecksPerRun', 'xmlQualification', 'totalLispChecks', 'importedStatuses',
157 'defaultDevShellBuild', 'fullOfflineRebuild']}, indent=2))
158 # A successful regression/provisioning must never conceal failed parser
159 # qualification. Preserve all evidence first, then report failure to CI.
160 if evidence['xmlQualification'] != 'PASS':
161 raise SystemExit(1)
162
163
164 if __name__ == '__main__':
165 main()