]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-19d1.py
Support arbitrary absolute XTM resourceData datatypes (Slice 22D)
[topic-maps.git] / scripts / validate-slice-19d1.py
1 #!/usr/bin/env python3
2 """Fresh offline XTM identity checks plus the unchanged XML and CTM regressions."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '1691c8e0712e30666de2e667a5050eccb0086f71'
16 POLICY = '(version 1)(allow default)(deny network*)'
17
18
19 def command(args):
20 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
21
22
23 def sha(path):
24 return hashlib.sha256(path.read_bytes()).hexdigest()
25
26
27 def main():
28 parser = argparse.ArgumentParser(description=__doc__)
29 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
30 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-19d1')
31 args = parser.parse_args()
32 runtime = shutil.which(args.sbcl)
33 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
34 output = args.output.resolve()
35 output.mkdir(parents=True, exist_ok=True)
36 protected_paths = ['src/tmdm', 'src/ctm', 'src/cxtm', 'src/xml', 'patches', 'flake.nix',
37 'flake.lock', 'vendor', 'references', 'scripts/prepare-cxml.py',
38 'tests/cxml-probe.lisp', 'tests/cxml-encoding-regressions.lisp', 'scripts/test.sh']
39 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
40 protected = {str(p.relative_to(ROOT)): sha(p) for directory in ['vendor/cxtm', 'references', 'patches']
41 for p in (ROOT / directory).rglob('*') if p.is_file()}
42 sources = {str(p.relative_to(ROOT)): sha(p) for directory in ['src', 'tests', 'scripts', 'docs']
43 for p in (ROOT / directory).rglob('*') if p.is_file() and p.suffix in ['.lisp', '.py', '.sh', '.md']}
44 sources.update({p: sha(ROOT / p) for p in ['topic-maps.asd', 'README.md', 'flake.nix', 'flake.lock']})
45 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
46 # The XTM source must have exactly one XML parse entry, the qualified boundary.
47 xtm_source = '\n'.join(p.read_text() for p in sorted((ROOT / 'src/xtm').glob('*.lisp')))
48 assert xtm_source.count('(topic-maps.xml:parse-octets ') == 1
49 assert not re.search(r'\(cxml:parse[-\w]*\s', xtm_source, re.I)
50 runs = []
51 with tempfile.TemporaryDirectory(prefix='topic-maps-19d1-') as tmp:
52 tmp = Path(tmp)
53 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(tmp / 'sources')]
54 provenance = json.loads(command(preparation))
55 environment = dict(os.environ, CL_SOURCE_REGISTRY=provenance['sourceRegistry'], PYTHONDONTWRITEBYTECODE='1')
56 for label, script in [('xml', 'tests/cxml-probe.lisp'), ('xtm-1', 'scripts/fresh-xtm.lisp'),
57 ('xtm-2', 'scripts/fresh-xtm.lisp')]:
58 env = dict(environment)
59 if label.startswith('xtm'):
60 env['TOPIC_MAPS_XTM_REPORT'] = str(output / (label + '.json'))
61 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime,
62 '--noinform', '--no-sysinit', '--no-userinit', '--script', script]
63 context = {'label': label, 'command': invocation, 'freshProcess': True,
64 'network': 'denied by sandbox-exec',
65 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
66 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
67 stderr=subprocess.STDOUT, timeout=120)
68 (output / (label + '.log')).write_bytes(result.stdout)
69 context.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
70 assert result.returncode == 0, label + ' failed; see saved log'
71 prefix = 'CXML' if label == 'xml' else 'XTM'
72 passed = re.findall(r'^' + prefix + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
73 assert len(passed) == len(set(passed))
74 assert prefix + '-SUMMARY ' + str(len(passed)) + ' PASS 0 FAIL' in result.stdout.decode()
75 context['passedChecks'] = len(passed)
76 if label == 'xml':
77 assert len(passed) == 146
78 else:
79 report = json.loads((output / (label + '.json')).read_text())
80 assert report['checks'] == passed
81 assert report['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
82 assert report['helperSource']['status'] == 'pass'
83 assert len(report['cases']) == 6
84 for case in report['actualAndExpected']:
85 assert case['actual'].encode() == case['expected'].encode()
86 assert report['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
87 runs.append(context)
88 assert (output / 'xtm-1.json').read_bytes() == (output / 'xtm-2.json').read_bytes()
89 regression_dir = tmp / 'regression'
90 invocation = ['python3', 'scripts/validate-slice-19c.py', '--sbcl', runtime, '--output', str(regression_dir)]
91 result = subprocess.run(invocation, cwd=ROOT, env=environment, stdout=subprocess.PIPE,
92 stderr=subprocess.STDOUT, timeout=180)
93 (output / 'regression-driver.log').write_bytes(result.stdout)
94 assert result.returncode == 0, 'Retained regression failed'
95 regression = json.loads((regression_dir / 'execution.json').read_text())
96 assert regression['totalLispChecks'] == 12100
97 assert regression['importedStatuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
98 identical = []
99 for previous in sorted((ROOT / 'reports/slice-19c').glob('*.json')):
100 if previous.name != 'execution.json':
101 assert (regression_dir / previous.name).read_bytes() == previous.read_bytes(), previous.name
102 identical.append(previous.name)
103 for name in ['tests.log', 'execution.json']:
104 shutil.copyfile(regression_dir / name, output / ('regression-' + name))
105 assert protected == {p: sha(ROOT / p) for p in protected}
106 assert sources == {p: sha(ROOT / p) for p in sources}
107 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
108 evidence = {'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
109 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
110 'runtime': 'SBCL 2.5.4', 'newDependencyDownloads': 0, 'sourcePreparationCommand': preparation,
111 'qualifiedSourceProvenance': provenance, 'sourceSHA256': sources,
112 'protectedCorpusAndDecoderPatchesUnchanged': True, 'xmlBoundaryStaticAndRuntimeChecks': 'PASS',
113 'runs': runs, 'ctmStatuses': regression['importedStatuses'],
114 'retainedLispChecks': regression['totalLispChecks'], 'retainedChecks': regression['regressionChecks'],
115 'retainedRegressionCommand': invocation, 'retainedReportsByteIdentical': identical,
116 'xtmStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0},
117 'unexecutedOtherXTMInputs': {'positive': 107, 'negative': 11},
118 'directXTMHelperSource': 'PASS; 5 Topics, 1 Association, 2 Roles; not a CTM Mergemap PASS',
119 'deterministicXTMReports': True, 'persistentLispImagesTouched': False,
120 'fullOfflineNixRebuild': 'NOT-RUN; original Store sources, verified existing local patches and fresh offline ASDF loads.'}
121 (output / 'execution.json').write_text(json.dumps(evidence, indent=2) + '\n')
122 print(json.dumps({k: evidence[k] for k in ['runtime', 'runs', 'ctmStatuses', 'retainedLispChecks', 'xtmStatuses']}, indent=2))
123
124
125 if __name__ == '__main__':
126 main()