]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21b.py
Support XTM item identity on occurrences associations and roles
[topic-maps.git] / scripts / validate-slice-21b.py
1 #!/usr/bin/env python3
2 """Execute the existing ASDF harness and offline regressions; retain byte evidence.
3
4 No alternate deserializer, model builder, or conformance classifier lives here.
5 Reports are produced by the existing Lisp harness. Use a new temporary output
6 folder; copy accepted evidence into version control only after this exits zero.
7 """
8 import argparse
9 import collections
10 import datetime
11 import hashlib
12 import importlib.util
13 import json
14 import os
15 from pathlib import Path
16 import re
17 import shutil
18 import subprocess
19 import sys
20 import tempfile
21
22 sys.dont_write_bytecode = True
23 ROOT = Path(__file__).resolve().parents[1]
24 EVIDENCE = ROOT / 'reports/slice-21b'
25 BASE = '1a2808b2d123184b2494f93b19738a31fc27fd52'
26 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
27 ALLOWED_TRACKED = {'tests/xtm-identity.lisp', 'topic-maps.asd'}
28
29
30 def read(path):
31 return json.loads(path.read_text())
32
33
34 def save(path, value):
35 path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
36
37
38 def sha(path):
39 return hashlib.sha256(path.read_bytes()).hexdigest()
40
41
42 def command(args):
43 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
44
45
46 def prepare_pinned(target):
47 prior = read(ROOT / 'reports/slice-20h/source-preparation.json')
48 spec = importlib.util.spec_from_file_location('prepare_cxml', ROOT / 'scripts/prepare-cxml.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare({'manifest': prior['sources'],
52 'localPatches': [{'package': p['package'], 'sha256': p['sha256']} for p in prior['patches']]}, target)
53
54
55 def main():
56 parser = argparse.ArgumentParser(description=__doc__)
57 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
58 parser.add_argument('--output', type=Path)
59 parser.add_argument('--prepare', type=Path)
60 args = parser.parse_args()
61 if args.prepare:
62 print(json.dumps(prepare_pinned(args.prepare)))
63 return
64 runtime = shutil.which(args.sbcl)
65 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
66 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
67 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21b-'))
68 assert not output.is_relative_to(ROOT), 'Run tests with repository writes denied; choose a temporary output directory.'
69 output.mkdir(parents=True, exist_ok=True)
70 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
71 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
72 changed = set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked)
73 assert changed <= ALLOWED_TRACKED
74 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED_TRACKED}
75 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
76 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED_TRACKED), 'scripts/validate-slice-21b.py',
77 'reports/slice-21b/candidate-inventory.json', 'reports/slice-21b/failure-controls.lisp']}
78 candidates = read(EVIDENCE / 'candidate-inventory.json')['candidates']
79 assert len(candidates) == len({c['id'] for c in candidates}) == 20
80 assert collections.Counter(c['group'] for c in candidates) == {
81 'item-identifier': 4, 'merging': 4, 'subject-identifier': 7, 'subject-locator': 5}
82 prior = ROOT / 'reports/slice-20j'
83 baseline = read(EVIDENCE / 'baseline-xtm.json')
84 assert baseline == read(prior / 'retained/xtm.json')
85 assert baseline['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
86 assert baseline['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
87 expected_ctm = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
88 assert read(prior / 'current.validation.json')['statuses'] == expected_ctm
89 assert not {c['id'] for c in baseline['cases']} & {c['id'] for c in candidates}
90 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
91 runs = []
92
93 def run(label, invocation, env, expected_exit=0):
94 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
95 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
96 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
97 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
98 (output / (label + '.log')).write_bytes(result.stdout)
99 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
100 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
101 runs.append(entry)
102 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
103 return result.stdout.decode()
104
105 retained = output / 'retained'
106 retained.mkdir()
107 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
108 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
109 with tempfile.TemporaryDirectory(prefix='topic-maps-21b-sources-') as temp_name:
110 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
111 save(output / 'source-preparation.json', provenance)
112 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
113 TOPIC_MAPS_REPORT=str(retained / 'core.json'), TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'),
114 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
115 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
116 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
117 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
118 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
119 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
120 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
121 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
122 labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
123 assert len(labels) == len(set(labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
124 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
125 xtm = read(output / 'xtm.json')
126 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
127 assert labels == xtm['checks'] and len(labels) == len(set(labels))
128 assert set(baseline['checks']) <= set(labels), 'A retained XTM check disappeared'
129 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 26 original CXTM PASS' in xtm_log
130 assert xtm['statuses'] == {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 0}
131 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 11}
132 assert xtm['cases'][:6] == baseline['cases']
133 assert len({c['id'] for c in xtm['cases']}) == len(xtm['cases']) == 26
134 assert [c['id'] for c in xtm['cases'][6:]] == [c['id'] for c in candidates]
135 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'])
136 assert xtm['helperSource'] == baseline['helperSource']
137 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
138 controls = []
139 for mode, outcome, condition_class in [('mismatch', 'cxtm-mismatch', None),
140 ('xml', 'execution-error', 'XTM-XML-ERROR'), ('xtm', 'execution-error', 'XTM-REJECTION'),
141 ('tmdm', 'execution-error', 'OCCURRENCE-IDENTIFIER-ERROR'), ('unsupported', 'unsupported', 'XTM-UNSUPPORTED')]:
142 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / ('control-' + mode + '.json')), TOPIC_MAPS_21B_FAULT=mode)
143 run('control-' + mode, prefix + [str(EVIDENCE / 'failure-controls.lisp')], control_env, expected_exit=1)
144 report = read(output / ('control-' + mode + '.json'))
145 assert report['statuses']['pass'] == 0 and len(report['cases']) == 1
146 assert report['cases'][0]['actualOutcome'] == outcome
147 condition = report['actualAndExpected'][0]['condition']
148 assert (condition['class'] if condition else None) == condition_class
149 controls.append({'mode': mode, 'expectedASDFFailureObserved': True, 'actualOutcome': outcome,
150 'condition': condition, 'countedAsOriginalPass': False})
151
152 verified = []
153 for old in sorted((prior / 'retained').glob('*.json')):
154 if old.name == 'xtm.json':
155 continue
156 assert read(old) == read(retained / old.name), str(old)
157 verified.append(str(old.relative_to(ROOT)))
158 for suffix in ['validation', 'cxtm', 'encoding']:
159 old = prior / f'current.{suffix}.json'
160 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
161 verified.append(str(old.relative_to(ROOT)))
162 assert read(output / 'ctm.validation.json')['statuses'] == expected_ctm
163 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
164 assert sum(totals.values()) == 12100
165 for marker, count in totals.items():
166 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
167 stages = {'XTM-MERGEMAP': 217, 'REGISTERED-ORIGINALS': 286, 'REGISTERED-NEGATIVES': 25,
168 'PROLOG': 171, 'EMPTY2': 13, 'CTM-INTEGER': 112, 'TRIPLE': 226, 'ENCODING': 722}
169 for marker, count in stages.items():
170 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
171 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
172 comparisons = []
173 (output / 'actual').mkdir()
174 for record, result in zip(xtm['cases'], xtm['actualAndExpected']):
175 assert record['id'] == result['id'] and result['condition'] is None
176 source = ROOT / 'vendor/cxtm' / record['id']
177 baseline_path = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
178 actual = result['actual'].encode('utf-8')
179 assert actual == result['expected'].encode('utf-8') == baseline_path.read_bytes()
180 assert result['byteLength'] == result['expectedByteLength'] == len(actual)
181 assert result['sourceByteLength'] == source.stat().st_size and result['firstDifferentByte'] is None
182 assert result['model']['canonicalParentsAndInverseReferences']
183 (output / 'actual' / baseline_path.name).write_bytes(actual)
184 comparison = {'id': record['id'], 'documentIRI': record['documentIRI'], 'sourceKey': record['id'],
185 'sourceBytes': source.stat().st_size, 'sourceSHA256': sha(source),
186 'baseline': str(baseline_path.relative_to(ROOT / 'vendor/cxtm')), 'baselineSHA256': sha(baseline_path),
187 'expectedBytes': len(actual), 'actualBytes': len(actual), 'actualSHA256': hashlib.sha256(actual).hexdigest(),
188 'firstDifferentByte': None, 'byteExact': True, 'condition': None, 'status': record['status'],
189 'model': result['model'], 'executedInASDFTestOp': True,
190 'checks': [c for c in labels if c.startswith(source.stem + ':')]}
191 if record['id'] in {c['id'] for c in candidates}:
192 candidate = next(c for c in candidates if c['id'] == record['id'])
193 assert candidate['sourceSHA256'] == comparison['sourceSHA256'] and candidate['sourceBytes'] == comparison['sourceBytes']
194 assert candidate['baselineSHA256'] == comparison['baselineSHA256'] and candidate['baselineBytes'] == len(actual)
195 assert candidate['documentIRI'] == comparison['documentIRI']
196 comparisons.append(comparison)
197 lock = read(ROOT / 'references/corpus-lock.json')
198 for entry in lock['files']:
199 p = ROOT / 'vendor/cxtm' / entry['path']
200 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
201 for snapshot in [protected, historical, sources]:
202 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
203 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED_TRACKED
204 save(output / 'comparisons.json', {'kind': 'actual-original-cxtm-comparisons', 'cases': comparisons})
205 save(output / 'failure-controls.json', {'kind': 'executor-fault-injection-controls-not-corpus-results', 'controls': controls})
206 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical,
207 'executionSourcesSHA256': sources})
208 save(output / 'execution.json', {'kind': 'xtm-original-registration-validation', 'baseCommit': BASE,
209 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
210 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
211 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
212 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
213 'retainedXTMCheckNamesPreserved': len(baseline['checks']), 'originalComparisons': 26, 'newOriginalComparisons': 20,
214 'xtmStatusesBefore': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
215 'xtmStatusesAfter': {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 98},
216 'xtmStatusChanges': [{'id': c['id'], 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for c in candidates],
217 'ctmStatusesBefore': expected_ctm, 'ctmStatusesAfter': expected_ctm, 'ctmStatusChanges': [],
218 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
219 'negativeXTMOriginalsRegistered': 0, 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
220 'faultInjectionControls': controls, 'allRetainedNonXTMReportsEqualToSlice20J': verified,
221 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected),
222 'pinnedCorpusFilesVerified': len(lock['files']), 'historicalReportsUnchanged': True,
223 'originalSourcesAndBaselinesUnchanged': True, 'languageImplementationUnchanged': True,
224 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
225 'fullOfflineNixRebuild': 'NOT-RUN; verified local pinned Store sources and existing qualified XML patches.',
226 'executionAuthorization': 'Regular require_escalated approval after managed sandbox symlink launch error; no symlink or configuration changes.',
227 'acceptancePassed': True})
228 print(json.dumps({'acceptancePassed': True, 'xtm': {'pass': 26, 'fail': 0, 'not-run': 98},
229 'ctm': expected_ctm, 'checks': len(labels), 'output': str(output)}))
230
231
232 if __name__ == '__main__':
233 main()