]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21c.py
Support arbitrary absolute XTM resourceData datatypes (Slice 22D)
[topic-maps.git] / scripts / validate-slice-21c.py
1 #!/usr/bin/env python3
2 """Validate cause-qualified negative originals through the existing XTM harness.
3
4 Use a fresh temporary output directory. No alternate parser/model/classifier is
5 introduced; all results come from ASDF and the existing foundation Case Records.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21c'
23 BASE = 'd498716b1e30ca1ef2b5c124038b372851ccb731'
24 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
25 ALLOWED_TRACKED = {'tests/xtm-identity.lisp', 'topic-maps.asd'}
26 EXCLUDED = ['reifier-collision.xtm', 'role-duplicate-reified.xtm', 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
27
28
29 def read(path):
30 return json.loads(path.read_text())
31
32
33 def save(path, value):
34 path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
35
36
37 def sha(path):
38 return hashlib.sha256(path.read_bytes()).hexdigest()
39
40
41 def command(args):
42 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
43
44
45 def prepare_pinned(target):
46 # Reuse the existing local Store verifier and patcher, without a Nix daemon
47 # evaluation or dependency download. The historical driver stays unchanged.
48 spec = importlib.util.spec_from_file_location('slice21b', ROOT / 'scripts/validate-slice-21b.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare_pinned(target)
52
53
54 def main():
55 parser = argparse.ArgumentParser(description=__doc__)
56 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
57 parser.add_argument('--output', type=Path)
58 parser.add_argument('--prepare', type=Path)
59 args = parser.parse_args()
60 if args.prepare:
61 print(json.dumps(prepare_pinned(args.prepare)))
62 return
63 runtime = shutil.which(args.sbcl)
64 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
65 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
66 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21c-'))
67 assert not output.is_relative_to(ROOT), 'Choose a temporary output directory outside the repository.'
68 output.mkdir(parents=True, exist_ok=True)
69 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
70 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
71 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED_TRACKED
72 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED_TRACKED}
73 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
74 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED_TRACKED), 'tests/xtm-negative-originals.lisp',
75 'scripts/validate-slice-21c.py', 'reports/slice-21c/candidate-inventory.json', 'reports/slice-21c/failure-controls.lisp']}
76 candidates = read(EVIDENCE / 'candidate-inventory.json')['cases']
77 ids = [c['id'] for c in candidates]
78 assert len(ids) == len(set(ids)) == 7
79 assert ids == ['xtm2/invalid/' + name + '.xtm' for name in ['id-invalid', 'itemid-collision', 'no-version',
80 'reifier-elem-in-2.0', 'subjid-ref-in-2.0', 'subjloc-ref-in-2.0', 'topic-no-id']]
81 prior = ROOT / 'reports/slice-21b'
82 previous_xtm = read(prior / 'xtm.json')
83 assert previous_xtm['statuses'] == {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 0}
84 assert previous_xtm['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 11}
85 ctm_statuses = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
86 assert read(prior / 'ctm.validation.json')['statuses'] == ctm_statuses
87 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
88 runs = []
89
90 def run(label, invocation, env, expected_exit=0):
91 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
92 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
93 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
94 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
95 (output / (label + '.log')).write_bytes(result.stdout)
96 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
97 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
98 runs.append(entry)
99 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
100 return result.stdout.decode()
101
102 retained = output / 'retained'
103 retained.mkdir()
104 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
105 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
106 with tempfile.TemporaryDirectory(prefix='topic-maps-21c-sources-') as temp_name:
107 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
108 save(output / 'source-preparation.json', provenance)
109 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
110 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
111 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
112 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
113 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
114 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
115 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
116 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
117 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
118 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
119 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
120 xtm = read(output / 'xtm.json')
121 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
122 assert labels == xtm['checks'] and len(labels) == len(set(labels)) and set(previous_xtm['checks']) <= set(labels)
123 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 26 original CXTM PASS; 7 negative originals PASS' in xtm_log
124 assert xtm['statuses'] == {'pass': 33, 'fail': 0, 'unsupported': 0, 'not-run': 0}
125 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 4}
126 assert xtm['cases'][:26] == previous_xtm['cases']
127 assert xtm['actualAndExpected'] == previous_xtm['actualAndExpected']
128 assert xtm['helperSource'] == previous_xtm['helperSource']
129 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 33
130 assert [c['id'] for c in xtm['cases'][26:]] == ids
131 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'ctm-rejection' for c in xtm['cases'][26:])
132 assert len(xtm['negativeOriginals']) == 7 and len(xtm['negativeControls']) == 13
133 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
134 fault_controls = []
135 # Preserve all five previously established positive-executor guards.
136 for mode, outcome in [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'),
137 ('tmdm', 'execution-error'), ('unsupported', 'unsupported')]:
138 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / ('retained-control-' + mode + '.json')), TOPIC_MAPS_21B_FAULT=mode)
139 run('retained-control-' + mode, prefix + [str(prior / 'failure-controls.lisp')], control_env, expected_exit=1)
140 result = read(output / ('retained-control-' + mode + '.json'))
141 assert result['statuses']['pass'] == 0 and result['cases'][0]['actualOutcome'] == outcome
142 for mode, outcome, condition_class in [('accepted', 'ctm-acceptance', None), ('xml', 'execution-error', 'XTM-XML-ERROR'),
143 ('unsupported', 'unsupported', 'XTM-UNSUPPORTED'), ('wrong-cause', 'execution-error', 'XTM-REJECTION')]:
144 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / ('negative-control-' + mode + '.json')), TOPIC_MAPS_21C_FAULT=mode)
145 run('negative-control-' + mode, prefix + [str(EVIDENCE / 'failure-controls.lisp')], control_env, expected_exit=1)
146 result = read(output / ('negative-control-' + mode + '.json'))
147 assert result['statuses']['pass'] == 26 and result['cases'][:26] == previous_xtm['cases']
148 assert len(result['cases']) == 27 and result['cases'][-1]['actualOutcome'] == outcome
149 negative = result['negativeOriginals'][0]
150 assert not negative['intendedCauseReached'] and negative['status'] != 'pass'
151 condition = negative['condition']
152 assert (condition['class'] if condition else None) == condition_class
153 fault_controls.append({'mode': mode, 'actualOutcome': outcome, 'condition': condition,
154 'expectedASDFFailureObserved': True, 'positiveOriginalPassesPreserved': 26, 'negativeOriginalCountedAsPass': False})
155
156 verified = []
157 for old in sorted((prior / 'retained').glob('*.json')):
158 assert read(old) == read(retained / old.name), str(old)
159 verified.append(str(old.relative_to(ROOT)))
160 for suffix in ['validation', 'cxtm', 'encoding']:
161 old = prior / f'ctm.{suffix}.json'
162 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
163 verified.append(str(old.relative_to(ROOT)))
164 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
165 assert sum(totals.values()) == 12100
166 stages = read(prior / 'execution.json')['stageChecks']
167 for marker, count in dict(totals, **stages).items():
168 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
169 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
170 assert read(output / 'ctm.validation.json')['statuses'] == ctm_statuses
171 for old, result in zip(candidates, xtm['negativeOriginals']):
172 assert old['id'] == result['id'] and old['condition'] == result['condition']
173 source = ROOT / 'vendor/cxtm' / old['id']
174 raw = source.read_bytes()
175 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
176 assert sha(source) == old['sourceSHA256'] and len(raw) == result['sourceBytes'] == old['sourceBytes']
177 assert result['xmlWellFormed'] and result['intendedCauseReached'] and result['status'] == 'pass'
178 assert not result['unsupported'] and result['condition']['cause'] is None
179 result['sourceSHA256'] = sha(source)
180 for positive in xtm['actualAndExpected']:
181 p = ROOT / 'vendor/cxtm/xtm2/baseline' / (Path(positive['id']).name + '.cxtm')
182 assert positive['actual'].encode('utf-8') == positive['expected'].encode('utf-8') == p.read_bytes()
183 pending_negative = sorted('xtm2/invalid/' + p.name for p in (ROOT / 'vendor/cxtm/xtm2/invalid').glob('*.xtm') if 'xtm2/invalid/' + p.name not in ids)
184 assert pending_negative == ['xtm2/invalid/' + name for name in EXCLUDED]
185 assert not set(pending_negative) & {c['id'] for c in xtm['cases']}
186 lock = read(ROOT / 'references/corpus-lock.json')
187 for entry in lock['files']:
188 p = ROOT / 'vendor/cxtm' / entry['path']
189 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
190 for snapshot in [protected, historical, sources]:
191 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
192 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED_TRACKED
193 save(output / 'negative-originals.json', {'kind': 'executed-cause-qualified-negative-originals', 'cases': xtm['negativeOriginals'],
194 'excludedOriginals': [{'id': p, 'registeredStatus': 'not-run', 'executedInThisSlice': False} for p in pending_negative],
195 'outcomeVocabulary': 'ctm-rejection is the existing foundation negative-outcome token; actual deserialization is XTM/TMDM.'})
196 save(output / 'failure-controls.json', {'kind': 'negative-executor-fault-controls-not-corpus-results', 'controls': fault_controls})
197 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
198 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
199 save(output / 'execution.json', {'kind': 'xtm-negative-original-registration-validation', 'baseCommit': BASE,
200 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
201 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs, 'sourcePreparation': 'source-preparation.json',
202 'retainedLispChecks': 12100, 'retainedChecks': totals, 'stageChecks': stages, 'xmlQualificationChecks': 146,
203 'standaloneXTMChecks': len(labels), 'retainedXTMCheckNamesPreserved': len(previous_xtm['checks']),
204 'positiveOriginalComparisons': 26, 'positiveOriginalResultsUnchanged': True, 'newNegativeOriginals': 7,
205 'validNegativeCounterexamples': 7, 'negativeClassificationControls': 6, 'positiveFaultControlsRetained': 5, 'negativeFaultControls': 4,
206 'xtmStatusesBefore': {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 98}, 'xtmStatusesAfter': statuses,
207 'xtmStatusChanges': [{'id': p, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'ctm-rejection'} for p in ids],
208 'ctmStatusesBefore': ctm_statuses, 'ctmStatusesAfter': ctm_statuses, 'ctmStatusChanges': [],
209 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
210 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsEqualToSlice21B': verified,
211 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected),
212 'pinnedCorpusFilesVerified': len(lock['files']), 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True,
213 'productionCodeUnchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
214 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
215 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
216 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': ctm_statuses, 'checks': len(labels), 'output': str(output)}))
217
218
219 if __name__ == '__main__':
220 main()