]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21d.py
Implement XTM occurrence reification with native conformance evidence
[topic-maps.git] / scripts / validate-slice-21d.py
1 #!/usr/bin/env python3
2 """Validate basic Name originals through the existing XTM harness.
3
4 Use a fresh temporary output directory. No alternate parser/model/classifier is
5 introduced; all results come from ASDF and the existing foundation Case Records.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21d'
23 BASE = '0eed7189a0cdd535c980c2160ae2065de21f5d93'
24 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
25 ALLOWED_TRACKED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-negative-originals.lisp', 'tests/ctm-xtm-mergemap.lisp', 'topic-maps.asd'}
26 EXCLUDED = ['reifier-collision.xtm', 'role-duplicate-reified.xtm', 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
27
28
29 def read(path):
30 return json.loads(path.read_text())
31
32
33 def save(path, value):
34 path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
35
36
37 def sha(path):
38 return hashlib.sha256(path.read_bytes()).hexdigest()
39
40
41 def command(args):
42 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
43
44
45 def prepare_pinned(target):
46 # Reuse the existing local Store verifier and patcher, without a Nix daemon
47 # evaluation or dependency download. The historical driver stays unchanged.
48 spec = importlib.util.spec_from_file_location('slice21b', ROOT / 'scripts/validate-slice-21b.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare_pinned(target)
52
53
54 def qualify_name_scope(value):
55 if isinstance(value, dict):
56 return {k: qualify_name_scope(v) for k, v in value.items()}
57 if isinstance(value, list):
58 return [qualify_name_scope(v) for v in value]
59 if isinstance(value, str) and (value == 'unsupported-name' or value.startswith('unsupported-name:')):
60 return value.replace('unsupported-name', 'unsupported-name-scope', 1)
61 return value
62
63
64 def main():
65 parser = argparse.ArgumentParser(description=__doc__)
66 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
67 parser.add_argument('--output', type=Path)
68 parser.add_argument('--prepare', type=Path)
69 args = parser.parse_args()
70 if args.prepare:
71 print(json.dumps(prepare_pinned(args.prepare)))
72 return
73 runtime = shutil.which(args.sbcl)
74 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
75 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
76 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21d-'))
77 assert not output.is_relative_to(ROOT), 'Choose a temporary output directory outside the repository.'
78 output.mkdir(parents=True, exist_ok=True)
79 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
80 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
81 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED_TRACKED
82 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED_TRACKED}
83 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
84 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED_TRACKED), 'tests/xtm-names.lisp',
85 'scripts/validate-slice-21d.py', 'reports/slice-21d/candidate-inventory.json', 'reports/slice-21d/failure-controls.lisp',
86 'reports/slice-21d/diagnostics.lisp', 'reports/slice-21d/before-diagnostics.json']}
87 candidates = read(EVIDENCE / 'candidate-inventory.json')['candidates']
88 ids = [c['id'] for c in candidates]
89 assert len(ids) == len(set(ids)) == 10
90 expected_names = ['name', 'name-unicode', 'name-escaping', 'name-duplicate', 'name-duplicate-merge',
91 'merge-itemid-with-names', 'name-type', 'name-type-before', 'name-type-after', 'name-type-after-bad']
92 assert ids == ['xtm2/in/' + n + '.xtm' for n in expected_names]
93 prior = ROOT / 'reports/slice-21c'
94 previous_xtm = read(prior / 'xtm.json')
95 assert previous_xtm['statuses'] == {'pass': 33, 'fail': 0, 'unsupported': 0, 'not-run': 0}
96 assert previous_xtm['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 4}
97 ctm_statuses = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
98 assert read(prior / 'ctm.validation.json')['statuses'] == ctm_statuses
99 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
100 runs = []
101
102 def run(label, invocation, env, expected_exit=0):
103 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
104 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
105 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
106 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
107 (output / (label + '.log')).write_bytes(result.stdout)
108 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
109 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
110 runs.append(entry)
111 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
112 return result.stdout.decode()
113
114 retained = output / 'retained'
115 retained.mkdir()
116 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
117 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
118 with tempfile.TemporaryDirectory(prefix='topic-maps-21d-sources-') as temp_name:
119 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
120 save(output / 'source-preparation.json', provenance)
121 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
122 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
123 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
124 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
125 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
126 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
127 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
128 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
129 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
130 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
131 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
132 xtm = read(output / 'xtm.json')
133 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
134 assert labels == xtm['checks'] and len(labels) == len(set(labels))
135 assert {x.replace('unsupported-name:', 'unsupported-name-scope:') for x in previous_xtm['checks']} <= set(labels)
136 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 36 original CXTM PASS; 7 negative originals PASS' in xtm_log
137 assert xtm['statuses'] == {'pass': 43, 'fail': 0, 'unsupported': 0, 'not-run': 0}
138 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 77, 'negativeUnregisteredUnexecuted': 4}
139 assert xtm['cases'][:33] == previous_xtm['cases']
140 assert xtm['actualAndExpected'] == previous_xtm['actualAndExpected']
141 assert xtm['negativeOriginals'] == previous_xtm['negativeOriginals']
142 assert xtm['helperSource'] == previous_xtm['helperSource']
143 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 43
144 assert [c['id'] for c in xtm['cases'][33:]] == ids
145 assert len(xtm['nameOriginals']) == 10 and len(xtm['nameControls']) == 50
146 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][33:])
147 for old, current in zip(previous_xtm['negativeControls'], xtm['negativeControls']):
148 if old['label'] != 'unsupported-is-not-pass':
149 assert old == current
150 continue
151 assert old['label'] == current['label'] and old['status'] == current['status'] == 'unsupported'
152 a, b = old['condition'], current['condition']
153 assert a['class'] == b['class'] == 'XTM-UNSUPPORTED' and a['code'] == b['code'] == 'unsupported-construct'
154 assert a['detail'] == 'XTM name is not implemented in this subset.'
155 assert b['detail'] == 'XTM scope is not implemented in this subset.'
156 assert b['column'] == a['column'] + 7
157 assert {k: v for k, v in a.items() if k not in ['detail', 'column', 'message']} == {k: v for k, v in b.items() if k not in ['detail', 'column', 'message']}
158 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
159 diag_env = dict(env, TOPIC_MAPS_21D_ROOT=str(ROOT), TOPIC_MAPS_21D_DIAGNOSTICS=str(output / 'after-diagnostics.json'))
160 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')], diag_env)
161 fault_controls = []
162 families = {'positive': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'), ('tmdm', 'execution-error'), ('unsupported', 'unsupported')],
163 'negative': [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')],
164 'name': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]}
165 for family, modes in families.items():
166 for mode, outcome in modes:
167 label = family + '-control-' + mode
168 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')),
169 TOPIC_MAPS_21D_FAMILY=family, TOPIC_MAPS_21D_FAULT=mode)
170 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')], control_env, expected_exit=1)
171 result = read(output / (label + '.json'))
172 retained_count = {'positive': 0, 'negative': 26, 'name': 33}[family]
173 assert result['statuses']['pass'] == retained_count
174 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
175 assert result['cases'][:retained_count] == previous_xtm['cases'][:retained_count]
176 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
177 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count,
178 'faultCountedAsPass': False})
179
180 verified = []
181 for old in sorted((prior / 'retained').glob('*.json')):
182 assert qualify_name_scope(read(old)) == read(retained / old.name), str(old)
183 verified.append(str(old.relative_to(ROOT)))
184 for suffix in ['validation', 'cxtm', 'encoding']:
185 old = prior / f'ctm.{suffix}.json'
186 assert qualify_name_scope(read(old)) == read(output / f'ctm.{suffix}.json'), str(old)
187 verified.append(str(old.relative_to(ROOT)))
188 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
189 assert sum(totals.values()) == 12100
190 stages = read(prior / 'execution.json')['stageChecks']
191 for marker, count in dict(totals, **stages).items():
192 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
193 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
194 assert read(output / 'ctm.validation.json')['statuses'] == ctm_statuses
195 comparisons = []
196 (output / 'actual').mkdir()
197 for old, result in zip(candidates, xtm['nameOriginals']):
198 assert old['id'] == result['id'] and result['condition'] is None and result['status'] == 'pass'
199 source = ROOT / 'vendor/cxtm' / old['id']
200 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
201 raw = source.read_bytes()
202 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
203 assert sha(source) == old['sourceSHA256'] and len(raw) == result['sourceBytes'] == old['sourceBytes']
204 actual = result['actualCXTM'].encode('utf-8')
205 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
206 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == old['baselineBytes']
207 assert sha(baseline) == old['baselineSHA256'] and result['firstDifferentByte'] is None
208 assert result['model']['canonicalNameParentTypeReferences'] and result['documentIRI'] == old['documentIRI']
209 (output / 'actual' / baseline.name).write_bytes(actual)
210 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
211 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
212 before_diag = read(EVIDENCE / 'before-diagnostics.json')
213 after_diag = read(output / 'after-diagnostics.json')
214 assert len(before_diag) == len(after_diag) == 124
215 changes = []
216 for a, b in zip(before_diag, after_diag):
217 assert a['id'] == b['id'] and a['documentIRI'] == b['documentIRI']
218 if a == b:
219 continue
220 assert a['condition']['class'] == 'XTM-UNSUPPORTED'
221 assert a['condition']['detail'] == 'XTM name is not implemented in this subset.'
222 if b['status'] == 'deserialized':
223 assert b['id'] in ids and b['byteExact']
224 else:
225 assert b['condition']['class'] == 'XTM-UNSUPPORTED'
226 assert b['condition']['code'] in ['unsupported-construct', 'reification']
227 changes.append({'id': a['id'], 'before': a, 'after': b,
228 'warrant': 'The formerly unsupported Name is now processed; any remaining excluded Name feature stays Unsupported.'})
229 assert {b['id'] for a, b in zip(before_diag, after_diag) if a['status'] != 'deserialized' and b['status'] == 'deserialized'} == set(ids)
230 save(output / 'diagnostic-changes.json', {'kind': 'read-only-diagnostic-observations-no-additional-registration', 'changes': changes,
231 'unchangedOriginalObservations': 124 - len(changes), 'newByteExactOriginals': ids})
232 for positive in xtm['actualAndExpected']:
233 p = ROOT / 'vendor/cxtm/xtm2/baseline' / (Path(positive['id']).name + '.cxtm')
234 assert positive['actual'].encode('utf-8') == positive['expected'].encode('utf-8') == p.read_bytes()
235 pending_negative = sorted('xtm2/invalid/' + p.name for p in (ROOT / 'vendor/cxtm/xtm2/invalid').glob('*.xtm') if 'xtm2/invalid/' + p.name not in {c['id'] for c in xtm['negativeOriginals']})
236 assert pending_negative == ['xtm2/invalid/' + name for name in EXCLUDED]
237 assert not set(pending_negative) & {c['id'] for c in xtm['cases']}
238 lock = read(ROOT / 'references/corpus-lock.json')
239 for entry in lock['files']:
240 p = ROOT / 'vendor/cxtm' / entry['path']
241 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
242 for snapshot in [protected, historical, sources]:
243 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
244 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED_TRACKED
245 save(output / 'name-originals.json', {'kind': 'executed-basic-name-originals', 'cases': comparisons,
246 'excludedNegativeOriginals': pending_negative})
247 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
248 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
249 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
250 save(output / 'execution.json', {'kind': 'xtm-basic-name-registration-validation', 'baseCommit': BASE,
251 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
252 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs, 'sourcePreparation': 'source-preparation.json',
253 'retainedLispChecks': 12100, 'retainedChecks': totals, 'stageChecks': stages, 'xmlQualificationChecks': 146,
254 'standaloneXTMChecks': len(labels), 'retainedXTMCheckNamesPreserved': len(set(previous_xtm['checks']) & set(labels)),
255 'retainedXTMChecksQualified': len(previous_xtm['checks']),
256 'renamedXTMChecks': [{'before': x, 'after': x.replace('unsupported-name:', 'unsupported-name-scope:')}
257 for x in previous_xtm['checks'] if x.startswith('unsupported-name:')],
258 'retainedOriginals': 33, 'retainedOriginalResultsUnchanged': True, 'newNameOriginals': 10,
259 'nameCounterprobes': len(xtm['nameControls']), 'nameCounterprobeKinds': {k: sum(c['kind'] == k for c in xtm['nameControls']) for k in sorted({c['kind'] for c in xtm['nameControls']})},
260 'faultControls': len(fault_controls), 'diagnosticOriginalObservations': 124, 'nameRelatedDiagnosticChanges': len(changes),
261 'xtmStatusesBefore': {'pass': 33, 'fail': 0, 'unsupported': 0, 'not-run': 91}, 'xtmStatusesAfter': statuses,
262 'xtmStatusChanges': [{'id': p, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for p in ids],
263 'ctmStatusesBefore': ctm_statuses, 'ctmStatusesAfter': ctm_statuses, 'ctmStatusChanges': [],
264 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
265 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsQualifiedAgainstSlice21C': verified, 'retainedReportQualification': 'Only unsupported-name -> unsupported-name-scope fixture/check labels change.',
266 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected),
267 'pinnedCorpusFilesVerified': len(lock['files']), 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True,
268 'productionChangesLimitedToXTMReader': True, 'TMDM_CTM_CXTM_XML_Unchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
269 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
270 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
271 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': ctm_statuses, 'checks': len(labels), 'output': str(output)}))
272
273
274 if __name__ == '__main__':
275 main()