]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21g.py
perf(tmdm): index association normalization merge candidates
[topic-maps.git] / scripts / validate-slice-21g.py
1 #!/usr/bin/env python3
2 """Validate textual XTM Variants through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21g'
23 PRIOR = ROOT / 'reports/slice-21f'
24 BASE = 'a8d514d5a3a6026bc5440c67ab666630783398b3'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-names.lisp',
27 'tests/xtm-negative-originals.lisp', 'tests/ctm-xtm-mergemap.lisp', 'tests/xtm-name-scopes.lisp',
28 'tests/xtm-name-item-identities.lisp', 'topic-maps.asd'}
29 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
30 NEGATIVE = 'xtm2/invalid/variant-missing-scope-duplicate.xtm'
31 EXCLUDED = ['reifier-collision.xtm', 'role-duplicate-reified.xtm', 'topicref-no-fragment-id.xtm']
32
33
34 def read(p):
35 return json.loads(p.read_text())
36
37
38 def save(p, v):
39 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
40
41
42 def sha(p):
43 return hashlib.sha256(p.read_bytes()).hexdigest()
44
45
46 def command(args):
47 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
48
49
50 def prepare_pinned(target):
51 spec = importlib.util.spec_from_file_location('slice21f', ROOT / 'scripts/validate-slice-21f.py')
52 module = importlib.util.module_from_spec(spec)
53 spec.loader.exec_module(module)
54 return module.prepare_pinned(target)
55
56
57 def qualify_label(v):
58 if isinstance(v, dict):
59 return {k: qualify_label(x) for k, x in v.items()}
60 if isinstance(v, list):
61 return [qualify_label(x) for x in v]
62 if isinstance(v, str):
63 if v == 'unsupported-name-variant' or v.startswith('unsupported-name-variant:'):
64 return v.replace('unsupported-name-variant', 'unsupported-name-variant-resourceRef', 1)
65 if v.startswith('name-syntax:unsupported-variant:'):
66 return v.replace('unsupported-variant:', 'unsupported-variant-resourceRef:', 1)
67 return v.replace('variant-stays-unsupported', 'variant-resourceRef-stays-unsupported')
68 return v
69
70
71 def main():
72 parser = argparse.ArgumentParser(description=__doc__)
73 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
74 parser.add_argument('--output', type=Path)
75 parser.add_argument('--prepare', type=Path)
76 args = parser.parse_args()
77 if args.prepare:
78 print(json.dumps(prepare_pinned(args.prepare)))
79 return
80 runtime = shutil.which(args.sbcl)
81 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
82 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
83 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21g-'))
84 assert not output.is_relative_to(ROOT)
85 output.mkdir(parents=True, exist_ok=True)
86 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
87 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
88 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
89 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
90 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
91 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'tests/xtm-variants.lisp', 'scripts/validate-slice-21g.py',
92 *['reports/slice-21g/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp', 'diagnostics.lisp',
93 'before-diagnostics.json', 'baseline-xtm.json']]]}
94 previous = read(PRIOR / 'xtm.json')
95 assert read(EVIDENCE / 'baseline-xtm.json') == previous
96 assert previous['statuses'] == {'pass': 50, 'fail': 0, 'unsupported': 0, 'not-run': 0}
97 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 70, 'negativeUnregisteredUnexecuted': 4}
98 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
99 candidates = read(EVIDENCE / 'candidate-inventory.json')['candidates']
100 positives = [c for c in candidates if c['category'] == 'valid']
101 ids = [c['id'] for c in positives]
102 assert ids == ['xtm2/in/' + n + '.xtm' for n in ['variant', 'variant-duplicate', 'variant-scope-duplicate',
103 'variant-scope-multiple', 'variant-inherit']]
104 assert [c['id'] for c in candidates if c['category'] == 'invalid'] == [NEGATIVE]
105 assert not set(ids + [NEGATIVE]) & {c['id'] for c in previous['cases']}
106 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
107 runs = []
108
109 def run(label, invocation, env, expected_exit=0):
110 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
111 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
112 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
113 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
114 (output / (label + '.log')).write_bytes(result.stdout)
115 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
116 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
117 runs.append(entry)
118 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
119 return result.stdout.decode()
120
121 retained = output / 'retained'
122 retained.mkdir()
123 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
124 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
125 with tempfile.TemporaryDirectory(prefix='topic-maps-21g-sources-') as temp_name:
126 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
127 save(output / 'source-preparation.json', provenance)
128 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
129 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
130 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
131 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
132 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
133 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
134 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
135 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
136 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
137 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
138 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
139 xtm = read(output / 'xtm.json')
140 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
141 assert labels == xtm['checks'] and len(labels) == len(set(labels))
142 assert {qualify_label(x) for x in previous['checks']} <= set(labels)
143 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 48 original CXTM PASS; 8 negative originals PASS' in xtm_log
144 assert xtm['statuses'] == {'pass': 56, 'fail': 0, 'unsupported': 0, 'not-run': 0}
145 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 65, 'negativeUnregisteredUnexecuted': 3}
146 assert xtm['cases'][:50] == previous['cases']
147 for field in ['actualAndExpected', 'negativeOriginals', 'nameOriginals', 'scopeOriginals', 'nameItemIdentityOriginals', 'helperSource']:
148 assert xtm[field] == previous[field], field
149 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 56
150 assert [c['id'] for c in xtm['cases'][50:]] == ids + [NEGATIVE]
151 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][50:55])
152 assert xtm['cases'][55]['status'] == 'pass' and xtm['cases'][55]['actualOutcome'] == 'ctm-rejection'
153 assert len(xtm['variantOriginals']) == 5 and len(xtm['variantNegativeOriginals']) == 1
154 guard_changes = []
155 fixture_labels = {'negativeControls': 'unsupported-is-not-pass', 'nameControls': 'unsupported-variant',
156 'scopeControls': 'scope:variant-stays-unsupported', 'nameItemIdentityControls': 'iid:variant-stays-unsupported'}
157 for field, target in fixture_labels.items():
158 assert len(previous[field]) == len(xtm[field])
159 for old, current in zip(previous[field], xtm[field]):
160 if old['label'] != target:
161 assert old == current, (field, old['label'])
162 continue
163 assert qualify_label(old['label']) == current['label'] or (old['label'] == 'unsupported-variant' and current['label'] == 'unsupported-variant-resourceRef')
164 a, b = old['condition'], current['condition']
165 assert a['class'] == b['class'] == 'XTM-UNSUPPORTED' and a['code'] == b['code'] == 'unsupported-construct'
166 assert a['detail'] == 'XTM variant is not implemented in this subset.'
167 assert b['detail'] == 'XTM resourceRef is not implemented in this subset.' and b['column'] == a['column'] + 70
168 excluded = ['detail', 'message', 'column', 'sourceKey']
169 assert {k: v for k, v in a.items() if k not in excluded} == {k: v for k, v in b.items() if k not in excluded}
170 if 'sourceText' in old:
171 assert current['sourceText'] == old['sourceText'].replace('<resourceData>V</resourceData>', "<resourceRef href='urn:excluded'/>")
172 if a['sourceKey'].startswith('name-syntax:'):
173 assert b['sourceKey'] == 'name-syntax:' + current['label']
174 else:
175 assert a['sourceKey'] == b['sourceKey']
176 guard_changes.append({'field': field, 'before': old, 'after': current,
177 'reason': 'Valid excluded resourceRef replaces the now supported textual Variant fixture.'})
178 assert len(guard_changes) == 4 and len(xtm['variantControls']) == 53
179 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
180 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
181 dict(env, TOPIC_MAPS_21G_ROOT=str(ROOT), TOPIC_MAPS_21G_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
182 fault_controls = []
183 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
184 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
185 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
186 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
187 'variant-negative': negative_modes + [('tmdm', 'execution-error')]}
188 for family, modes in families.items():
189 for mode, outcome in modes:
190 label = family + '-control-' + mode
191 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
192 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21G_FAMILY=family, TOPIC_MAPS_21G_FAULT=mode), expected_exit=1)
193 result = read(output / (label + '.json'))
194 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55}[family]
195 assert result['statuses']['pass'] == retained_count
196 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
197 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
198 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
199 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
200
201 verified = []
202 for old in sorted((PRIOR / 'retained').glob('*.json')):
203 assert qualify_label(read(old)) == read(retained / old.name), str(old)
204 verified.append(str(old.relative_to(ROOT)))
205 for suffix in ['validation', 'cxtm', 'encoding']:
206 old = PRIOR / f'ctm.{suffix}.json'
207 assert qualify_label(read(old)) == read(output / f'ctm.{suffix}.json'), str(old)
208 verified.append(str(old.relative_to(ROOT)))
209 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
210 stages = read(PRIOR / 'execution.json')['stageChecks']
211 assert sum(totals.values()) == 12100
212 for marker, count in dict(totals, **stages).items():
213 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
214 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
215 assert read(output / 'ctm.validation.json')['statuses'] == CTM
216 comparisons = []
217 (output / 'actual').mkdir()
218 for old, result in zip(positives, xtm['variantOriginals']):
219 assert old['id'] == result['id'] and result['condition'] is None and result['status'] == 'pass'
220 source = ROOT / 'vendor/cxtm' / old['id']
221 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
222 raw = source.read_bytes()
223 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
224 assert sha(source) == old['sourceSHA256'] and len(raw) == result['sourceBytes'] == old['sourceBytes']
225 actual = result['actualCXTM'].encode('utf-8')
226 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
227 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == old['baselineBytes']
228 assert sha(baseline) == old['baselineSHA256'] and result['firstDifferentByte'] is None
229 assert result['model']['canonicalVariantReferences'] and result['documentIRI'] == old['documentIRI'] and result['sourceKey'] == result['id']
230 (output / 'actual' / baseline.name).write_bytes(actual)
231 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
232 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
233 negative = xtm['variantNegativeOriginals'][0]
234 raw = (ROOT / 'vendor/cxtm' / NEGATIVE).read_bytes()
235 assert raw == bytes.fromhex(negative['sourceHex']) == negative['sourceText'].encode('utf-8')
236 assert len(raw) == negative['sourceBytes'] == candidates[-1]['sourceBytes']
237 assert hashlib.sha256(raw).hexdigest() == candidates[-1]['sourceSHA256']
238 assert negative['intendedCauseReached'] and negative['status'] == 'pass'
239 condition = negative['condition']
240 assert condition['class'] == 'VARIANT-ERROR' and condition['code'] == 'invalid-variant-scope'
241 assert condition['detail'] == 'Variant scope must properly contain its Parent Name scope.'
242 assert condition['line'] is condition['column'] is condition['cause'] is None
243 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
244 assert len(before) == len(after) == 124
245 changes = []
246 for a, b in zip(before, after):
247 assert a['id'] == b['id'] and a['documentIRI'] == b['documentIRI']
248 if a == b:
249 continue
250 assert a['condition']['class'] == 'XTM-UNSUPPORTED' and a['condition']['detail'] == 'XTM variant is not implemented in this subset.'
251 if b['id'] in ids:
252 assert b['status'] == 'deserialized' and b['byteExact']
253 elif b['id'] == NEGATIVE:
254 assert b['condition'] == condition
255 elif b['status'] == 'deserialized':
256 # Diagnostic execution is separate from official Case Records.
257 assert b['id'] == 'xtm2/in/merge-itemid-with-variants.xtm' and b['byteExact']
258 assert b['id'] not in {c['id'] for c in xtm['cases']}
259 else:
260 assert b['condition']['class'] == 'XTM-UNSUPPORTED'
261 changes.append({'id': a['id'], 'before': a, 'after': b,
262 'registeredInThisSlice': b['id'] in ids + [NEGATIVE], 'warrant': 'Previously unsupported textual Variants now reach their intended processing; excluded features remain Unsupported.'})
263 pending_negative = sorted('xtm2/invalid/' + p.name for p in (ROOT / 'vendor/cxtm/xtm2/invalid').glob('*.xtm')
264 if 'xtm2/invalid/' + p.name not in {c['id'] for c in xtm['negativeOriginals'] + xtm['variantNegativeOriginals']})
265 assert pending_negative == ['xtm2/invalid/' + name for name in EXCLUDED]
266 assert not set(pending_negative) & {c['id'] for c in xtm['cases']}
267 lock = read(ROOT / 'references/corpus-lock.json')
268 for entry in lock['files']:
269 p = ROOT / 'vendor/cxtm' / entry['path']
270 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
271 for snapshot in [protected, historical, sources]:
272 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
273 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
274 save(output / 'variant-originals.json', {'kind': 'executed-textual-variant-originals', 'cases': comparisons,
275 'negativeOriginal': dict(negative, sourceSHA256=hashlib.sha256(raw).hexdigest()), 'excludedNegativeOriginals': pending_negative})
276 save(output / 'diagnostic-changes.json', {'kind': 'observations-no-additional-registration', 'changes': changes,
277 'unchangedOriginalObservations': 124 - len(changes), 'counterprobeFixtureChanges': guard_changes})
278 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
279 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
280 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
281 save(output / 'execution.json', {'kind': 'xtm-textual-variant-registration-validation', 'baseCommit': BASE,
282 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
283 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
284 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
285 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
286 'retainedXTMCheckNamesPreserved': len(set(previous['checks']) & set(labels)), 'retainedXTMChecksQualified': len(previous['checks']),
287 'renamedXTMChecks': [{'before': x, 'after': qualify_label(x)} for x in previous['checks'] if x != qualify_label(x)],
288 'retainedOriginals': 50, 'retainedOriginalResultsUnchanged': True, 'newPositiveVariantOriginals': 5, 'newNegativeVariantOriginals': 1,
289 'variantCounterprobes': len(xtm['variantControls']),
290 'variantCounterprobeKinds': {k: sum(c['kind'] == k for c in xtm['variantControls']) for k in sorted({c['kind'] for c in xtm['variantControls']})},
291 'faultControls': len(fault_controls), 'diagnosticOriginalObservations': 124, 'variantRelatedDiagnosticChanges': len(changes),
292 'additionalDiagnosticMatchesNotRegistered': [c['id'] for c in changes if c['after']['status'] == 'deserialized' and not c['registeredInThisSlice']],
293 'xtmStatusesBefore': {'pass': 50, 'fail': 0, 'unsupported': 0, 'not-run': 74}, 'xtmStatusesAfter': statuses,
294 'xtmStatusChanges': [{'id': p, 'before': 'not-run', 'after': 'pass'} for p in ids + [NEGATIVE]],
295 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
296 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
297 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsQualifiedAgainstSlice21F': verified,
298 'retainedReportQualification': 'Only unsupported-name-variant -> unsupported-name-variant-resourceRef fixture/check labels change.',
299 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
300 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionChangesLimitedToXTMReader': True,
301 'TMDM_CTM_CXTM_XML_Unchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
302 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
303 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
304 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
305
306
307 if __name__ == '__main__':
308 main()