]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21i.py
perf(tmdm): index topic normalization merge candidates
[topic-maps.git] / scripts / validate-slice-21i.py
1 #!/usr/bin/env python3
2 """Validate XTM Variant ResourceRef originals through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21i'
23 PRIOR = ROOT / 'reports/slice-21h'
24 BASE = '034d2c14fbf6a58a062351e4b64680283d241d22'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-variants.lisp', 'tests/xtm-negative-originals.lisp',
27 'tests/ctm-xtm-mergemap.lisp', 'tests/xtm-names.lisp', 'tests/xtm-name-scopes.lisp', 'tests/xtm-name-item-identities.lisp', 'topic-maps.asd'}
28 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
29 TARGETS = ['xtm2/in/variant-resourceref.xtm', 'xtm2/in/variant-resourceref-relative.xtm']
30
31
32 def read(p):
33 return json.loads(p.read_text())
34
35
36 def save(p, v):
37 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
38
39
40 def sha(p):
41 return hashlib.sha256(p.read_bytes()).hexdigest()
42
43
44 def command(args):
45 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
46
47
48 def prepare_pinned(target):
49 spec = importlib.util.spec_from_file_location('slice21g', ROOT / 'scripts/validate-slice-21h.py')
50 module = importlib.util.module_from_spec(spec)
51 spec.loader.exec_module(module)
52 return module.prepare_pinned(target)
53
54
55 def main():
56 parser = argparse.ArgumentParser(description=__doc__)
57 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
58 parser.add_argument('--output', type=Path)
59 parser.add_argument('--prepare', type=Path)
60 args = parser.parse_args()
61 if args.prepare:
62 print(json.dumps(prepare_pinned(args.prepare)))
63 return
64 runtime = shutil.which(args.sbcl)
65 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
66 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
67 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21i-'))
68 assert not output.is_relative_to(ROOT)
69 output.mkdir(parents=True, exist_ok=True)
70 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
71 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
72 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
73 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
74 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
75 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'tests/xtm-variant-resourcerefs.lisp', 'scripts/validate-slice-21i.py',
76 *['reports/slice-21i/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp', 'diagnostics.lisp',
77 'before-diagnostics.json', 'baseline-xtm.json']]]}
78 previous = read(PRIOR / 'xtm.json')
79 assert read(EVIDENCE / 'baseline-xtm.json') == previous
80 assert previous['statuses'] == {'pass': 57, 'fail': 0, 'unsupported': 0, 'not-run': 0}
81 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 64, 'negativeUnregisteredUnexecuted': 3}
82 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
83 candidate = read(EVIDENCE / 'candidate-inventory.json')
84 assert [c['id'] for c in candidate['candidates']] == TARGETS
85 assert [c['sourceBytes'] for c in candidate['candidates']] == [324, 311]
86 assert [c['baselineBytes'] for c in candidate['candidates']] == [621, 608]
87 assert not set(TARGETS) & {c['id'] for c in previous['cases']}
88 assert read(EVIDENCE / 'before-diagnostics.json') == read(PRIOR / 'after-diagnostics.json')
89 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
90 runs = []
91
92 def run(label, invocation, env, expected_exit=0):
93 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
94 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
95 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
96 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
97 (output / (label + '.log')).write_bytes(result.stdout)
98 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
99 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
100 runs.append(entry)
101 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
102 return result.stdout.decode()
103
104 retained = output / 'retained'
105 retained.mkdir()
106 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
107 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
108 with tempfile.TemporaryDirectory(prefix='topic-maps-21i-sources-') as temp_name:
109 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
110 save(output / 'source-preparation.json', provenance)
111 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
112 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
113 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
114 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
115 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
116 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
117 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
118 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
119 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
120 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
121 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
122 xtm = read(output / 'xtm.json')
123 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
124 assert labels == xtm['checks'] and len(labels) == len(set(labels))
125 rename = {}
126 for label in previous['checks']:
127 if label.startswith('variant-syntax:resourceRef:'):
128 rename[label] = label.replace('resourceRef', 'resourceData-anyURI')
129 elif 'variant-resourceRef' in label:
130 rename[label] = label.replace('variant-resourceRef', 'variant-resourceData-datatype')
131 assert len(rename) == 12
132 retained_labels = [rename.get(label, label) for label in previous['checks']]
133 assert set(retained_labels) <= set(labels)
134 assert [label for label in labels if label in set(retained_labels)] == retained_labels
135 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 51 original CXTM PASS; 8 negative originals PASS' in xtm_log
136 assert xtm['statuses'] == {'pass': 59, 'fail': 0, 'unsupported': 0, 'not-run': 0}
137 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 62, 'negativeUnregisteredUnexecuted': 3}
138 assert xtm['cases'][:57] == previous['cases']
139 control_retargets = {
140 'negativeControls': ('unsupported-is-not-pass', 'unsupported-is-not-pass'),
141 'nameControls': ('unsupported-variant-resourceRef', 'unsupported-variant-resourceData-datatype'),
142 'scopeControls': ('scope:variant-resourceRef-stays-unsupported', 'scope:variant-resourceData-datatype-stays-unsupported'),
143 'nameItemIdentityControls': ('iid:variant-resourceRef-stays-unsupported', 'iid:variant-resourceData-datatype-stays-unsupported'),
144 'variantControls': ('resourceRef', 'resourceData-anyURI')}
145 for field, value in previous.items():
146 if field in control_retargets:
147 assert len(xtm[field]) == len(value)
148 changed = [(old, new) for old, new in zip(value, xtm[field]) if old != new]
149 assert len(changed) == 1, field
150 old, new = changed[0]
151 assert (old['label'], new['label']) == control_retargets[field]
152 assert new['condition']['class'] == 'XTM-UNSUPPORTED'
153 assert new['condition']['code'] == 'resource-data-datatype'
154 elif field not in ['scope', 'statuses', 'otherCorpusInputs', 'checks', 'cases']:
155 assert xtm[field] == value, field
156 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 59
157 assert [c['id'] for c in xtm['cases'][57:]] == TARGETS
158 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][57:])
159 assert len(xtm['variantResourceRefOriginals']) == 2
160 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
161 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
162 dict(env, TOPIC_MAPS_21I_ROOT=str(ROOT), TOPIC_MAPS_21I_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
163 fault_controls = []
164 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
165 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
166 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
167 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
168 'variant-negative': negative_modes + [('tmdm', 'execution-error')],
169 'merge-variant': positive_modes + [('xtm', 'execution-error')],
170 'resource-ref': positive_modes + [('xtm', 'execution-error')],
171 'resource-ref-relative': positive_modes + [('xtm', 'execution-error')]}
172 for family, modes in families.items():
173 for mode, outcome in modes:
174 label = family + '-control-' + mode
175 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
176 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21I_FAMILY=family, TOPIC_MAPS_21I_FAULT=mode), expected_exit=1)
177 result = read(output / (label + '.json'))
178 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55, 'merge-variant': 56, 'resource-ref': 57, 'resource-ref-relative': 58}[family]
179 assert result['statuses']['pass'] == retained_count
180 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
181 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
182 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
183 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
184
185 def retarget_mergemap(value):
186 if isinstance(value, dict):
187 changed = {k: retarget_mergemap(v) for k, v in value.items()}
188 if value.get('label') == 'unsupported-name-variant-resourceRef':
189 assert value['code'] == 'UNSUPPORTED-CONSTRUCT'
190 changed['code'] = 'RESOURCE-DATA-DATATYPE'
191 return changed
192 if isinstance(value, list):
193 return [retarget_mergemap(v) for v in value]
194 if isinstance(value, str) and (value == 'unsupported-name-variant-resourceRef' or value.startswith('unsupported-name-variant-resourceRef:')):
195 return value.replace('unsupported-name-variant-resourceRef', 'unsupported-name-variant-resourceData-datatype', 1)
196 return value
197
198 verified, retargeted_reports = [], []
199 for old in sorted((PRIOR / 'retained').glob('*.json')):
200 prior_value, actual_value = read(old), read(retained / old.name)
201 expected_value = retarget_mergemap(prior_value) if old.name.startswith('xtm-mergemap.') else prior_value
202 assert expected_value == actual_value, str(old)
203 if prior_value != actual_value:
204 retargeted_reports.append(str(old.relative_to(ROOT)))
205 verified.append(str(old.relative_to(ROOT)))
206 assert len(retargeted_reports) == 2
207 for suffix in ['validation', 'cxtm', 'encoding']:
208 old = PRIOR / f'ctm.{suffix}.json'
209 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
210 verified.append(str(old.relative_to(ROOT)))
211 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
212 stages = read(PRIOR / 'execution.json')['stageChecks']
213 assert sum(totals.values()) == 12100
214 for marker, count in dict(totals, **stages).items():
215 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
216 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
217 assert read(output / 'ctm.validation.json')['statuses'] == CTM
218 comparisons = []
219 (output / 'actual').mkdir()
220 for candidate_case, result in zip(candidate['candidates'], xtm['variantResourceRefOriginals']):
221 assert result['id'] == candidate_case['id'] and result['condition'] is None and result['status'] == 'pass'
222 source = ROOT / 'vendor/cxtm' / result['id']
223 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
224 raw, actual = source.read_bytes(), result['actualCXTM'].encode('utf-8')
225 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
226 assert sha(source) == candidate_case['sourceSHA256'] and len(raw) == result['sourceBytes'] == candidate_case['sourceBytes']
227 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
228 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == candidate_case['baselineBytes']
229 assert sha(baseline) == candidate_case['baselineSHA256'] and result['firstDifferentByte'] is None
230 assert result['documentIRI'] == candidate_case['documentIRI'] and result['sourceKey'] == result['id']
231 assert result['model']['canonicalVariantReferences']
232 assert len(result['model']['variants']) == 1
233 variant = result['model']['variants'][0]
234 assert variant['value'] == result['absoluteValue'] and variant['datatype'] == 'http://www.w3.org/2001/XMLSchema#anyURI'
235 assert result['absoluteValue'].startswith(('http:', 'https:'))
236 (output / 'actual' / baseline.name).write_bytes(actual)
237 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
238 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
239 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
240 assert len(before) == len(after) == 124
241 assert [c['id'] for c in before] == [c['id'] for c in after]
242 changes = [{'id': a['id'], 'before': b, 'after': a,
243 'reason': 'Variant resourceRef now resolves the resource locator and creates an anyURI Variant through add-variant.'}
244 for b, a in zip(before, after) if b != a]
245 assert {c['id'] for c in changes} == set(TARGETS)
246 for change in changes:
247 assert change['before']['condition']['class'] == 'XTM-UNSUPPORTED'
248 assert change['before']['condition']['code'] == 'unsupported-construct'
249 assert change['after']['condition'] is None and change['after']['byteExact']
250 comparison = next(c for c in comparisons if c['id'] == change['id'])
251 assert change['after']['actualCXTM'] == comparison['actualCXTM']
252 excluded = next(c for c in after if c['id'] == 'xtm2/in/variant-resourcedata-uri.xtm')
253 assert excluded['condition']['class'] == 'XTM-UNSUPPORTED' and excluded['condition']['code'] == 'resource-data-datatype'
254 save(output / 'diagnostic-changes.json', changes)
255 lock = read(ROOT / 'references/corpus-lock.json')
256 for entry in lock['files']:
257 p = ROOT / 'vendor/cxtm' / entry['path']
258 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
259 for snapshot in [protected, historical, sources]:
260 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
261 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
262 save(output / 'variant-resource-ref-originals.json', {'kind': 'executed-original-variant-resource-ref-registration', 'cases': comparisons})
263 save(output / 'diagnostics.json', {'kind': 'read-only-observations-no-extra-registration',
264 'originalObservations': 124, 'unchangedOriginalObservations': 124 - len(changes),
265 'diagnosticChanges': changes, 'registeredCases': TARGETS,
266 'outOfScopeResourceDataURIStillUnsupported': True})
267 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
268 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
269 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
270 save(output / 'execution.json', {'kind': 'xtm-variant-resource-ref-original-registration-validation', 'baseCommit': BASE,
271 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
272 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
273 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
274 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
275 'retainedXTMChecks': len(previous['checks']), 'retargetedUnsupportedCheckNames': rename,
276 'newXTMChecks': len(labels) - len(previous['checks']), 'retargetedUnsupportedControlRows': control_retargets,
277 'retainedOriginals': 57, 'retainedOriginalResultsUnchanged': True, 'newPositiveOriginals': 2,
278 'originalByteComparisons': [{'id': c['id'], 'actual': c['actualBytes'], 'baseline': c['expectedBytes'], 'firstDifferentByte': None} for c in comparisons], 'faultControls': len(fault_controls),
279 'diagnosticOriginalObservations': 124, 'diagnosticChangedOriginals': [c['id'] for c in changes],
280 'xtmStatusesBefore': {'pass': 57, 'fail': 0, 'unsupported': 0, 'not-run': 67}, 'xtmStatusesAfter': statuses,
281 'xtmStatusChanges': [{'id': id, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for id in TARGETS],
282 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
283 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
284 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsVerifiedAgainstSlice21H': verified,
285 'retargetedMergemapUnsupportedReports': retargeted_reports, 'unchangedNonXTMReports': len(verified) - len(retargeted_reports),
286 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
287 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionChangeLimitedToXTMReader': True, 'protectedSubsystemsUnchanged': ['TMDM', 'CTM', 'CXTM', 'XML'],
288 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
289 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
290 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
291 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
292
293
294 if __name__ == '__main__':
295 main()