]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21k.py
Document XTM normative coverage and remaining qualification gaps
[topic-maps.git] / scripts / validate-slice-21k.py
1 #!/usr/bin/env python3
2 """Validate XTM Variant Item Identity originals through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21k'
23 PRIOR = ROOT / 'reports/slice-21j'
24 BASE = '49e4f2efb3df426d8ce249bc9b714bbd81323525'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-variants.lisp', 'topic-maps.asd'}
27 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
28 TARGETS = ['xtm2/in/itemid-variant.xtm', 'xtm2/in/variant-duplicate-iid.xtm']
29
30
31 def read(p):
32 return json.loads(p.read_text())
33
34
35 def save(p, v):
36 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
37
38
39 def sha(p):
40 return hashlib.sha256(p.read_bytes()).hexdigest()
41
42
43 def command(args):
44 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
45
46
47 def prepare_pinned(target):
48 spec = importlib.util.spec_from_file_location('slice21g', ROOT / 'scripts/validate-slice-21j.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare_pinned(target)
52
53
54 def main():
55 parser = argparse.ArgumentParser(description=__doc__)
56 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
57 parser.add_argument('--output', type=Path)
58 parser.add_argument('--prepare', type=Path)
59 args = parser.parse_args()
60 if args.prepare:
61 print(json.dumps(prepare_pinned(args.prepare)))
62 return
63 runtime = shutil.which(args.sbcl)
64 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
65 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
66 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21k-'))
67 assert not output.is_relative_to(ROOT)
68 output.mkdir(parents=True, exist_ok=True)
69 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
70 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
71 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
72 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
73 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
74 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'tests/xtm-variant-item-identities.lisp', 'scripts/validate-slice-21k.py',
75 *['reports/slice-21k/' + p for p in ['candidate-inventory.json', 'api-preflight.lisp', 'failure-controls.lisp', 'diagnostics.lisp',
76 'before-diagnostics.json', 'baseline-xtm.json']]]}
77 previous = read(PRIOR / 'xtm.json')
78 assert read(EVIDENCE / 'baseline-xtm.json') == previous
79 assert previous['statuses'] == {'pass': 60, 'fail': 0, 'unsupported': 0, 'not-run': 0}
80 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 61, 'negativeUnregisteredUnexecuted': 3}
81 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
82 candidate = read(EVIDENCE / 'candidate-inventory.json')
83 assert [c['id'] for c in candidate['candidates']] == TARGETS
84 assert [c['sourceBytes'] for c in candidate['candidates']] == [351, 543]
85 assert [c['baselineBytes'] for c in candidate['candidates']] == [666, 666]
86 assert not set(TARGETS) & {c['id'] for c in previous['cases']}
87 assert read(EVIDENCE / 'before-diagnostics.json') == read(PRIOR / 'after-diagnostics.json')
88 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
89 runs = []
90
91 def run(label, invocation, env, expected_exit=0):
92 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
93 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
94 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
95 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
96 (output / (label + '.log')).write_bytes(result.stdout)
97 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
98 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
99 runs.append(entry)
100 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
101 return result.stdout.decode()
102
103 retained = output / 'retained'
104 retained.mkdir()
105 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
106 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
107 with tempfile.TemporaryDirectory(prefix='topic-maps-21k-sources-') as temp_name:
108 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
109 save(output / 'source-preparation.json', provenance)
110 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
111 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
112 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
113 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
114 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
115 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
116 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
117 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
118 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
119 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
120 api_probe = run('variant-api-preflight', prefix + [str(EVIDENCE / 'api-preflight.lisp')], env)
121 assert 'VARIANT-II-API-PREFLIGHT-PASS:' in api_probe
122 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
123 xtm = read(output / 'xtm.json')
124 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
125 assert labels == xtm['checks'] and len(labels) == len(set(labels))
126 rename = {label: label.replace('variant-syntax:itemIdentity:', 'variant-syntax:reified-itemIdentity:')
127 for label in previous['checks'] if label.startswith('variant-syntax:itemIdentity:')}
128 assert len(rename) == 2
129 retained_labels = [rename.get(label, label) for label in previous['checks']]
130 assert set(retained_labels) <= set(labels)
131 assert [label for label in labels if label in set(retained_labels)] == retained_labels
132 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 54 original CXTM PASS; 8 negative originals PASS' in xtm_log
133 assert xtm['statuses'] == {'pass': 62, 'fail': 0, 'unsupported': 0, 'not-run': 0}
134 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 59, 'negativeUnregisteredUnexecuted': 3}
135 assert xtm['cases'][:60] == previous['cases']
136 control_retargets = {'variantControls': [('itemIdentity', 'reified-itemIdentity')]}
137 for field, value in previous.items():
138 if field == 'variantControls':
139 assert len(xtm[field]) == len(value)
140 changed = [(old, new) for old, new in zip(value, xtm[field]) if old != new]
141 assert [(old['label'], new['label']) for old, new in changed] == control_retargets[field]
142 old, new = changed[0]
143 assert old['condition']['class'] == new['condition']['class'] == 'XTM-UNSUPPORTED'
144 assert old['condition']['code'] == 'unsupported-construct' and new['condition']['code'] == 'reification'
145 assert "reifier='#r'" in new['sourceText'] and "<itemIdentity href='urn:v'/>" in new['sourceText']
146 elif field not in ['scope', 'statuses', 'otherCorpusInputs', 'checks', 'cases']:
147 assert xtm[field] == value, field
148 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 62
149 assert [c['id'] for c in xtm['cases'][60:]] == TARGETS
150 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][60:])
151 assert len(xtm['variantItemIdentityOriginals']) == 2
152 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
153 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
154 dict(env, TOPIC_MAPS_21K_ROOT=str(ROOT), TOPIC_MAPS_21K_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
155 fault_controls = []
156 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
157 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
158 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
159 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
160 'variant-negative': negative_modes + [('tmdm', 'execution-error')],
161 'merge-variant': positive_modes + [('xtm', 'execution-error')],
162 'resource-ref': positive_modes + [('xtm', 'execution-error')],
163 'resource-ref-relative': positive_modes + [('xtm', 'execution-error')],
164 'uri-data': positive_modes + [('xtm', 'execution-error')],
165 'variant-iid': positive_modes + [('xtm', 'execution-error')],
166 'variant-iid-duplicate': positive_modes + [('xtm', 'execution-error')]}
167 for family, modes in families.items():
168 for mode, outcome in modes:
169 label = family + '-control-' + mode
170 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
171 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21K_FAMILY=family, TOPIC_MAPS_21K_FAULT=mode), expected_exit=1)
172 result = read(output / (label + '.json'))
173 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55, 'merge-variant': 56, 'resource-ref': 57, 'resource-ref-relative': 58, 'uri-data': 59, 'variant-iid': 60, 'variant-iid-duplicate': 61}[family]
174 assert result['statuses']['pass'] == retained_count
175 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
176 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
177 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
178 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
179
180 verified = []
181 for old in sorted((PRIOR / 'retained').glob('*.json')):
182 assert read(old) == read(retained / old.name), str(old)
183 verified.append(str(old.relative_to(ROOT)))
184 for suffix in ['validation', 'cxtm', 'encoding']:
185 old = PRIOR / f'ctm.{suffix}.json'
186 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
187 verified.append(str(old.relative_to(ROOT)))
188 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
189 stages = read(PRIOR / 'execution.json')['stageChecks']
190 assert sum(totals.values()) == 12100
191 for marker, count in dict(totals, **stages).items():
192 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
193 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
194 assert read(output / 'ctm.validation.json')['statuses'] == CTM
195 comparisons = []
196 (output / 'actual').mkdir()
197 for candidate_case, result in zip(candidate['candidates'], xtm['variantItemIdentityOriginals']):
198 assert result['id'] == candidate_case['id'] and result['condition'] is None and result['status'] == 'pass'
199 source = ROOT / 'vendor/cxtm' / result['id']
200 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
201 raw, actual = source.read_bytes(), result['actualCXTM'].encode('utf-8')
202 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
203 assert sha(source) == candidate_case['sourceSHA256'] and len(raw) == result['sourceBytes'] == candidate_case['sourceBytes']
204 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
205 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == candidate_case['baselineBytes']
206 assert sha(baseline) == candidate_case['baselineSHA256'] and result['firstDifferentByte'] is None
207 assert result['documentIRI'] == candidate_case['documentIRI'] and result['sourceKey'] == result['id']
208 assert result['model']['canonicalVariantReferences']
209 assert len(result['model']['variants']) == 1
210 variant = result['model']['variants'][0]
211 assert variant['value'] == 'TOPIC!' and variant['datatype'] == 'http://www.w3.org/2001/XMLSchema#string'
212 assert result['variantIdentifier'] == result['documentIRI'] + '#the-variant'
213 assert variant['itemIdentifiers'] == [result['variantIdentifier']]
214 assert result['sourceVariantCount'] == (1 if result['id'] == TARGETS[0] else 2)
215 (output / 'actual' / baseline.name).write_bytes(actual)
216 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
217 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
218 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
219 assert len(before) == len(after) == 124
220 assert [c['id'] for c in before] == [c['id'] for c in after]
221 changes = [{'id': a['id'], 'before': b, 'after': a,
222 'reason': 'Variant itemIdentity now uses the existing add-item-identifier API after native Variant normalization.'}
223 for b, a in zip(before, after) if b != a]
224 assert {c['id'] for c in changes} == set(TARGETS)
225 for change in changes:
226 assert change['before']['condition']['class'] == 'XTM-UNSUPPORTED'
227 assert change['before']['condition']['code'] == 'unsupported-construct'
228 assert change['after']['condition'] is None and change['after']['byteExact']
229 comparison = next(c for c in comparisons if c['id'] == change['id'])
230 assert comparison['actualCXTM'] == change['after']['actualCXTM']
231 save(output / 'diagnostic-changes.json', changes)
232 lock = read(ROOT / 'references/corpus-lock.json')
233 for entry in lock['files']:
234 p = ROOT / 'vendor/cxtm' / entry['path']
235 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
236 for snapshot in [protected, historical, sources]:
237 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
238 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
239 save(output / 'variant-item-identity-originals.json', {'kind': 'executed-original-variant-item-identity-registration', 'cases': comparisons})
240 save(output / 'diagnostics.json', {'kind': 'read-only-observations-no-extra-registration',
241 'originalObservations': 124, 'unchangedOriginalObservations': 124 - len(changes),
242 'diagnosticChanges': changes, 'registeredCases': TARGETS,
243 'other122ObservationsUnchanged': True})
244 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
245 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
246 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
247 save(output / 'execution.json', {'kind': 'xtm-variant-item-identity-original-registration-validation', 'baseCommit': BASE,
248 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
249 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
250 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
251 'stageChecks': stages, 'xmlQualificationChecks': 146, 'nativeVariantIdentifierAPIPreflightPassed': True, 'standaloneXTMChecks': len(labels),
252 'retainedXTMChecks': len(previous['checks']), 'retargetedUnsupportedCheckNames': rename,
253 'newXTMChecks': len(labels) - len(previous['checks']), 'retargetedUnsupportedControlRows': control_retargets,
254 'retainedOriginals': 60, 'retainedOriginalResultsUnchanged': True, 'newPositiveOriginals': 2,
255 'originalByteComparisons': [{'id': c['id'], 'actual': c['actualBytes'], 'baseline': c['expectedBytes'], 'firstDifferentByte': None} for c in comparisons], 'faultControls': len(fault_controls),
256 'diagnosticOriginalObservations': 124, 'diagnosticChangedOriginals': [c['id'] for c in changes],
257 'xtmStatusesBefore': {'pass': 60, 'fail': 0, 'unsupported': 0, 'not-run': 64}, 'xtmStatusesAfter': statuses,
258 'xtmStatusChanges': [{'id': id, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for id in TARGETS],
259 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
260 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
261 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsIdenticalToSlice21J': verified,
262 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
263 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionChangeLimitedToXTMReader': True, 'protectedSubsystemsUnchanged': ['TMDM', 'CTM', 'CXTM', 'XML'],
264 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
265 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
266 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
267 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
268
269
270 if __name__ == '__main__':
271 main()