]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21f.py
feat(xtm): support anyURI resource data on name variants
[topic-maps.git] / scripts / validate-slice-21f.py
1 #!/usr/bin/env python3
2 """Run Name Item Identity acceptance through the existing ASDF harness, offline.
3
4 All model construction, conformance classification and original execution use
5 existing Lisp functions. This driver verifies their reports and retains evidence.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21f'
23 PRIOR = ROOT / 'reports/slice-21e'
24 BASE = '5d3ed0455de32b9d19ac791b9e4c4882a03c79b8'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-names.lisp',
27 'tests/xtm-negative-originals.lisp', 'tests/ctm-xtm-mergemap.lisp', 'tests/xtm-name-scopes.lisp', 'topic-maps.asd'}
28 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
29 EXCLUDED = ['reifier-collision.xtm', 'role-duplicate-reified.xtm',
30 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
31
32
33 def read(p):
34 return json.loads(p.read_text())
35
36
37 def save(p, v):
38 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
39
40
41 def sha(p):
42 return hashlib.sha256(p.read_bytes()).hexdigest()
43
44
45 def command(args):
46 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
47
48
49 def prepare_pinned(target):
50 spec = importlib.util.spec_from_file_location('slice21e', ROOT / 'scripts/validate-slice-21e.py')
51 module = importlib.util.module_from_spec(spec)
52 spec.loader.exec_module(module)
53 return module.prepare_pinned(target)
54
55
56 def qualify_guard(v):
57 if isinstance(v, dict):
58 return {k: qualify_guard(x) for k, x in v.items()}
59 if isinstance(v, list):
60 return [qualify_guard(x) for x in v]
61 if isinstance(v, str) and (v == 'unsupported-name-item-identity' or v.startswith('unsupported-name-item-identity:')):
62 return v.replace('unsupported-name-item-identity', 'unsupported-name-variant', 1)
63 return v
64
65
66 def main():
67 parser = argparse.ArgumentParser(description=__doc__)
68 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
69 parser.add_argument('--output', type=Path)
70 parser.add_argument('--prepare', type=Path)
71 args = parser.parse_args()
72 if args.prepare:
73 print(json.dumps(prepare_pinned(args.prepare)))
74 return
75 runtime = shutil.which(args.sbcl)
76 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
77 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
78 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21f-'))
79 assert not output.is_relative_to(ROOT)
80 output.mkdir(parents=True, exist_ok=True)
81 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
82 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
83 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
84 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
85 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
86 source_paths = [*sorted(ALLOWED), 'tests/xtm-name-item-identities.lisp', 'scripts/validate-slice-21f.py',
87 *['reports/slice-21f/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp',
88 'diagnostics.lisp', 'before-diagnostics.json', 'baseline-xtm.json', 'model-preflight.lisp', 'model-preflight.json']]]
89 sources = {p: sha(ROOT / p) for p in source_paths}
90 previous = read(PRIOR / 'xtm.json')
91 assert read(EVIDENCE / 'baseline-xtm.json') == previous
92 assert previous['statuses'] == {'pass': 48, 'fail': 0, 'unsupported': 0, 'not-run': 0}
93 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 72, 'negativeUnregisteredUnexecuted': 4}
94 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
95 candidates = read(EVIDENCE / 'candidate-inventory.json')['candidates']
96 ids = [c['id'] for c in candidates]
97 assert ids == ['xtm2/in/itemid-name.xtm', 'xtm2/in/name-duplicate-iid.xtm']
98 assert len(set(ids)) == 2 and not set(ids) & {c['id'] for c in previous['cases']}
99 assert read(EVIDENCE / 'model-preflight.json')['allSixMergesPassed']
100 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
101 runs = []
102
103 def run(label, invocation, env, expected_exit=0):
104 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
105 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
106 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
107 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
108 stderr=subprocess.STDOUT, timeout=300)
109 (output / (label + '.log')).write_bytes(result.stdout)
110 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
111 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
112 runs.append(entry)
113 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
114 return result.stdout.decode()
115
116 retained = output / 'retained'
117 retained.mkdir()
118 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
119 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
120 with tempfile.TemporaryDirectory(prefix='topic-maps-21f-sources-') as temp_name:
121 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
122 save(output / 'source-preparation.json', provenance)
123 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
124 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
125 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
126 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
127 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
128 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
129 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
130 preflight_env = dict(env, TOPIC_MAPS_21F_ROOT=str(ROOT), TOPIC_MAPS_21F_PREFLIGHT=str(output / 'model-preflight.json'))
131 run('model-preflight', prefix + [str(EVIDENCE / 'model-preflight.lisp')], preflight_env)
132 assert read(output / 'model-preflight.json') == read(EVIDENCE / 'model-preflight.json')
133 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
134 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
135 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
136 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
137 xtm = read(output / 'xtm.json')
138 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
139 assert labels == xtm['checks'] and len(labels) == len(set(labels))
140 removed = ['name-syntax:unsupported-itemIdentity:class-code', 'name-syntax:unsupported-itemIdentity:context-position',
141 'name-syntax:scope:name-II-stays-unsupported:class-code', 'name-syntax:scope:name-II-stays-unsupported:context-position']
142 assert {qualify_guard(x) for x in previous['checks'] if x not in removed} <= set(labels)
143 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 43 original CXTM PASS; 7 negative originals PASS' in xtm_log
144 assert xtm['statuses'] == {'pass': 50, 'fail': 0, 'unsupported': 0, 'not-run': 0}
145 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 70, 'negativeUnregisteredUnexecuted': 4}
146 assert xtm['cases'][:48] == previous['cases']
147 for field in ['actualAndExpected', 'negativeOriginals', 'nameOriginals', 'scopeOriginals', 'helperSource']:
148 assert xtm[field] == previous[field], field
149 assert xtm['nameControls'] == [c for c in previous['nameControls'] if c['label'] != 'unsupported-itemIdentity']
150 assert xtm['scopeControls'] == [c for c in previous['scopeControls'] if c['label'] != 'scope:name-II-stays-unsupported']
151 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 50
152 assert [c['id'] for c in xtm['cases'][48:]] == ids and len(xtm['nameItemIdentityOriginals']) == 2
153 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][48:])
154 guard_changes = []
155 assert len(previous['negativeControls']) == len(xtm['negativeControls'])
156 for old, current in zip(previous['negativeControls'], xtm['negativeControls']):
157 if old['label'] != 'unsupported-is-not-pass':
158 assert old == current
159 continue
160 assert old['label'] == current['label'] and old['status'] == current['status'] == 'unsupported'
161 a, b = old['condition'], current['condition']
162 assert a['class'] == b['class'] == 'XTM-UNSUPPORTED' and a['code'] == b['code'] == 'unsupported-construct'
163 assert a['detail'] == 'XTM itemIdentity is not implemented in this subset.'
164 assert b['detail'] == 'XTM variant is not implemented in this subset.'
165 assert {k: v for k, v in a.items() if k not in ['detail', 'message']} == {k: v for k, v in b.items() if k not in ['detail', 'message']}
166 guard_changes.append({'before': old, 'after': current, 'reason': 'Valid Name Variant replaces the now supported Name Item Identity as Unsupported guard.'})
167 assert len(xtm['nameItemIdentityControls']) == 46
168 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
169 diag_env = dict(env, TOPIC_MAPS_21F_ROOT=str(ROOT), TOPIC_MAPS_21F_DIAGNOSTICS=str(output / 'after-diagnostics.json'))
170 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')], diag_env)
171 fault_controls = []
172 families = {'positive': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'), ('tmdm', 'execution-error'), ('unsupported', 'unsupported')],
173 'negative': [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')],
174 'name': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')],
175 'scope': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')],
176 'iid': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]}
177 for family, modes in families.items():
178 for mode, outcome in modes:
179 label = family + '-control-' + mode
180 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')),
181 TOPIC_MAPS_21F_FAMILY=family, TOPIC_MAPS_21F_FAULT=mode)
182 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')], control_env, expected_exit=1)
183 result = read(output / (label + '.json'))
184 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48}[family]
185 assert result['statuses']['pass'] == retained_count
186 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
187 assert result['cases'][:retained_count] == previous['cases'][:retained_count]
188 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
189 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
190
191 verified = []
192 for old in sorted((PRIOR / 'retained').glob('*.json')):
193 assert qualify_guard(read(old)) == read(retained / old.name), str(old)
194 verified.append(str(old.relative_to(ROOT)))
195 for suffix in ['validation', 'cxtm', 'encoding']:
196 old = PRIOR / f'ctm.{suffix}.json'
197 assert qualify_guard(read(old)) == read(output / f'ctm.{suffix}.json'), str(old)
198 verified.append(str(old.relative_to(ROOT)))
199 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
200 assert sum(totals.values()) == 12100
201 stages = read(PRIOR / 'execution.json')['stageChecks']
202 for marker, count in dict(totals, **stages).items():
203 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
204 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
205 assert read(output / 'ctm.validation.json')['statuses'] == CTM
206 comparisons = []
207 (output / 'actual').mkdir()
208 for old, result in zip(candidates, xtm['nameItemIdentityOriginals']):
209 assert old['id'] == result['id'] and result['condition'] is None and result['status'] == 'pass'
210 source = ROOT / 'vendor/cxtm' / old['id']
211 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
212 raw = source.read_bytes()
213 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
214 assert sha(source) == old['sourceSHA256'] and len(raw) == result['sourceBytes'] == old['sourceBytes']
215 actual = result['actualCXTM'].encode('utf-8')
216 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
217 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == old['baselineBytes']
218 assert sha(baseline) == old['baselineSHA256'] and result['firstDifferentByte'] is None
219 assert result['model']['canonicalNameParentTypeReferences'] and result['documentIRI'] == old['documentIRI']
220 assert result['sourceKey'] == result['id']
221 (output / 'actual' / baseline.name).write_bytes(actual)
222 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
223 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
224 before = read(EVIDENCE / 'before-diagnostics.json')
225 after = read(output / 'after-diagnostics.json')
226 assert len(before) == len(after) == 124
227 changes = []
228 for a, b in zip(before, after):
229 assert a['id'] == b['id'] and a['documentIRI'] == b['documentIRI']
230 if a == b:
231 continue
232 assert a['condition']['class'] == 'XTM-UNSUPPORTED'
233 assert a['condition']['detail'] == 'XTM itemIdentity is not implemented in this subset.'
234 if b['status'] == 'deserialized':
235 assert b['id'] in ids and b['byteExact']
236 else:
237 assert b['condition']['class'] == 'XTM-UNSUPPORTED' and b['condition']['code'] == 'unsupported-construct'
238 changes.append({'id': a['id'], 'before': a, 'after': b,
239 'warrant': 'A formerly Unsupported Name Item Identity is processed; excluded features remain Unsupported and native model conflicts remain distinct.'})
240 assert {b['id'] for a, b in zip(before, after) if a['status'] != 'deserialized' and b['status'] == 'deserialized'} == set(ids)
241 pending_negative = sorted('xtm2/invalid/' + p.name for p in (ROOT / 'vendor/cxtm/xtm2/invalid').glob('*.xtm')
242 if 'xtm2/invalid/' + p.name not in {c['id'] for c in xtm['negativeOriginals']})
243 assert pending_negative == ['xtm2/invalid/' + name for name in EXCLUDED]
244 assert not set(pending_negative) & {c['id'] for c in xtm['cases']}
245 lock = read(ROOT / 'references/corpus-lock.json')
246 for entry in lock['files']:
247 p = ROOT / 'vendor/cxtm' / entry['path']
248 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
249 for snapshot in [protected, historical, sources]:
250 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
251 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
252 save(output / 'name-item-identity-originals.json', {'kind': 'executed-name-item-identity-originals', 'cases': comparisons,
253 'excludedNegativeOriginals': pending_negative})
254 save(output / 'diagnostic-changes.json', {'kind': 'observations-no-additional-registration',
255 'changes': changes, 'unchangedOriginalObservations': 124 - len(changes),
256 'newByteExactOriginals': ids, 'counterprobeFixtureChanges': guard_changes})
257 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
258 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical,
259 'executionSourcesSHA256': sources})
260 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
261 save(output / 'execution.json', {'kind': 'xtm-name-item-identity-registration-validation', 'baseCommit': BASE,
262 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
263 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
264 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
265 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
266 'retainedXTMCheckNamesPreserved': len(set(previous['checks']) & set(labels)),
267 'retainedXTMChecksQualified': len(previous['checks']) - len(removed),
268 'removedItemIdentityUnsupportedChecksReplacedByPositiveCoverage': removed,
269 'renamedXTMChecks': [{'before': x, 'after': qualify_guard(x)} for x in previous['checks'] if x != qualify_guard(x)],
270 'retainedOriginals': 48, 'retainedOriginalResultsUnchanged': True, 'newNameItemIdentityOriginals': 2,
271 'nameItemIdentityCounterprobes': len(xtm['nameItemIdentityControls']),
272 'nameItemIdentityCounterprobeKinds': {k: sum(c['kind'] == k for c in xtm['nameItemIdentityControls'])
273 for k in sorted({c['kind'] for c in xtm['nameItemIdentityControls']})},
274 'faultControls': len(fault_controls), 'diagnosticOriginalObservations': 124,
275 'itemIdentityRelatedDiagnosticChanges': len(changes),
276 'xtmStatusesBefore': {'pass': 48, 'fail': 0, 'unsupported': 0, 'not-run': 76}, 'xtmStatusesAfter': statuses,
277 'xtmStatusChanges': [{'id': p, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for p in ids],
278 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
279 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
280 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
281 'allRetainedNonXTMReportsQualifiedAgainstSlice21E': verified,
282 'retainedReportQualification': 'Only unsupported-name-item-identity -> unsupported-name-variant fixture/check labels change.',
283 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected),
284 'pinnedCorpusFilesVerified': len(lock['files']), 'historicalReportsUnchanged': True,
285 'originalSourcesAndBaselinesUnchanged': True, 'productionChangesLimitedToXTMReader': True,
286 'TMDM_CTM_CXTM_XML_Unchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
287 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
288 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.',
289 'acceptancePassed': True})
290 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
291
292
293 if __name__ == '__main__':
294 main()