]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21e.py
Implement XTM association reification with native conformance evidence
[topic-maps.git] / scripts / validate-slice-21e.py
1 #!/usr/bin/env python3
2 """Run Name Scope acceptance through the existing ASDF harness, offline.
3
4 All model construction, conformance classification and original execution use
5 existing Lisp functions. This driver verifies their reports and retains evidence.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21e'
23 PRIOR = ROOT / 'reports/slice-21d'
24 BASE = 'd034a8deafa7cf74b2264ea8a7bd5cc0950806bf'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-names.lisp',
27 'tests/xtm-negative-originals.lisp', 'tests/ctm-xtm-mergemap.lisp', 'topic-maps.asd'}
28 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
29 EXCLUDED = ['reifier-collision.xtm', 'role-duplicate-reified.xtm',
30 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
31
32
33 def read(p):
34 return json.loads(p.read_text())
35
36
37 def save(p, v):
38 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
39
40
41 def sha(p):
42 return hashlib.sha256(p.read_bytes()).hexdigest()
43
44
45 def command(args):
46 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
47
48
49 def prepare_pinned(target):
50 spec = importlib.util.spec_from_file_location('slice21d', ROOT / 'scripts/validate-slice-21d.py')
51 module = importlib.util.module_from_spec(spec)
52 spec.loader.exec_module(module)
53 return module.prepare_pinned(target)
54
55
56 def qualify_guard(v):
57 if isinstance(v, dict):
58 return {k: qualify_guard(x) for k, x in v.items()}
59 if isinstance(v, list):
60 return [qualify_guard(x) for x in v]
61 if isinstance(v, str) and (v == 'unsupported-name-scope' or v.startswith('unsupported-name-scope:')):
62 return v.replace('unsupported-name-scope', 'unsupported-name-item-identity', 1)
63 return v
64
65
66 def main():
67 parser = argparse.ArgumentParser(description=__doc__)
68 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
69 parser.add_argument('--output', type=Path)
70 parser.add_argument('--prepare', type=Path)
71 args = parser.parse_args()
72 if args.prepare:
73 print(json.dumps(prepare_pinned(args.prepare)))
74 return
75 runtime = shutil.which(args.sbcl)
76 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
77 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
78 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21e-'))
79 assert not output.is_relative_to(ROOT)
80 output.mkdir(parents=True, exist_ok=True)
81 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
82 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
83 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
84 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
85 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
86 source_paths = [*sorted(ALLOWED), 'tests/xtm-name-scopes.lisp', 'scripts/validate-slice-21e.py',
87 *['reports/slice-21e/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp',
88 'diagnostics.lisp', 'before-diagnostics.json', 'baseline-xtm.json']]]
89 sources = {p: sha(ROOT / p) for p in source_paths}
90 previous = read(PRIOR / 'xtm.json')
91 assert read(EVIDENCE / 'baseline-xtm.json') == previous
92 assert previous['statuses'] == {'pass': 43, 'fail': 0, 'unsupported': 0, 'not-run': 0}
93 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 77, 'negativeUnregisteredUnexecuted': 4}
94 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
95 candidates = read(EVIDENCE / 'candidate-inventory.json')['candidates']
96 ids = [c['id'] for c in candidates]
97 assert ids == ['xtm2/in/' + n + '.xtm' for n in ['name-scope', 'name-scope-multiple',
98 'name-scope-duplicate', 'name-scope-duplicate-merged', 'name-type-scope']]
99 assert len(set(ids)) == 5 and not set(ids) & {c['id'] for c in previous['cases']}
100 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
101 runs = []
102
103 def run(label, invocation, env, expected_exit=0):
104 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
105 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
106 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
107 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
108 stderr=subprocess.STDOUT, timeout=300)
109 (output / (label + '.log')).write_bytes(result.stdout)
110 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
111 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
112 runs.append(entry)
113 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
114 return result.stdout.decode()
115
116 retained = output / 'retained'
117 retained.mkdir()
118 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
119 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
120 with tempfile.TemporaryDirectory(prefix='topic-maps-21e-sources-') as temp_name:
121 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
122 save(output / 'source-preparation.json', provenance)
123 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
124 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
125 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
126 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
127 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
128 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
129 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
130 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
131 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
132 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
133 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
134 xtm = read(output / 'xtm.json')
135 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
136 assert labels == xtm['checks'] and len(labels) == len(set(labels))
137 removed = ['name-syntax:unsupported-scope:class-code', 'name-syntax:unsupported-scope:context-position']
138 assert {qualify_guard(x) for x in previous['checks'] if x not in removed} <= set(labels)
139 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 41 original CXTM PASS; 7 negative originals PASS' in xtm_log
140 assert xtm['statuses'] == {'pass': 48, 'fail': 0, 'unsupported': 0, 'not-run': 0}
141 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 72, 'negativeUnregisteredUnexecuted': 4}
142 assert xtm['cases'][:43] == previous['cases']
143 for field in ['actualAndExpected', 'negativeOriginals', 'nameOriginals', 'helperSource']:
144 assert xtm[field] == previous[field], field
145 assert xtm['nameControls'] == [c for c in previous['nameControls'] if c['label'] != 'unsupported-scope']
146 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 48
147 assert [c['id'] for c in xtm['cases'][43:]] == ids and len(xtm['scopeOriginals']) == 5
148 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][43:])
149 guard_changes = []
150 assert len(previous['negativeControls']) == len(xtm['negativeControls'])
151 for old, current in zip(previous['negativeControls'], xtm['negativeControls']):
152 if old['label'] != 'unsupported-is-not-pass':
153 assert old == current
154 continue
155 assert old['label'] == current['label'] and old['status'] == current['status'] == 'unsupported'
156 a, b = old['condition'], current['condition']
157 assert a['class'] == b['class'] == 'XTM-UNSUPPORTED' and a['code'] == b['code'] == 'unsupported-construct'
158 assert a['detail'] == 'XTM scope is not implemented in this subset.'
159 assert b['detail'] == 'XTM itemIdentity is not implemented in this subset.' and b['column'] == a['column'] + 18
160 assert {k: v for k, v in a.items() if k not in ['detail', 'column', 'message']} == {k: v for k, v in b.items() if k not in ['detail', 'column', 'message']}
161 guard_changes.append({'before': old, 'after': current, 'reason': 'Valid Name Item Identity replaces the now supported Scope as Unsupported guard.'})
162 assert len(xtm['scopeControls']) == 38
163 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
164 diag_env = dict(env, TOPIC_MAPS_21E_ROOT=str(ROOT), TOPIC_MAPS_21E_DIAGNOSTICS=str(output / 'after-diagnostics.json'))
165 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')], diag_env)
166 fault_controls = []
167 families = {'positive': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'), ('tmdm', 'execution-error'), ('unsupported', 'unsupported')],
168 'negative': [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')],
169 'name': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')],
170 'scope': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]}
171 for family, modes in families.items():
172 for mode, outcome in modes:
173 label = family + '-control-' + mode
174 control_env = dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')),
175 TOPIC_MAPS_21E_FAMILY=family, TOPIC_MAPS_21E_FAULT=mode)
176 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')], control_env, expected_exit=1)
177 result = read(output / (label + '.json'))
178 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43}[family]
179 assert result['statuses']['pass'] == retained_count
180 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
181 assert result['cases'][:retained_count] == previous['cases'][:retained_count]
182 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
183 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
184
185 verified = []
186 for old in sorted((PRIOR / 'retained').glob('*.json')):
187 assert qualify_guard(read(old)) == read(retained / old.name), str(old)
188 verified.append(str(old.relative_to(ROOT)))
189 for suffix in ['validation', 'cxtm', 'encoding']:
190 old = PRIOR / f'ctm.{suffix}.json'
191 assert qualify_guard(read(old)) == read(output / f'ctm.{suffix}.json'), str(old)
192 verified.append(str(old.relative_to(ROOT)))
193 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
194 assert sum(totals.values()) == 12100
195 stages = read(PRIOR / 'execution.json')['stageChecks']
196 for marker, count in dict(totals, **stages).items():
197 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
198 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
199 assert read(output / 'ctm.validation.json')['statuses'] == CTM
200 comparisons = []
201 (output / 'actual').mkdir()
202 for old, result in zip(candidates, xtm['scopeOriginals']):
203 assert old['id'] == result['id'] and result['condition'] is None and result['status'] == 'pass'
204 source = ROOT / 'vendor/cxtm' / old['id']
205 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
206 raw = source.read_bytes()
207 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
208 assert sha(source) == old['sourceSHA256'] and len(raw) == result['sourceBytes'] == old['sourceBytes']
209 actual = result['actualCXTM'].encode('utf-8')
210 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
211 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == old['baselineBytes']
212 assert sha(baseline) == old['baselineSHA256'] and result['firstDifferentByte'] is None
213 assert result['model']['canonicalNameParentTypeReferences'] and result['documentIRI'] == old['documentIRI']
214 assert result['sourceKey'] == result['id']
215 (output / 'actual' / baseline.name).write_bytes(actual)
216 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
217 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
218 before = read(EVIDENCE / 'before-diagnostics.json')
219 after = read(output / 'after-diagnostics.json')
220 assert len(before) == len(after) == 124
221 changes = []
222 for a, b in zip(before, after):
223 assert a['id'] == b['id'] and a['documentIRI'] == b['documentIRI']
224 if a == b:
225 continue
226 assert a['condition']['class'] == 'XTM-UNSUPPORTED'
227 assert a['condition']['detail'] == 'XTM scope is not implemented in this subset.'
228 if b['status'] == 'deserialized':
229 assert b['id'] in ids and b['byteExact']
230 else:
231 assert b['condition']['class'] == 'XTM-UNSUPPORTED' and b['condition']['code'] == 'unsupported-construct'
232 changes.append({'id': a['id'], 'before': a, 'after': b,
233 'warrant': 'A formerly Unsupported Name Scope is processed; the next excluded feature remains Unsupported.'})
234 assert {b['id'] for a, b in zip(before, after) if a['status'] != 'deserialized' and b['status'] == 'deserialized'} == set(ids)
235 pending_negative = sorted('xtm2/invalid/' + p.name for p in (ROOT / 'vendor/cxtm/xtm2/invalid').glob('*.xtm')
236 if 'xtm2/invalid/' + p.name not in {c['id'] for c in xtm['negativeOriginals']})
237 assert pending_negative == ['xtm2/invalid/' + name for name in EXCLUDED]
238 assert not set(pending_negative) & {c['id'] for c in xtm['cases']}
239 lock = read(ROOT / 'references/corpus-lock.json')
240 for entry in lock['files']:
241 p = ROOT / 'vendor/cxtm' / entry['path']
242 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
243 for snapshot in [protected, historical, sources]:
244 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
245 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
246 save(output / 'scope-originals.json', {'kind': 'executed-name-scope-originals', 'cases': comparisons,
247 'excludedNegativeOriginals': pending_negative})
248 save(output / 'diagnostic-changes.json', {'kind': 'observations-no-additional-registration',
249 'changes': changes, 'unchangedOriginalObservations': 124 - len(changes),
250 'newByteExactOriginals': ids, 'counterprobeFixtureChanges': guard_changes})
251 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
252 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical,
253 'executionSourcesSHA256': sources})
254 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
255 save(output / 'execution.json', {'kind': 'xtm-name-scope-registration-validation', 'baseCommit': BASE,
256 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
257 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
258 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
259 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
260 'retainedXTMCheckNamesPreserved': len(set(previous['checks']) & set(labels)),
261 'retainedXTMChecksQualified': len(previous['checks']) - len(removed),
262 'removedScopeUnsupportedChecksReplacedByPositiveCoverage': removed,
263 'renamedXTMChecks': [{'before': x, 'after': qualify_guard(x)} for x in previous['checks'] if x != qualify_guard(x)],
264 'retainedOriginals': 43, 'retainedOriginalResultsUnchanged': True, 'newScopeOriginals': 5,
265 'scopeCounterprobes': len(xtm['scopeControls']),
266 'scopeCounterprobeKinds': {k: sum(c['kind'] == k for c in xtm['scopeControls'])
267 for k in sorted({c['kind'] for c in xtm['scopeControls']})},
268 'faultControls': len(fault_controls), 'diagnosticOriginalObservations': 124,
269 'scopeRelatedDiagnosticChanges': len(changes),
270 'xtmStatusesBefore': {'pass': 43, 'fail': 0, 'unsupported': 0, 'not-run': 81}, 'xtmStatusesAfter': statuses,
271 'xtmStatusChanges': [{'id': p, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for p in ids],
272 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
273 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
274 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
275 'allRetainedNonXTMReportsQualifiedAgainstSlice21D': verified,
276 'retainedReportQualification': 'Only unsupported-name-scope -> unsupported-name-item-identity fixture/check labels change.',
277 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected),
278 'pinnedCorpusFilesVerified': len(lock['files']), 'historicalReportsUnchanged': True,
279 'originalSourcesAndBaselinesUnchanged': True, 'productionChangesLimitedToXTMReader': True,
280 'TMDM_CTM_CXTM_XML_Unchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
281 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
282 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.',
283 'acceptancePassed': True})
284 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
285
286
287 if __name__ == '__main__':
288 main()