]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21h.py
Document XTM normative coverage and remaining qualification gaps
[topic-maps.git] / scripts / validate-slice-21h.py
1 #!/usr/bin/env python3
2 """Register one existing XTM Variant Merge original through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21h'
23 PRIOR = ROOT / 'reports/slice-21g'
24 BASE = '39c125bf9d462f538403aa633c56054511c210da'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'tests/xtm-identity.lisp', 'tests/xtm-variants.lisp', 'topic-maps.asd'}
27 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
28 TARGET = 'xtm2/in/merge-itemid-with-variants.xtm'
29
30
31 def read(p):
32 return json.loads(p.read_text())
33
34
35 def save(p, v):
36 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
37
38
39 def sha(p):
40 return hashlib.sha256(p.read_bytes()).hexdigest()
41
42
43 def command(args):
44 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
45
46
47 def prepare_pinned(target):
48 spec = importlib.util.spec_from_file_location('slice21g', ROOT / 'scripts/validate-slice-21g.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare_pinned(target)
52
53
54 def main():
55 parser = argparse.ArgumentParser(description=__doc__)
56 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
57 parser.add_argument('--output', type=Path)
58 parser.add_argument('--prepare', type=Path)
59 args = parser.parse_args()
60 if args.prepare:
61 print(json.dumps(prepare_pinned(args.prepare)))
62 return
63 runtime = shutil.which(args.sbcl)
64 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
65 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
66 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21h-'))
67 assert not output.is_relative_to(ROOT)
68 output.mkdir(parents=True, exist_ok=True)
69 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
70 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
71 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
72 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
73 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
74 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'scripts/validate-slice-21h.py',
75 *['reports/slice-21h/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp', 'diagnostics.lisp',
76 'before-diagnostics.json', 'baseline-xtm.json']]]}
77 previous = read(PRIOR / 'xtm.json')
78 assert read(EVIDENCE / 'baseline-xtm.json') == previous
79 assert previous['statuses'] == {'pass': 56, 'fail': 0, 'unsupported': 0, 'not-run': 0}
80 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 65, 'negativeUnregisteredUnexecuted': 3}
81 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
82 candidate = read(EVIDENCE / 'candidate-inventory.json')
83 assert candidate['id'] == TARGET and candidate['sourceBytes'] == 641 and candidate['baselineBytes'] == 919
84 assert TARGET not in {c['id'] for c in previous['cases']}
85 assert read(EVIDENCE / 'before-diagnostics.json') == read(PRIOR / 'after-diagnostics.json')
86 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
87 runs = []
88
89 def run(label, invocation, env, expected_exit=0):
90 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
91 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
92 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
93 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
94 (output / (label + '.log')).write_bytes(result.stdout)
95 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
96 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
97 runs.append(entry)
98 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
99 return result.stdout.decode()
100
101 retained = output / 'retained'
102 retained.mkdir()
103 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
104 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
105 with tempfile.TemporaryDirectory(prefix='topic-maps-21h-sources-') as temp_name:
106 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
107 save(output / 'source-preparation.json', provenance)
108 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
109 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
110 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
111 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
112 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
113 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
114 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
115 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
116 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
117 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
118 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
119 xtm = read(output / 'xtm.json')
120 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
121 assert labels == xtm['checks'] and len(labels) == len(set(labels))
122 assert set(previous['checks']) <= set(labels)
123 assert [label for label in labels if label in set(previous['checks'])] == previous['checks']
124 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 49 original CXTM PASS; 8 negative originals PASS' in xtm_log
125 assert xtm['statuses'] == {'pass': 57, 'fail': 0, 'unsupported': 0, 'not-run': 0}
126 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 64, 'negativeUnregisteredUnexecuted': 3}
127 assert xtm['cases'][:56] == previous['cases']
128 for field, value in previous.items():
129 if field not in ['scope', 'statuses', 'otherCorpusInputs', 'checks', 'cases']:
130 assert xtm[field] == value, field
131 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 57
132 assert xtm['cases'][56]['id'] == TARGET and xtm['cases'][56]['status'] == 'pass'
133 assert xtm['cases'][56]['actualOutcome'] == 'cxtm-match' and len(xtm['variantMergeOriginals']) == 1
134 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
135 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
136 dict(env, TOPIC_MAPS_21H_ROOT=str(ROOT), TOPIC_MAPS_21H_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
137 fault_controls = []
138 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
139 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
140 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
141 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
142 'variant-negative': negative_modes + [('tmdm', 'execution-error')],
143 'merge-variant': positive_modes + [('xtm', 'execution-error')]}
144 for family, modes in families.items():
145 for mode, outcome in modes:
146 label = family + '-control-' + mode
147 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
148 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21H_FAMILY=family, TOPIC_MAPS_21H_FAULT=mode), expected_exit=1)
149 result = read(output / (label + '.json'))
150 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55, 'merge-variant': 56}[family]
151 assert result['statuses']['pass'] == retained_count
152 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
153 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
154 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
155 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
156
157 verified = []
158 for old in sorted((PRIOR / 'retained').glob('*.json')):
159 assert read(old) == read(retained / old.name), str(old)
160 verified.append(str(old.relative_to(ROOT)))
161 for suffix in ['validation', 'cxtm', 'encoding']:
162 old = PRIOR / f'ctm.{suffix}.json'
163 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
164 verified.append(str(old.relative_to(ROOT)))
165 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
166 stages = read(PRIOR / 'execution.json')['stageChecks']
167 assert sum(totals.values()) == 12100
168 for marker, count in dict(totals, **stages).items():
169 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
170 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
171 assert read(output / 'ctm.validation.json')['statuses'] == CTM
172 result = xtm['variantMergeOriginals'][0]
173 assert result['id'] == TARGET and result['condition'] is None and result['status'] == 'pass'
174 source = ROOT / 'vendor/cxtm' / TARGET
175 baseline = ROOT / 'vendor/cxtm/xtm2/baseline/merge-itemid-with-variants.xtm.cxtm'
176 raw, actual = source.read_bytes(), result['actualCXTM'].encode('utf-8')
177 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
178 assert sha(source) == candidate['sourceSHA256'] and len(raw) == result['sourceBytes'] == 641
179 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
180 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == 919
181 assert sha(baseline) == candidate['baselineSHA256'] and result['firstDifferentByte'] is None
182 assert result['model']['canonicalVariantReferences'] and result['documentIRI'] == candidate['documentIRI'] and result['sourceKey'] == TARGET
183 assert len(result['originalReaderStages']) == 2
184 assert result['originalReaderStages'][1]['model'] == result['model']
185 (output / 'actual').mkdir()
186 (output / 'actual' / baseline.name).write_bytes(actual)
187 comparison = dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
188 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True)
189 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
190 assert len(before) == len(after) == 124 and before == after
191 observation = next(c for c in after if c['id'] == TARGET)
192 assert observation == candidate['priorEvidence']['observation']
193 assert observation['byteExact'] and observation['actualCXTM'].encode('utf-8') == actual
194 lock = read(ROOT / 'references/corpus-lock.json')
195 for entry in lock['files']:
196 p = ROOT / 'vendor/cxtm' / entry['path']
197 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
198 for snapshot in [protected, historical, sources]:
199 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
200 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
201 save(output / 'variant-merge-original.json', {'kind': 'executed-original-variant-merge-registration', 'case': comparison})
202 save(output / 'diagnostics.json', {'kind': 'read-only-observations-no-extra-registration',
203 'originalObservations': 124, 'unchangedOriginalObservations': 124, 'diagnosticChanges': [],
204 'priorSlice21GObservationReproduced': True, 'newlyRegisteredCase': TARGET})
205 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
206 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
207 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
208 save(output / 'execution.json', {'kind': 'xtm-variant-merge-original-registration-validation', 'baseCommit': BASE,
209 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
210 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
211 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
212 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
213 'retainedXTMCheckNamesPreserved': len(previous['checks']), 'newXTMChecks': len(labels) - len(previous['checks']),
214 'retainedOriginals': 56, 'retainedOriginalResultsUnchanged': True, 'newPositiveOriginals': 1,
215 'originalByteComparison': {'actual': len(actual), 'baseline': 919, 'firstDifferentByte': None},
216 'originalReaderStageCount': len(result['originalReaderStages']), 'faultControls': len(fault_controls),
217 'diagnosticOriginalObservations': 124, 'diagnosticChanges': [],
218 'xtmStatusesBefore': {'pass': 56, 'fail': 0, 'unsupported': 0, 'not-run': 68}, 'xtmStatusesAfter': statuses,
219 'xtmStatusChanges': [{'id': TARGET, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'}],
220 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
221 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
222 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsIdenticalToSlice21G': verified,
223 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
224 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionCodeUnchanged': True,
225 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
226 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
227 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
228 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
229
230
231 if __name__ == '__main__':
232 main()