]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21l.py
Qualify closed XML/C14N 1.0 infrastructure for XTM resourceData
[topic-maps.git] / scripts / validate-slice-21l.py
1 #!/usr/bin/env python3
2 """Validate XTM Topic Map Reification original through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21l'
23 PRIOR = ROOT / 'reports/slice-21k'
24 BASE = 'da36095dddc1354ee8558dc9d729f5cb2fe53ff5'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/ctm-xtm-mergemap.lisp', 'topic-maps.asd'}
27 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
28 TARGETS = ['xtm2/in/tm-reifier.xtm']
29
30
31 def read(p):
32 return json.loads(p.read_text())
33
34
35 def save(p, v):
36 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
37
38
39 def sha(p):
40 return hashlib.sha256(p.read_bytes()).hexdigest()
41
42
43 def command(args):
44 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
45
46
47 def prepare_pinned(target):
48 spec = importlib.util.spec_from_file_location('slice21g', ROOT / 'scripts/validate-slice-21k.py')
49 module = importlib.util.module_from_spec(spec)
50 spec.loader.exec_module(module)
51 return module.prepare_pinned(target)
52
53
54 def main():
55 parser = argparse.ArgumentParser(description=__doc__)
56 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
57 parser.add_argument('--output', type=Path)
58 parser.add_argument('--prepare', type=Path)
59 args = parser.parse_args()
60 if args.prepare:
61 print(json.dumps(prepare_pinned(args.prepare)))
62 return
63 runtime = shutil.which(args.sbcl)
64 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
65 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
66 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21l-'))
67 assert not output.is_relative_to(ROOT)
68 output.mkdir(parents=True, exist_ok=True)
69 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
70 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
71 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
72 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
73 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
74 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'tests/xtm-map-reification.lisp', 'scripts/validate-slice-21l.py',
75 *['reports/slice-21l/' + p for p in ['candidate-inventory.json', 'api-preflight.lisp', 'failure-controls.lisp', 'diagnostics.lisp',
76 'before-diagnostics.json', 'baseline-xtm.json']]]}
77 previous = read(PRIOR / 'xtm.json')
78 assert read(EVIDENCE / 'baseline-xtm.json') == previous
79 assert previous['statuses'] == {'pass': 62, 'fail': 0, 'unsupported': 0, 'not-run': 0}
80 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 59, 'negativeUnregisteredUnexecuted': 3}
81 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
82 candidate = read(EVIDENCE / 'candidate-inventory.json')
83 assert [c['id'] for c in candidate['candidates']] == TARGETS
84 assert [c['sourceBytes'] for c in candidate['candidates']] == [94]
85 assert [c['baselineBytes'] for c in candidate['candidates']] == [128]
86 assert not set(TARGETS) & {c['id'] for c in previous['cases']}
87 assert read(EVIDENCE / 'before-diagnostics.json') == read(PRIOR / 'after-diagnostics.json')
88 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
89 runs = []
90
91 def run(label, invocation, env, expected_exit=0):
92 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
93 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
94 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
95 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
96 (output / (label + '.log')).write_bytes(result.stdout)
97 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
98 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
99 runs.append(entry)
100 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
101 return result.stdout.decode()
102
103 retained = output / 'retained'
104 retained.mkdir()
105 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
106 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
107 with tempfile.TemporaryDirectory(prefix='topic-maps-21l-sources-') as temp_name:
108 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
109 save(output / 'source-preparation.json', provenance)
110 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
111 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
112 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
113 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
114 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
115 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
116 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
117 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
118 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
119 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
120 api_probe = run('map-reifier-api-preflight', prefix + [str(EVIDENCE / 'api-preflight.lisp')], env)
121 assert 'MAP-REIFIER-API-PREFLIGHT-PASS:' in api_probe
122 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
123 xtm = read(output / 'xtm.json')
124 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
125 assert labels == xtm['checks'] and len(labels) == len(set(labels))
126 rename = {}
127 retained_labels = previous['checks']
128 assert set(retained_labels) <= set(labels)
129 assert [label for label in labels if label in set(retained_labels)] == retained_labels
130 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 55 original CXTM PASS; 8 negative originals PASS' in xtm_log
131 assert xtm['statuses'] == {'pass': 63, 'fail': 0, 'unsupported': 0, 'not-run': 0}
132 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 58, 'negativeUnregisteredUnexecuted': 3}
133 assert xtm['cases'][:62] == previous['cases']
134 control_retargets = {'standaloneUnsupportedReifier': 'Map fixture -> excluded Name Reification; same labels/class/code.',
135 'mergemapUnsupportedReification': 'Map fixture -> excluded Name Reification; same labels/class/code.'}
136 for field, value in previous.items():
137 if field not in ['scope', 'statuses', 'otherCorpusInputs', 'checks', 'cases']:
138 assert xtm[field] == value, field
139 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 63
140 assert [c['id'] for c in xtm['cases'][62:]] == TARGETS
141 assert all(c['status'] == 'pass' and c['actualOutcome'] == 'cxtm-match' for c in xtm['cases'][62:])
142 assert len(xtm['mapReifierOriginals']) == 1
143 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
144 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
145 dict(env, TOPIC_MAPS_21L_ROOT=str(ROOT), TOPIC_MAPS_21L_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
146 fault_controls = []
147 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
148 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
149 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
150 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
151 'variant-negative': negative_modes + [('tmdm', 'execution-error')],
152 'merge-variant': positive_modes + [('xtm', 'execution-error')],
153 'resource-ref': positive_modes + [('xtm', 'execution-error')],
154 'resource-ref-relative': positive_modes + [('xtm', 'execution-error')],
155 'uri-data': positive_modes + [('xtm', 'execution-error')],
156 'variant-iid': positive_modes + [('xtm', 'execution-error')],
157 'variant-iid-duplicate': positive_modes + [('xtm', 'execution-error')],
158 'map-reifier': positive_modes + [('xtm', 'execution-error')],
159 'map-reifier-condition': [('wrong-cause', 'foreign-reifier-rejected-as-wrong-condition')]}
160 for family, modes in families.items():
161 for mode, outcome in modes:
162 label = family + '-control-' + mode
163 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
164 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21L_FAMILY=family, TOPIC_MAPS_21L_FAULT=mode), expected_exit=1)
165 result = read(output / (label + '.json'))
166 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55, 'merge-variant': 56, 'resource-ref': 57, 'resource-ref-relative': 58, 'uri-data': 59, 'variant-iid': 60, 'variant-iid-duplicate': 61, 'map-reifier': 62, 'map-reifier-condition': 63}[family]
167 assert result['statuses']['pass'] == retained_count
168 if family == 'map-reifier-condition':
169 assert len(result['cases']) == retained_count
170 wrong = next(row for row in result['mapReifierControls'] if row['label'] == 'reuse-reifier')
171 assert wrong['expectedCode'] == 'reifier-conflict'
172 assert wrong['condition']['class'] == 'REIFICATION-ERROR' and wrong['condition']['code'] == 'foreign-reifier'
173 assert wrong['atomicNativeOperation']
174 assert result['checks'][-1] == 'map-reifier-error:reuse-reifier:specific-class-code'
175 else:
176 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
177 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
178 if family == 'map-reifier' and mode == 'tmdm':
179 actual_condition = result['mapReifierOriginals'][0]['condition']
180 assert actual_condition['class'] == 'REIFICATION-ERROR' and actual_condition['code'] == 'reifier-conflict'
181 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
182 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
183
184 verified = []
185 for old in sorted((PRIOR / 'retained').glob('*.json')):
186 assert read(old) == read(retained / old.name), str(old)
187 verified.append(str(old.relative_to(ROOT)))
188 for suffix in ['validation', 'cxtm', 'encoding']:
189 old = PRIOR / f'ctm.{suffix}.json'
190 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
191 verified.append(str(old.relative_to(ROOT)))
192 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
193 stages = read(PRIOR / 'execution.json')['stageChecks']
194 assert sum(totals.values()) == 12100
195 for marker, count in dict(totals, **stages).items():
196 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
197 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
198 assert read(output / 'ctm.validation.json')['statuses'] == CTM
199 comparisons = []
200 (output / 'actual').mkdir()
201 for candidate_case, result in zip(candidate['candidates'], xtm['mapReifierOriginals']):
202 assert result['id'] == candidate_case['id'] and result['condition'] is None and result['status'] == 'pass'
203 source = ROOT / 'vendor/cxtm' / result['id']
204 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
205 raw, actual = source.read_bytes(), result['actualCXTM'].encode('utf-8')
206 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
207 assert sha(source) == candidate_case['sourceSHA256'] and len(raw) == result['sourceBytes'] == candidate_case['sourceBytes']
208 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
209 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == candidate_case['baselineBytes']
210 assert sha(baseline) == candidate_case['baselineSHA256'] and result['firstDifferentByte'] is None
211 assert result['documentIRI'] == candidate_case['documentIRI'] and result['sourceKey'] == result['id']
212 model = result['model']
213 identity = model['identity']['nameModel']['identity']
214 assert model['canonicalReifierAndInverseEQ'] and model['reifierTopic'] == 1 and model['topicReifiedMapEQ'] == [True]
215 assert model['mapItemIdentifiers'] == [] and identity['topicCount'] == 1 and identity['associationCount'] == 0
216 assert identity['topics'] == [{'ordinal': 1, 'itemIdentifiers': [result['documentIRI'] + '#reifier'],
217 'subjectIdentifiers': [], 'subjectLocators': []}]
218 assert model['identity']['nameModel']['names'] == [] and model['identity']['variants'] == []
219 assert result['reifierLocator'] == result['documentIRI'] + '#reifier'
220 (output / 'actual' / baseline.name).write_bytes(actual)
221 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
222 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
223 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
224 assert len(before) == len(after) == 124
225 assert [c['id'] for c in before] == [c['id'] for c in after]
226 changes = [{'id': a['id'], 'before': b, 'after': a,
227 'reason': ('Map Reification now constructs and assigns the native II-resolved reifier Topic.'
228 if a['id'] in TARGETS else 'Map attribute is supported; schema validation now reaches excluded Occurrence processing. No negative contract registered.')}
229 for b, a in zip(before, after) if b != a]
230 assert {c['id'] for c in changes} == set(TARGETS + ['xtm2/invalid/reifier-collision.xtm'])
231 for change in changes:
232 assert change['before']['condition']['class'] == 'XTM-UNSUPPORTED'
233 assert change['before']['condition']['code'] == 'reification'
234 if change['id'] in TARGETS:
235 assert change['after']['condition'] is None and change['after']['byteExact']
236 assert comparisons[0]['actualCXTM'] == change['after']['actualCXTM']
237 else:
238 assert change['after']['condition']['class'] == 'XTM-UNSUPPORTED'
239 assert change['after']['condition']['code'] == 'unsupported-construct'
240 assert change['after']['condition']['detail'] == 'XTM occurrence is not implemented in this subset.'
241 assert change['after']['status'] == 'rejected' and not change['after']['byteExact']
242 pending_negative = ['xtm2/invalid/' + stem + '.xtm' for stem in
243 ['reifier-collision', 'role-duplicate-reified', 'topicref-no-fragment-id']]
244 assert not set(pending_negative) & {case['id'] for case in xtm['cases']}
245 save(output / 'diagnostic-changes.json', changes)
246 lock = read(ROOT / 'references/corpus-lock.json')
247 for entry in lock['files']:
248 p = ROOT / 'vendor/cxtm' / entry['path']
249 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
250 for snapshot in [protected, historical, sources]:
251 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
252 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
253 save(output / 'map-reifier-originals.json', {'kind': 'executed-original-map-reifier-registration', 'cases': comparisons})
254 save(output / 'diagnostics.json', {'kind': 'read-only-observations-no-extra-registration',
255 'originalObservations': 124, 'unchangedOriginalObservations': 124 - len(changes),
256 'diagnosticChanges': changes, 'registeredCases': TARGETS, 'negativeOriginalsRemainingNotRun': pending_negative,
257 'other122ObservationsUnchanged': True})
258 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
259 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
260 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
261 save(output / 'execution.json', {'kind': 'xtm-map-reifier-original-registration-validation', 'baseCommit': BASE,
262 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
263 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
264 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
265 'stageChecks': stages, 'xmlQualificationChecks': 146, 'nativeMapReifierAPIPreflightPassed': True, 'standaloneXTMChecks': len(labels),
266 'retainedXTMChecks': len(previous['checks']), 'retargetedUnsupportedCheckNames': rename,
267 'newXTMChecks': len(labels) - len(previous['checks']), 'retargetedUnsupportedControlRows': control_retargets,
268 'retainedOriginals': 62, 'retainedOriginalResultsUnchanged': True, 'newPositiveOriginals': 1,
269 'originalByteComparisons': [{'id': c['id'], 'actual': c['actualBytes'], 'baseline': c['expectedBytes'], 'firstDifferentByte': None} for c in comparisons], 'faultControls': len(fault_controls),
270 'diagnosticOriginalObservations': 124, 'diagnosticChangedOriginals': [c['id'] for c in changes],
271 'xtmStatusesBefore': {'pass': 62, 'fail': 0, 'unsupported': 0, 'not-run': 62}, 'xtmStatusesAfter': statuses,
272 'xtmStatusChanges': [{'id': id, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for id in TARGETS],
273 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
274 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
275 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsIdenticalToSlice21K': verified,
276 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
277 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionChangeLimitedToXTMReader': True, 'protectedSubsystemsUnchanged': ['TMDM', 'CTM', 'CXTM', 'XML'],
278 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
279 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
280 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
281 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
282
283
284 if __name__ == '__main__':
285 main()