]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-21j.py
Implement XTM occurrence core and register eleven originals (Slice 22G)
[topic-maps.git] / scripts / validate-slice-21j.py
1 #!/usr/bin/env python3
2 """Validate XTM Variant anyURI ResourceData original through the existing ASDF/Case Record harness.
3
4 The driver checks and retains execution evidence; Lisp performs all parsing,
5 model construction and conformance classification. Historical reports are inputs.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import importlib.util
11 import json
12 import os
13 from pathlib import Path
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import tempfile
19
20 sys.dont_write_bytecode = True
21 ROOT = Path(__file__).resolve().parents[1]
22 EVIDENCE = ROOT / 'reports/slice-21j'
23 PRIOR = ROOT / 'reports/slice-21i'
24 BASE = '4d7adecb3c2d5df90579aa37dfa41128988cb56c'
25 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT) + '"))'
26 ALLOWED = {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-variants.lisp', 'tests/xtm-negative-originals.lisp',
27 'tests/ctm-xtm-mergemap.lisp', 'tests/xtm-names.lisp', 'tests/xtm-name-scopes.lisp', 'tests/xtm-name-item-identities.lisp', 'topic-maps.asd'}
28 CTM = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
29 TARGETS = ['xtm2/in/variant-resourcedata-uri.xtm']
30 OLD_DETAIL = 'Only textual xsd:string resourceData is implemented.'
31 NEW_DETAIL = 'Only textual xsd:string and xsd:anyURI resourceData are implemented.'
32
33
34 def read(p):
35 return json.loads(p.read_text())
36
37
38 def save(p, v):
39 p.write_text(json.dumps(v, indent=2, ensure_ascii=False) + '\n')
40
41
42 def sha(p):
43 return hashlib.sha256(p.read_bytes()).hexdigest()
44
45
46 def command(args):
47 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
48
49
50 def prepare_pinned(target):
51 spec = importlib.util.spec_from_file_location('slice21g', ROOT / 'scripts/validate-slice-21i.py')
52 module = importlib.util.module_from_spec(spec)
53 spec.loader.exec_module(module)
54 return module.prepare_pinned(target)
55
56
57 def main():
58 parser = argparse.ArgumentParser(description=__doc__)
59 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
60 parser.add_argument('--output', type=Path)
61 parser.add_argument('--prepare', type=Path)
62 args = parser.parse_args()
63 if args.prepare:
64 print(json.dumps(prepare_pinned(args.prepare)))
65 return
66 runtime = shutil.which(args.sbcl)
67 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
68 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
69 output = args.output.resolve() if args.output else Path(tempfile.mkdtemp(prefix='topic-maps-21j-'))
70 assert not output.is_relative_to(ROOT)
71 output.mkdir(parents=True, exist_ok=True)
72 assert not (output / 'execution.json').exists(), 'Choose an unused output directory.'
73 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
74 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
75 protected = {p: sha(ROOT / p) for p in tracked if p not in ALLOWED}
76 historical = {p: h for p, h in protected.items() if p.startswith('reports/')}
77 sources = {p: sha(ROOT / p) for p in [*sorted(ALLOWED), 'tests/xtm-variant-uri-data.lisp', 'scripts/validate-slice-21j.py',
78 *['reports/slice-21j/' + p for p in ['candidate-inventory.json', 'failure-controls.lisp', 'diagnostics.lisp',
79 'before-diagnostics.json', 'baseline-xtm.json']]]}
80 previous = read(PRIOR / 'xtm.json')
81 assert read(EVIDENCE / 'baseline-xtm.json') == previous
82 assert previous['statuses'] == {'pass': 59, 'fail': 0, 'unsupported': 0, 'not-run': 0}
83 assert previous['otherCorpusInputs'] == {'positiveUnexecuted': 62, 'negativeUnregisteredUnexecuted': 3}
84 assert read(PRIOR / 'ctm.validation.json')['statuses'] == CTM
85 candidate = read(EVIDENCE / 'candidate-inventory.json')
86 assert [c['id'] for c in candidate['candidates']] == TARGETS
87 assert [c['sourceBytes'] for c in candidate['candidates']] == [405]
88 assert [c['baselineBytes'] for c in candidate['candidates']] == [621]
89 assert not set(TARGETS) & {c['id'] for c in previous['cases']}
90 assert read(EVIDENCE / 'before-diagnostics.json') == read(PRIOR / 'after-diagnostics.json')
91 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
92 runs = []
93
94 def run(label, invocation, env, expected_exit=0):
95 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
96 entry = {'label': label, 'command': invocation, 'network': 'denied', 'repositoryWrites': 'denied',
97 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
98 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=300)
99 (output / (label + '.log')).write_bytes(result.stdout)
100 entry.update(exitCode=result.returncode, expectedExitCode=expected_exit,
101 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
102 runs.append(entry)
103 assert result.returncode == expected_exit, f'{label}: see {output / (label + ".log")}'
104 return result.stdout.decode()
105
106 retained = output / 'retained'
107 retained.mkdir()
108 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
109 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
110 with tempfile.TemporaryDirectory(prefix='topic-maps-21j-sources-') as temp_name:
111 provenance = json.loads(run('source-preparation', ['python3', str(Path(__file__)), '--prepare', temp_name + '/sources'], env))
112 save(output / 'source-preparation.json', provenance)
113 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'], TOPIC_MAPS_REPORT=str(retained / 'core.json'),
114 TOPIC_MAPS_XTM_REPORT=str(output / 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
115 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
116 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
117 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
118 TOPIC_MAPS_ENCODING_REPORT=str(output / 'ctm'))
119 prefix = [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script']
120 xml = run('xml', prefix + ['tests/cxml-probe.lisp'], env)
121 xml_labels = re.findall(r'^CXML-CHECK-PASS (.+)$', xml, re.M)
122 assert len(xml_labels) == len(set(xml_labels)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
123 xtm_log = run('xtm', prefix + ['scripts/fresh-xtm.lisp'], env)
124 xtm = read(output / 'xtm.json')
125 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', xtm_log, re.M)
126 assert labels == xtm['checks'] and len(labels) == len(set(labels))
127 rename = {label: label.replace('resourceData-anyURI', 'resourceData-integer')
128 for label in previous['checks'] if label.startswith('variant-syntax:resourceData-anyURI:')}
129 assert len(rename) == 2
130 retained_labels = [rename.get(label, label) for label in previous['checks']]
131 assert set(retained_labels) <= set(labels)
132 assert [label for label in labels if label in set(retained_labels)] == retained_labels
133 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 52 original CXTM PASS; 8 negative originals PASS' in xtm_log
134 assert xtm['statuses'] == {'pass': 60, 'fail': 0, 'unsupported': 0, 'not-run': 0}
135 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 61, 'negativeUnregisteredUnexecuted': 3}
136 assert xtm['cases'][:59] == previous['cases']
137 control_retargets = {
138 'negativeControls': [('unsupported-is-not-pass', 'unsupported-is-not-pass')],
139 'nameControls': [('unsupported-variant-resourceData-datatype', 'unsupported-variant-resourceData-datatype')],
140 'scopeControls': [('scope:variant-resourceData-datatype-stays-unsupported', 'scope:variant-resourceData-datatype-stays-unsupported')],
141 'nameItemIdentityControls': [('iid:variant-resourceData-datatype-stays-unsupported', 'iid:variant-resourceData-datatype-stays-unsupported')],
142 'variantControls': [('resourceData-anyURI', 'resourceData-integer'), ('datatype', 'datatype')]}
143 for field, value in previous.items():
144 if field in control_retargets:
145 assert len(xtm[field]) == len(value)
146 changed = [(old, new) for old, new in zip(value, xtm[field]) if old != new]
147 assert [(old['label'], new['label']) for old, new in changed] == control_retargets[field], field
148 for old, new in changed:
149 assert new['condition']['class'] == 'XTM-UNSUPPORTED'
150 assert new['condition']['code'] == 'resource-data-datatype'
151 assert new['condition']['detail'] == NEW_DETAIL
152 if old['label'] == 'datatype':
153 expected = json.loads(json.dumps(old))
154 expected['condition']['detail'] = NEW_DETAIL
155 expected['condition']['message'] = expected['condition']['message'].replace(OLD_DETAIL, NEW_DETAIL)
156 assert expected == new
157 elif field not in ['scope', 'statuses', 'otherCorpusInputs', 'checks', 'cases']:
158 assert xtm[field] == value, field
159 assert len(xtm['cases']) == len({c['id'] for c in xtm['cases']}) == 60
160 assert [c['id'] for c in xtm['cases'][59:]] == TARGETS
161 assert xtm['cases'][59]['status'] == 'pass' and xtm['cases'][59]['actualOutcome'] == 'cxtm-match'
162 assert len(xtm['variantResourceDataURIOriginals']) == 1
163 regression = run('regression', [str(ROOT / 'scripts/test.sh')], env)
164 run('diagnostics', prefix + [str(EVIDENCE / 'diagnostics.lisp')],
165 dict(env, TOPIC_MAPS_21J_ROOT=str(ROOT), TOPIC_MAPS_21J_DIAGNOSTICS=str(output / 'after-diagnostics.json')))
166 fault_controls = []
167 positive_modes = [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]
168 negative_modes = [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')]
169 families = {'positive': positive_modes + [('xtm', 'execution-error')], 'negative': negative_modes,
170 'name': positive_modes, 'scope': positive_modes, 'iid': positive_modes, 'variant': positive_modes,
171 'variant-negative': negative_modes + [('tmdm', 'execution-error')],
172 'merge-variant': positive_modes + [('xtm', 'execution-error')],
173 'resource-ref': positive_modes + [('xtm', 'execution-error')],
174 'resource-ref-relative': positive_modes + [('xtm', 'execution-error')],
175 'uri-data': positive_modes + [('xtm', 'execution-error')]}
176 for family, modes in families.items():
177 for mode, outcome in modes:
178 label = family + '-control-' + mode
179 run(label, prefix + [str(EVIDENCE / 'failure-controls.lisp')],
180 dict(env, TOPIC_MAPS_XTM_REPORT=str(output / (label + '.json')), TOPIC_MAPS_21J_FAMILY=family, TOPIC_MAPS_21J_FAULT=mode), expected_exit=1)
181 result = read(output / (label + '.json'))
182 retained_count = {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48, 'variant': 50, 'variant-negative': 55, 'merge-variant': 56, 'resource-ref': 57, 'resource-ref-relative': 58, 'uri-data': 59}[family]
183 assert result['statuses']['pass'] == retained_count
184 assert len(result['cases']) == retained_count + 1 and result['cases'][-1]['actualOutcome'] == outcome
185 assert result['cases'][:retained_count] == xtm['cases'][:retained_count]
186 fault_controls.append({'family': family, 'mode': mode, 'actualOutcome': outcome,
187 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count, 'faultCountedAsPass': False})
188
189 verified = []
190 for old in sorted((PRIOR / 'retained').glob('*.json')):
191 assert read(old) == read(retained / old.name), str(old)
192 verified.append(str(old.relative_to(ROOT)))
193 for suffix in ['validation', 'cxtm', 'encoding']:
194 old = PRIOR / f'ctm.{suffix}.json'
195 assert read(old) == read(output / f'ctm.{suffix}.json'), str(old)
196 verified.append(str(old.relative_to(ROOT)))
197 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
198 stages = read(PRIOR / 'execution.json')['stageChecks']
199 assert sum(totals.values()) == 12100
200 for marker, count in dict(totals, **stages).items():
201 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
202 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
203 assert read(output / 'ctm.validation.json')['statuses'] == CTM
204 comparisons = []
205 (output / 'actual').mkdir()
206 for candidate_case, result in zip(candidate['candidates'], xtm['variantResourceDataURIOriginals']):
207 assert result['id'] == candidate_case['id'] and result['condition'] is None and result['status'] == 'pass'
208 source = ROOT / 'vendor/cxtm' / result['id']
209 baseline = ROOT / 'vendor/cxtm/xtm2/baseline' / (source.name + '.cxtm')
210 raw, actual = source.read_bytes(), result['actualCXTM'].encode('utf-8')
211 assert raw == bytes.fromhex(result['sourceHex']) == result['sourceText'].encode('utf-8')
212 assert sha(source) == candidate_case['sourceSHA256'] and len(raw) == result['sourceBytes'] == candidate_case['sourceBytes']
213 assert actual == result['expectedCXTM'].encode('utf-8') == baseline.read_bytes()
214 assert len(actual) == result['actualBytes'] == result['expectedBytes'] == candidate_case['baselineBytes']
215 assert sha(baseline) == candidate_case['baselineSHA256'] and result['firstDifferentByte'] is None
216 assert result['documentIRI'] == candidate_case['documentIRI'] and result['sourceKey'] == result['id']
217 assert result['model']['canonicalVariantReferences']
218 assert len(result['model']['variants']) == 1
219 variant = result['model']['variants'][0]
220 assert variant['value'] == result['absoluteValue'] and variant['datatype'] == 'http://www.w3.org/2001/XMLSchema#anyURI'
221 assert result['absoluteValue'].startswith(('http:', 'https:'))
222 (output / 'actual' / baseline.name).write_bytes(actual)
223 comparisons.append(dict(result, sourceSHA256=sha(source), baselineSHA256=sha(baseline),
224 actualSHA256=hashlib.sha256(actual).hexdigest(), executedInASDFTestOp=True))
225 before, after = read(EVIDENCE / 'before-diagnostics.json'), read(output / 'after-diagnostics.json')
226 assert len(before) == len(after) == 124
227 assert [c['id'] for c in before] == [c['id'] for c in after]
228 changes = []
229 for b, a in zip(before, after):
230 if b == a:
231 continue
232 if a['id'] in TARGETS:
233 assert b['condition']['class'] == 'XTM-UNSUPPORTED' and b['condition']['code'] == 'resource-data-datatype'
234 assert a['condition'] is None and a['byteExact']
235 assert a['actualCXTM'] == comparisons[0]['actualCXTM']
236 reason = 'anyURI resourceData text now uses the existing XTM IRI resolver and add-variant API.'
237 else:
238 assert a['id'] == 'xtm2/in/variant-datatype-unknown.xtm'
239 expected = json.loads(json.dumps(b))
240 expected['condition']['detail'] = NEW_DETAIL
241 expected['condition']['message'] = expected['condition']['message'].replace(OLD_DETAIL, NEW_DETAIL)
242 assert a == expected
243 reason = 'Unsupported diagnostic wording now lists xsd:string and xsd:anyURI; class, code, location and rejection unchanged.'
244 changes.append({'id': a['id'], 'before': b, 'after': a, 'reason': reason})
245 assert {c['id'] for c in changes} == {*TARGETS, 'xtm2/in/variant-datatype-unknown.xtm'}
246 save(output / 'diagnostic-changes.json', changes)
247 lock = read(ROOT / 'references/corpus-lock.json')
248 for entry in lock['files']:
249 p = ROOT / 'vendor/cxtm' / entry['path']
250 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
251 for snapshot in [protected, historical, sources]:
252 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
253 assert set(command(['git', 'diff', '--name-only', BASE]).splitlines()) & set(tracked) <= ALLOWED
254 save(output / 'variant-uri-data-original.json', {'kind': 'executed-original-variant-uri-data-registration', 'cases': comparisons})
255 save(output / 'diagnostics.json', {'kind': 'read-only-observations-no-extra-registration',
256 'originalObservations': 124, 'unchangedOriginalObservations': 124 - len(changes),
257 'diagnosticChanges': changes, 'registeredCases': TARGETS,
258 'outOfScopeUnknownDatatypeStillUnsupported': True})
259 save(output / 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls})
260 save(output / 'integrity.json', {'protectedBaseFilesSHA256': protected, 'historicalReportsSHA256': historical, 'executionSourcesSHA256': sources})
261 statuses = dict(xtm['statuses'], **{'not-run': sum(xtm['otherCorpusInputs'].values())})
262 save(output / 'execution.json', {'kind': 'xtm-variant-uri-data-original-registration-validation', 'baseCommit': BASE,
263 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started,
264 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runs': runs,
265 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals,
266 'stageChecks': stages, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels),
267 'retainedXTMChecks': len(previous['checks']), 'retargetedUnsupportedCheckNames': rename,
268 'newXTMChecks': len(labels) - len(previous['checks']), 'retargetedUnsupportedControlRows': control_retargets,
269 'retainedOriginals': 59, 'retainedOriginalResultsUnchanged': True, 'newPositiveOriginals': 1,
270 'originalByteComparisons': [{'id': c['id'], 'actual': c['actualBytes'], 'baseline': c['expectedBytes'], 'firstDifferentByte': None} for c in comparisons], 'faultControls': len(fault_controls),
271 'diagnosticOriginalObservations': 124, 'diagnosticChangedOriginals': [c['id'] for c in changes],
272 'xtmStatusesBefore': {'pass': 59, 'fail': 0, 'unsupported': 0, 'not-run': 65}, 'xtmStatusesAfter': statuses,
273 'xtmStatusChanges': [{'id': id, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for id in TARGETS],
274 'ctmStatusesBefore': CTM, 'ctmStatusesAfter': CTM, 'ctmStatusChanges': [],
275 'knownFailuresUnchanged': [c['id'] for c in read(output / 'ctm.validation.json')['cases'] if c['status'] == 'fail'],
276 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsIdenticalToSlice21I': verified,
277 'historicalReportFilesVerified': len(historical), 'protectedBaseFilesVerified': len(protected), 'pinnedCorpusFilesVerified': len(lock['files']),
278 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True, 'productionChangeLimitedToXTMReader': True, 'protectedSubsystemsUnchanged': ['TMDM', 'CTM', 'CXTM', 'XML'],
279 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
280 'executionAuthorization': 'Regular require_escalated approval; tests additionally deny network and repository writes; no symlink/configuration changes.',
281 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
282 print(json.dumps({'acceptancePassed': True, 'xtm': statuses, 'ctm': CTM, 'checks': len(labels), 'output': str(output)}))
283
284
285 if __name__ == '__main__':
286 main()