]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-20f.py
feat(ctm): decode ISO-8859-5 with qualified charset aliases
[topic-maps.git] / scripts / validate-slice-20f.py
1 #!/usr/bin/env python3
2 """Validate the CTM-integer literal, original contract and all retained suites offline."""
3 import argparse
4 import datetime
5 import importlib.util
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = '341cfa2306788e2943bdf11ced5ee32a8ba74894'
16 ID = 'ctm/in/occurrence-ctm-integer.ctm'
17 POLICY = '(version 1)(allow default)(deny network*)'
18 SOURCE = b'topic\nocctype: *;.'
19 EXPECTED = (ROOT / 'vendor/cxtm/ctm/baseline/occurrence-ctm-integer.ctm.cxtm').read_bytes()
20
21
22 def main():
23 # Reuse existing report/hash utilities and historical label qualification.
24 def previous(name):
25 spec = importlib.util.spec_from_file_location(name, ROOT / ('scripts/validate-' + name + '.py'))
26 module = importlib.util.module_from_spec(spec)
27 spec.loader.exec_module(module)
28 return module
29 common = previous('slice-20a')
30 read, save, sha, command, snapshot = common.read, common.save, common.sha, common.command, common.snapshot
31 parser = argparse.ArgumentParser(description=__doc__)
32 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
33 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-20f')
34 parser.add_argument('--preimplementation', type=Path, default=ROOT / 'reports/slice-20f/preimplementation.json')
35 args = parser.parse_args()
36 preimplementation = read(args.preimplementation)
37 assert preimplementation['baseCommit'] == BASE and len(preimplementation['cases']) == 24
38 runtime = shutil.which(args.sbcl)
39 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
40 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
41 mutable = {'src/ctm/lexer.lisp', 'src/ctm/reader.lisp', 'src/ctm/templates.lisp',
42 'tests/ctm-registered-originals.lisp', 'tests/ctm-occurrences.lisp', 'tests/ctm-integers.lisp',
43 'tests/ctm-prolog.lisp', 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
44 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
45 protected_paths = [p for p in tracked if p not in mutable]
46 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
47 output = args.output.resolve()
48 output.mkdir(parents=True, exist_ok=False)
49 retained = output / 'retained'
50 retained.mkdir()
51 save(output / 'preimplementation.json', preimplementation)
52 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
53 protected = snapshot(ROOT / p for p in protected_paths)
54 historical = snapshot(p for p in (ROOT / 'reports').rglob('*') if output not in p.parents)
55 sources = snapshot(p for directory in ['src', 'tests', 'scripts', 'docs']
56 for p in (ROOT / directory).rglob('*') if p.suffix in ['.lisp', '.py', '.sh', '.md'])
57 sources.update(snapshot(ROOT / p for p in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
58 source = ROOT / 'vendor/cxtm' / ID
59 baseline = ROOT / 'vendor/cxtm/ctm/baseline/occurrence-ctm-integer.ctm.cxtm'
60 assert source.read_bytes() == SOURCE and baseline.read_bytes() == EXPECTED
61 assert len(EXPECTED) == 356 and common.digest(EXPECTED) == '51c024274bb50d36bd583998ee26dd120a177fa010bbf5259dc1bc18f045c190'
62 runs = []
63 with tempfile.TemporaryDirectory(prefix='topic-maps-20f-') as temp_name:
64 temp = Path(temp_name)
65 preparation = ['python3', 'scripts/prepare-cxml.py', '--directory', str(temp / 'sources')]
66 provenance = json.loads(command(preparation)) # nix eval --offline; verifies local Store source hashes.
67 save(output / 'source-preparation.json', provenance)
68 env = dict(os.environ, SBCL=runtime, GIT_OPTIONAL_LOCKS='0', PYTHONDONTWRITEBYTECODE='1',
69 CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
70 TOPIC_MAPS_REPORT=str(retained / 'core.json'),
71 TOPIC_MAPS_XTM_REPORT=str(retained / 'xtm.json'),
72 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
73 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
74 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
75 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'),
76 TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
77 TOPIC_MAPS_CTM_INTEGER_REPORT=str(output / 'current'))
78 invocations = [
79 ('xml', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
80 ('xtm', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
81 ('regression', [str(ROOT / 'scripts/test.sh')]),
82 ]
83 for label, invocation in invocations:
84 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY] + invocation
85 run = {'label': label, 'command': invocation, 'freshProcess': True,
86 'network': 'denied by sandbox-exec',
87 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
88 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
89 stderr=subprocess.STDOUT, timeout=300)
90 (output / (label + '.log')).write_bytes(result.stdout)
91 run.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
92 runs.append(run)
93 assert result.returncode == 0, f'{label} failed; see {label}.log'
94 if label in ['xml', 'xtm']:
95 marker = 'CXML' if label == 'xml' else 'XTM'
96 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', result.stdout.decode(), re.M)
97 assert len(labels) == len(set(labels)) == (146 if label == 'xml' else 327)
98 assert f'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result.stdout.decode()
99 run['passedChecks'] = len(labels)
100 # Historical report bytes remain immutable. Regenerated reports permit only
101 # explicitly reviewed replacements of obsolete '*' unsupported assertions.
102 label_changes = {
103 'occurrence-diagnostic:a t:*.': 'occurrence-diagnostic:a t:*x.',
104 'excluded-numeric-sugar:*': 'integer-slice-now-allows-ctm-integer',
105 'known-literal-unsupported:specific-condition-code-position-source':
106 'known-triple-string-unsupported:specific-condition-code-position-source',
107 'known-literal-unsupported:position-units': 'known-triple-string-unsupported:position-units',
108 }
109 old_prolog = read(ROOT / 'reports/slice-20e/retained/slice20c.prolog.json')
110 old_literal = next(r for r in old_prolog['negativeControls'] if r['label'] == 'known-literal-unsupported')
111 new_condition = dict(old_literal['condition'],
112 code='triple-quoted-string', source='known-triple-string-unsupported',
113 message='TRIPLE-QUOTED-STRING in known-triple-string-unsupported at CHARACTER 11 (line 1, column 12): Triple quoted literals are outside Slice 1.')
114 def qualified_retained(value):
115 if isinstance(value, dict):
116 if value.get('label') == 'known-literal-unsupported':
117 assert value == old_literal
118 return {'label': 'known-triple-string-unsupported', 'condition': new_condition}
119 return {k: qualified_retained(v) for k, v in value.items()}
120 if isinstance(value, list):
121 return [qualified_retained(v) for v in value]
122 if isinstance(value, str):
123 return label_changes.get(value, value)
124 return value
125 verified = []
126 for old in sorted((ROOT / 'reports/slice-20e/retained').glob('*.json')):
127 assert qualified_retained(read(old)) == read(retained / old.name), str(old)
128 verified.append(str(old.relative_to(ROOT)))
129 for suffix in ['validation', 'cxtm', 'registration']:
130 old = ROOT / f'reports/slice-20e/current.{suffix}.json'
131 assert qualified_retained(read(old)) == read(retained / f'slice20e.{suffix}.json'), str(old)
132 verified.append(str(old.relative_to(ROOT)))
133 before = read(retained / 'slice20e.validation.json')
134 final = read(output / 'current.validation.json')
135 literal = read(output / 'current.literal.json')
136 cxtm = read(output / 'current.cxtm.json')
137 assert before['statuses'] == literal['entryStatuses'] == {
138 'pass': 256, 'fail': 4, 'unsupported': 0, 'not-run': 13}
139 assert final['statuses'] == {'pass': 257, 'fail': 4, 'unsupported': 0, 'not-run': 12}
140 assert literal['passed'] == len(literal['checks']) == len(set(literal['checks']))
141 assert literal['failed'] == 0
142 row = literal['original']
143 assert row['id'] == ID and row['sourceBytes'] == 18
144 assert row['status'] == 'pass' and row['actualOutcome'] == 'cxtm-match' and row['diagnostics'] == []
145 assert row['comparison']['actualBytes'] == row['comparison']['expectedBytes'] == 356
146 assert row['comparison']['firstDifferentByte'] is None
147 assert row['comparison']['model']['topicCount'] == 2 and row['comparison']['model']['associationCount'] == 0
148 model_occurrences = [o for t in row['comparison']['model']['topics'] for o in t['occurrences']]
149 assert len(model_occurrences) == 1
150 assert model_occurrences[0]['value'] == '*' and model_occurrences[0]['datatype'] == 'http://psi.topicmaps.org/iso13250/ctm-integer'
151 changes = []
152 assert len(before['cases']) == len(final['cases']) == 273
153 for a, b in zip(before['cases'], final['cases']):
154 assert a['id'] == b['id']
155 if a['id'] != ID:
156 assert a == b, a['id']
157 continue
158 assert a['category'] == 'valid' and a['expectedOutcome'] == 'cxtm-equal'
159 assert a['status'] == 'not-run' and a['actualOutcome'] is None
160 assert a['documentIRI'] == row['documentIRI'] == (
161 'https://ctm-tests.example.invalid/' + before['commit'] + '/' + ID)
162 assert b == dict(a, status='pass', actualOutcome='cxtm-match')
163 changes.append({'id': ID, 'before': a['status'], 'after': b['status']})
164 assert len(changes) == 1
165 assert {k: v for k, v in before.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
166 k: v for k, v in final.items() if k not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
167 old_cxtm = read(retained / 'slice20e.cxtm.json')
168 assert [c for c in cxtm['cases'] if c['id'] != ID] == old_cxtm['cases']
169 assert cxtm['contractCounts'] == {'pass': 219, 'fail': 4}
170 for case in cxtm['cases']:
171 pinned = ROOT / 'vendor/cxtm/ctm/baseline' / (Path(case['id']).name + '.cxtm')
172 actual = case['actualCXTM'].encode('utf-8')
173 assert (actual == pinned.read_bytes()) == (case['status'] == 'pass')
174 assert case['expectedCXTM'].encode('utf-8') == pinned.read_bytes()
175 comparison = next(c for c in cxtm['cases'] if c['id'] == ID)
176 assert comparison['actualCXTM'].encode() == comparison['expectedCXTM'].encode() == EXPECTED
177 positives = {p['label']: p for p in literal['positives']}
178 assert len(positives) == len(literal['positives']) == 31
179 qualified = positives['original:qualified-control']
180 assert qualified['actualCXTM'].encode() == EXPECTED
181 for label, positive in positives.items():
182 assert positive['finalCounter'] == (44 if label.startswith('counter:interleaved') else positive['initialCounter'])
183 assert positives['equivalence:bare']['actualCXTM'] == positives['equivalence:qualified']['actualCXTM']
184 assert positives['equivalence:bare']['actualCXTM'] != positives['distinction:string']['actualCXTM']
185 assert positives['variant:bare']['actualCXTM'] == positives['variant:qualified']['actualCXTM']
186 negatives = {r['label']: r for r in literal['negativeControls']}
187 assert len(negatives) == len(literal['negativeControls']) == 26
188 diagnostic_changes = []
189 for prior in preimplementation['cases']:
190 current = negatives[prior['label']]
191 assert current['sourceText'] == prior['sourceText']
192 old, new = prior['result'], current['condition']
193 keys = ['condition', 'code', 'position', 'positionUnit', 'line', 'column', 'source']
194 if prior['label'] in ['qualified-bare', 'missing-separator']:
195 expected_position = 16 if prior['label'] == 'qualified-bare' else 17
196 assert old['condition'] == 'CTM-UNSUPPORTED' and old['code'] == 'occurrence-literal-datatype'
197 assert new['condition'] == 'CTM-REJECTION' and new['code'] == 'expected-separator'
198 assert new['position'] == expected_position and new['line'] == 2 and new['column'] == expected_position - 5
199 diagnostic_changes.append({'label': prior['label'], 'before': old, 'after': new,
200 'reason': "The now-valid isolated '*' reaches the following forbidden token."})
201 else:
202 assert {k: old[k] for k in keys} == {k: new[k] for k in keys}, prior['label']
203 assert len(diagnostic_changes) == 2
204 log = (output / 'regression.log').read_text()
205 prior_totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
206 totals = {marker: int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for marker in prior_totals}
207 assert totals == prior_totals and sum(totals.values()) == 12100
208 for marker, count in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286),
209 ('REGISTERED-NEGATIVES', 25), ('PROLOG', 171), ('EMPTY2', 13), ('CTM-INTEGER', literal['passed'])]:
210 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
211 assert log.count('CTM-INTEGER-ORIGINAL ctm/in/occurrence-ctm-integer.ctm: PASS; 356 byte-exact CXTM bytes') == 1
212 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
213 xtm = read(retained / 'xtm.json')
214 assert xtm == read(ROOT / 'reports/slice-19d2/xtm.json')
215 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
216 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
217 lock = read(ROOT / 'references/corpus-lock.json')
218 for entry in lock['files']:
219 path = ROOT / 'vendor/cxtm' / entry['path']
220 assert path.stat().st_size == entry['size'] and sha(path) == entry['sha256'], str(path)
221 assert protected == {p: sha(ROOT / p) for p in protected}
222 assert historical == {p: sha(ROOT / p) for p in historical}
223 assert sources == {p: sha(ROOT / p) for p in sources}
224 assert not command(['git', 'diff', BASE, '--'] + protected_paths)
225 save(output / 'remaining.json', [c for c in final['cases'] if c['status'] == 'not-run'])
226 save(output / 'integrity.json', {'protectedFilesSHA256': protected, 'historicalReportsSHA256': historical})
227 evidence = {
228 'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
229 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
230 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources, 'newDependencyDownloads': 0,
231 'sourcePreparationCommand': preparation, 'qualifiedSourceProvenance': provenance, 'runs': runs,
232 'retainedLispChecks': sum(totals.values()), 'retainedChecks': totals,
233 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
234 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': 13, 'ctmIntegerChecks': literal['passed'],
235 'positiveControls': len(positives), 'negativeControls': len(negatives),
236 'unchangedDiagnosticSignatures': 22, 'newlyReachedDiagnostics': diagnostic_changes,
237 'retainedReportLabelChanges': label_changes,
238 'updatedBoundaryAssertion': 'occurrence-feature-boundary:a t:* . now checks acceptance',
239 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
240 'ctmStatusesBefore': before['statuses'], 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': changes,
241 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
242 'knownFailuresUnchanged': [c['id'] for c in final['cases'] if c['status'] == 'fail'],
243 'retainedReportsVerified': verified, 'allHistoricalReportBytesUnchanged': True,
244 'allProtectedFileBytesUnchanged': True, 'pinnedCorpusFilesVerified': len(lock['files']),
245 'originalComparison': {
246 'id': ID, 'documentIRI': row['documentIRI'], 'sourceBytes': len(SOURCE), 'sourceSHA256': sha(source),
247 'expectedBytes': len(EXPECTED), 'expectedSHA256': sha(baseline),
248 'actualBytes': len(EXPECTED), 'actualSHA256': common.digest(comparison['actualCXTM'].encode()),
249 'firstDifferentByte': None, 'topics': 2, 'associations': 0,
250 'value': '*', 'datatype': 'http://psi.topicmaps.org/iso13250/ctm-integer',
251 'qualifiedControlBytes': len(qualified['actualCXTM'].encode()),
252 'qualifiedControlSHA256': common.digest(qualified['actualCXTM'].encode()),
253 'canonicalizationPreservesMap': True, 'executedInFullRegression': True,
254 },
255 'persistentLispImagesTouched': False,
256 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
257 'sandboxStartError': 'writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.',
258 'executionAuthorization': 'Regular require_escalated auto-review approval; no symlink or sandbox configuration changes.',
259 'acceptancePassed': True,
260 }
261 save(output / 'execution.json', evidence)
262 print(json.dumps({k: evidence[k] for k in [
263 'runtime', 'retainedLispChecks', 'xmlQualificationChecks', 'standaloneXTMChecks', 'xtmMergemapChecks',
264 'slice20aChecks', 'slice20bChecks', 'prologChecks', 'empty2Checks', 'ctmIntegerChecks', 'positiveControls', 'negativeControls', 'ctmStatusesAfter',
265 'xtmCorpusStatuses', 'originalComparison', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent=2))
266
267
268 if __name__ == '__main__':
269 main()