]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-20j/reproduce.py
perf(tmdm): index topic normalization merge candidates
[topic-maps.git] / reports / slice-20j / reproduce.py
1 #!/usr/bin/env python3
2 """Reproduce documentary observations and the unchanged offline regression."""
3 import argparse
4 import datetime
5 import hashlib
6 import importlib.util
7 import json
8 import os
9 from pathlib import Path
10 import re
11 import shutil
12 import subprocess
13 import tempfile
14
15 ROOT = Path(__file__).resolve().parents[2]
16 EVIDENCE = Path(__file__).resolve().parent
17 BASE = 'bc04a1657f0b548ec5cdf2862aeed6a1ad0630b8'
18 POLICY = '(version 1)(allow default)(deny network*)'
19 READ_ONLY = POLICY + '(deny file-write* (subpath "' + str(ROOT) + '"))'
20
21
22 def read(path):
23 return json.loads(path.read_text())
24
25
26 def save(path, value):
27 path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
28
29
30 def sha(path):
31 return hashlib.sha256(path.read_bytes()).hexdigest()
32
33
34 def command(args, **kwargs):
35 return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
36
37
38 def load_module(name, path):
39 spec = importlib.util.spec_from_file_location(name, path)
40 module = importlib.util.module_from_spec(spec)
41 spec.loader.exec_module(module)
42 return module
43
44
45 def prepare_pinned(target):
46 # Reuse the existing verifier/patcher with the already recorded Store
47 # manifest. No Nix evaluation or daemon/network connection is needed.
48 prior = read(ROOT / 'reports/slice-20h/source-preparation.json')
49 module = load_module('prepare_cxml', ROOT / 'scripts/prepare-cxml.py')
50 environment = {'manifest': prior['sources'],
51 'localPatches': [{'package': p['package'], 'sha256': p['sha256']} for p in prior['patches']]}
52 return module.prepare(environment, target)
53
54
55 def main():
56 parser = argparse.ArgumentParser(description=__doc__)
57 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
58 parser.add_argument('--output', type=Path, default=EVIDENCE)
59 parser.add_argument('--prepare', type=Path)
60 args = parser.parse_args()
61 if args.prepare:
62 print(json.dumps(prepare_pinned(args.prepare)))
63 return
64 runtime = shutil.which(args.sbcl)
65 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
66 subprocess.run(['git', 'merge-base', '--is-ancestor', BASE, 'HEAD'], cwd=ROOT, check=True)
67 output = args.output.resolve()
68 output.mkdir(parents=True, exist_ok=True)
69 assert not (output / 'execution.json').exists(), 'Choose an unused output directory'
70 tracked = command(['git', 'ls-tree', '-r', '--name-only', BASE]).splitlines()
71 assert not command(['git', 'diff', BASE, '--'] + tracked)
72 protected = {p: sha(ROOT / p) for p in tracked}
73 historical = {str(p.relative_to(ROOT)): sha(p) for p in (ROOT / 'reports').rglob('*')
74 if p.is_file() and output not in p.parents}
75 inputs = {str(p.relative_to(ROOT)): sha(p) for p in [*EVIDENCE.glob('*.py'), *EVIDENCE.glob('*.lisp'),
76 EVIDENCE / 'import.json', ROOT / 'docs/slice-20j-template-tail-grammar.md', *sorted((EVIDENCE / 'slice20i').glob('*'))]}
77 imported = read(EVIDENCE / 'import.json')
78 assert imported['baseCommit'] == BASE
79 for name, record in imported['files'].items():
80 path = EVIDENCE / 'slice20i' / name
81 assert path.stat().st_size == record['bytes'] and sha(path) == record['sha256']
82 old = read(EVIDENCE / 'slice20i/investigation.json')
83 old_controls = read(EVIDENCE / 'slice20i/controls.json')
84 detailed = read(EVIDENCE / 'slice20i/detailed-inventory.json')
85 assert read(EVIDENCE / 'slice20i/before-sha256.json') == protected
86 assert old['harnessCaseStatesUnchanged'] and len(old['cases']) == len(detailed['cases']) == 7
87 retained = output / 'retained'
88 retained.mkdir()
89 runs = []
90 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
91
92 def run(label, invocation, env, policy=POLICY):
93 invocation = ['/usr/bin/sandbox-exec', '-p', policy] + invocation
94 entry = {'label': label, 'command': invocation, 'network': 'denied',
95 'freshProcess': True, 'startedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
96 result = subprocess.run(invocation, cwd=ROOT, env=env, stdout=subprocess.PIPE,
97 stderr=subprocess.STDOUT, timeout=300)
98 (output / (label + '.log')).write_bytes(result.stdout)
99 entry.update(exitCode=result.returncode, finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat())
100 runs.append(entry)
101 assert result.returncode == 0, f'{label}: see {label}.log'
102 return result.stdout.decode()
103
104 with tempfile.TemporaryDirectory(prefix='topic-maps-20j-') as temp_name:
105 temp = Path(temp_name)
106 env = {k: v for k, v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
107 env.update(PYTHONDONTWRITEBYTECODE='1', GIT_OPTIONAL_LOCKS='0', SBCL=runtime)
108 provenance = json.loads(run('source-preparation',
109 ['python3', str(Path(__file__)), '--prepare', str(temp / 'sources')], env, READ_ONLY))
110 save(output / 'source-preparation.json', provenance)
111 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'],
112 TOPIC_MAPS_REPORT=str(retained / 'core.json'), TOPIC_MAPS_XTM_REPORT=str(retained / 'xtm.json'),
113 TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained / 'xtm-mergemap'),
114 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained / 'slice20a'),
115 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained / 'slice20b'),
116 TOPIC_MAPS_PROLOG_REPORT=str(retained / 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT=str(retained / 'slice20e'),
117 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained / 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT=str(retained / 'slice20g'),
118 TOPIC_MAPS_ENCODING_REPORT=str(output / 'current'))
119 for label, script, marker, count in [('xml', 'tests/cxml-probe.lisp', 'CXML', 146),
120 ('xtm', 'scripts/fresh-xtm.lisp', 'XTM', 327)]:
121 log = run(label, [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', script], env)
122 labels = re.findall(r'^' + marker + r'-CHECK-PASS (.+)$', log, re.M)
123 assert len(labels) == len(set(labels)) == count
124 assert f'{marker}-SUMMARY {count} PASS 0 FAIL' in log
125 log = run('regression', [str(ROOT / 'scripts/test.sh')], env)
126 obs_env = {k: v for k, v in env.items() if not k.startswith('TOPIC_MAPS_')}
127 obs_env.update(TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(temp / 'probe-xtm'),
128 TOPIC_MAPS_20J_ROOT=str(ROOT), TOPIC_MAPS_20J_OUTPUT=str(temp))
129 driver = temp / 'observe.lisp'
130 driver.write_text('(load ' + json.dumps(str(EVIDENCE / 'observations.lisp')) + ')\n'
131 '(load ' + json.dumps(str(EVIDENCE / 'controls.lisp')) + ')\n')
132 run('observations', [runtime, '--noinform', '--no-sysinit', '--no-userinit', '--script', str(driver)], obs_env, READ_ONLY)
133 current = read(temp / 'investigation.json')
134 controls = read(temp / 'controls.json')
135 assert current == old, 'Original observations differ from verified Slice 20I'
136 assert controls == old_controls, 'Explorative controls differ from verified Slice 20I'
137 save(output / 'observed-originals.json', current)
138 save(output / 'observed-controls.json', controls)
139
140 verified_reports = []
141 for prior in sorted((ROOT / 'reports/slice-20h/retained').glob('*.json')):
142 assert read(prior) == read(retained / prior.name), str(prior)
143 verified_reports.append(str(prior.relative_to(ROOT)))
144 for suffix in ['validation', 'cxtm', 'encoding']:
145 prior = ROOT / f'reports/slice-20h/current.{suffix}.json'
146 assert read(prior) == read(output / f'current.{suffix}.json'), str(prior)
147 verified_reports.append(str(prior.relative_to(ROOT)))
148 final = read(output / 'current.validation.json')
149 expected_statuses = {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
150 assert final['statuses'] == current['harnessStatuses'] == expected_statuses
151 pending = [c for c in final['cases'] if c['status'] == 'not-run']
152 assert [c['id'] for c in pending] == [c['id'] for c in current['cases']]
153 cases = []
154 for original, detail in zip(current['cases'], detailed['cases']):
155 assert original['id'] == detail['id']
156 path = ROOT / 'vendor/cxtm' / original['id']
157 raw = path.read_bytes()
158 assert raw == bytes.fromhex(original['sourceHex']) == original['sourceText'].encode('ascii')
159 assert len(raw) == original['sourceBytes'] and sha(path) == detail['sourceSHA256']
160 matches = list(re.finditer(r'\$[A-Za-z_][A-Za-z0-9_]*', original['sourceText']))
161 assert len(matches) == len(detail['variables'])
162 variables = []
163 for match, variable in zip(matches, detail['variables']):
164 pos = match.start()
165 assert match.group() == variable['variable'] and pos == variable['position']
166 assert variable['line'] == raw[:pos].count(b'\n') + 1
167 assert variable['column'] == pos - raw.rfind(b'\n', 0, pos)
168 variables.append(dict(variable, roleOrigin='observed' if variable['role'] == 'Parameterdeklaration' else 'inferred'))
169 condition = original['condition']
170 assert condition['condition'] == 'INVALID-TEMPLATE-BODY' and condition['code'] == 'template-identity-tail'
171 assert condition['positionUnit'] == 'character' and condition['cause'] is None and condition['context'] is None
172 assert original['model'] is None and original['actualCXTM'] is None
173 case = dict(detail, status='not-run', variables=variables, observation=original,
174 originalComparisonPerformed=False, argumentCheckReached=False,
175 evidenceOrigins={'sourceAndCondition': 'observed', 'intendedRoles': 'inferred',
176 'grammar': 'norm-derived', 'compatibilityRule': 'hypothesized'},
177 compatibilityRule='H-SI-tail', normativeReferences=['N-tail', 'N-variables', 'N-templates'])
178 if original['baselineText']:
179 baseline = Path(original['baselinePath'])
180 assert baseline.read_bytes() == original['baselineText'].encode('utf-8') and sha(baseline) == detail['baselineSHA256']
181 control = detail['expandedControl']
182 assert control in controls and control['byteExact'] and control['condition'] is None
183 actual = control['actualCXTM'].encode('utf-8')
184 assert actual == baseline.read_bytes() and len(actual) == control['actualBytes'] == control['expectedBytes']
185 case['expandedComparison'] = {'epistemicOrigin': 'observed', 'originalTemplateExecutedSuccessfully': False,
186 'actualBytes': len(actual), 'expectedBytes': len(actual), 'actualSHA256': hashlib.sha256(actual).hexdigest(),
187 'expectedSHA256': sha(baseline), 'firstDifferentByte': None}
188 else:
189 second = '2nd-arg' in original['id']
190 case['intendedArgumentCheck'] = {'epistemicOrigin': 'inferred', 'argumentIndex': 2 if second else 1,
191 'parameter': '$iri2' if second else '$iri', 'identityKind': 'item-identifier' if 'item-identifier' in original['id'] else 'subject-locator',
192 'targetContext': 'unmarked Subject Identifier Tail slot',
193 'basis': 'invalid corpus category, source argument marker and, for two arguments, original invalid comment'}
194 cases.append(case)
195 report = {'kind': 'ctm-template-tail-investigation', 'baseCommit': BASE, 'runtime': 'SBCL 2.5.4',
196 'corpusCommit': final['commit'], 'statuses': expected_statuses, 'caseStatusChanges': [],
197 'originalComparisonsPerformed': 0, 'expandedControlComparisons': 3,
198 'normativeReference': detailed['normativeReference'],
199 'normativeFindings': [
200 {'id': 'N-tail', 'epistemicOrigin': 'norm-derived', 'sections': {'3.9': [50, 51, 52], '3.3.6': [17], '3.3.5': [12]},
201 'derivation': '[50] -> [51] -> [52] -> [17] -> [12]', 'result': 'No variable alternative in this unmarked Tail slot.'},
202 {'id': 'N-variables', 'epistemicOrigin': 'norm-derived', 'sections': {'3.3.6': [13, 18, 19], '3.3.7': [20], '3.4.1': [28]},
203 'result': 'Explicit variable uses in Topic identity/reference, marked SL/II and literal contexts.'},
204 {'id': 'N-templates', 'epistemicOrigin': 'norm-derived', 'sections': {'3.14': [63, 64, 65, 66, 67], '3.15': [68, 69, 70]},
205 'result': 'Template-only variables, positional binding and use-context constraints.'}],
206 'compatibilityHypothesis': {'id': 'H-SI-tail', 'epistemicOrigin': 'hypothesized', 'implemented': False,
207 'scope': 'Only the Subject Identifier branch of [52] inside a Template Tail.',
208 'additionalRule': 'template-tail-subject-identifier ::= iri-ref | variable',
209 'bindingRequirement': 'Variable must denote an IRI reference (IRI or resolved QName), added as an SI of the owner.',
210 'rejectedBindings': ['marked item identifier', 'marked subject locator', 'Topic-only argument', 'non-IRI literal'],
211 'identityKindsMustBePreserved': True, 'textualMarkerStrippingAllowed': False,
212 'doesNotChange': ['Topic-head variables', 'Type/Player Topic references', 'literal substitution'],
213 'historicalIntentEstablished': False},
214 'historicalQualification': detailed['historicalQualification'], 'cases': cases}
215 save(output / 'current.investigation.json', report)
216 totals = read(ROOT / 'reports/slice-19d1/regression-execution.json')['regressionChecks']
217 assert sum(totals.values()) == 12100
218 for marker, count in totals.items():
219 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
220 stage_counts = {'XTM-MERGEMAP': 217, 'REGISTERED-ORIGINALS': 286, 'REGISTERED-NEGATIVES': 25,
221 'PROLOG': 171, 'EMPTY2': 13, 'CTM-INTEGER': 112, 'TRIPLE': 226, 'ENCODING': 722}
222 for marker, count in stage_counts.items():
223 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) == count
224 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log
225 xtm = read(retained / 'xtm.json')
226 assert xtm['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
227 assert xtm['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
228 lock = read(ROOT / 'references/corpus-lock.json')
229 for entry in lock['files']:
230 p = ROOT / 'vendor/cxtm' / entry['path']
231 assert p.stat().st_size == entry['size'] and sha(p) == entry['sha256']
232 for snapshot in [protected, historical, inputs]:
233 assert snapshot == {p: sha(ROOT / p) for p in snapshot}
234 assert not command(['git', 'diff', BASE, '--'] + tracked)
235 save(output / 'integrity.json', {'baseTrackedFilesSHA256': protected, 'historicalReportsSHA256': historical,
236 'documentaryInputsSHA256': inputs})
237 save(output / 'execution.json', {'kind': 'ctm-template-tail-documentation-validation', 'baseCommit': BASE,
238 'branch': command(['git', 'branch', '--show-current']), 'startedAtUTC': started,
239 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'runtime': 'SBCL 2.5.4',
240 'runs': runs, 'sourcePreparation': provenance, 'retainedLispChecks': 12100, 'retainedChecks': totals,
241 'stageChecks': stage_counts, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327,
242 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'originalObservationsReproduced': 7,
243 'explorativeControlsReproduced': len(controls), 'expandedBaselineComparisons': 3,
244 'ctmStatusesBefore': expected_statuses, 'ctmStatusesAfter': final['statuses'], 'ctmStatusChanges': [],
245 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
246 'allRetainedReportsEqualToSlice20H': verified_reports, 'allBaseTrackedFileBytesUnchanged': True,
247 'baseTrackedFilesVerified': len(protected), 'historicalReportsVerified': len(historical),
248 'pinnedCorpusFilesVerified': len(lock['files']), 'newDependencyDownloads': 0,
249 'languageOrRegistrationChanges': False, 'persistentLispImagesTouched': False,
250 'fullOfflineNixRebuild': 'NOT-RUN; verified local pinned Store sources and unchanged XML patches.',
251 'executionAuthorization': 'Regular require_escalated approval; no symlink or sandbox configuration changes.',
252 'acceptancePassed': True})
253 print(json.dumps({'acceptancePassed': True, 'ctm': final['statuses'], 'xtm': {'pass': 6, 'fail': 0, 'not-run': 118},
254 'originalObservations': 7, 'expandedComparisons': 3, 'protectedBaseFiles': len(protected)}))
255
256
257 if __name__ == '__main__':
258 main()