]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22r/reproduce.py
test: prove CTM/XTM anyType equivalence (slice 22T)
[topic-maps.git] / reports / slice-22r / reproduce.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 22R with the existing offline collector; audit against 22Q.
3 --output must be outside the repository. --audit-only verifies saved execution,
4 never claims a new test run. No downloads or changes to existing collectors.
5 """
6 import argparse
7 import datetime
8 import hashlib
9 import importlib.util
10 import json
11 import re
12 import subprocess
13 import sys
14 from pathlib import Path
15
16 sys.dont_write_bytecode = True
17 ROOT = Path(__file__).resolve().parents[2]
18 D = ROOT / 'reports/slice-22r'
19 PRIOR = ROOT / 'reports/slice-22q/after'
20 BASE = '0cb182ebab1b90dd871048c031612acd9d302678'
21 ALLOWED = {'src/xml/package.lisp', 'topic-maps.asd', 'tests/xtm-identity.lisp'}
22 NEW_SOURCES = {'src/xml/resource-c14n.lisp', 'tests/xml-resource-c14n.lisp'}
23
24
25 def read(p):
26 return json.loads(p.read_text())
27
28
29 def sha(p):
30 return hashlib.sha256(p.read_bytes()).hexdigest()
31
32
33 def save(p, value):
34 p.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n')
35
36
37 def audit(out):
38 initial = read(D / 'initial-state.json')
39 assert initial['baseCommit'] == BASE and initial['initialIndexAndWorktreeClean']
40 protected = {p: h for p, h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
41 assert protected == {p: sha(ROOT / p) for p in protected}
42 run = read(out / 'runs.json')
43 assert len(run['runs']) == 119
44 assert all(r['exitCode'] == r['expectedExitCode'] and
45 r['network'] == r['repositoryWrites'] == r['hyperdocWrites'] == 'denied' and
46 sha(out / (r['label'] + '.log')) == r['logSHA256'] for r in run['runs'])
47 assert run['executionSourcesSHA256'] == {p: sha(ROOT / p) for p in run['executionSourcesSHA256']}
48 assert run['rawReportSHA256'] == {p: sha(out / p) for p in run['rawReportSHA256']}
49 old, new = read(PRIOR / 'xtm.json'), read(out / 'xtm.json')
50 for key in old:
51 if key != 'checks':
52 assert old[key] == new[key], key
53 assert [c for c in new['checks'] if c in set(old['checks'])] == old['checks']
54 additions = [c for c in new['checks'] if c not in set(old['checks'])]
55 assert additions and all(c.startswith(('resource-c14n:', 'resource-c14n-native:',
56 'resource-c14n-security:', 'resource-c14n-api:')) for c in additions)
57 labels = re.findall(r'^XTM-CHECK-PASS (.+)$', (out / 'xtm.log').read_text(), re.M)
58 assert labels == new['checks'] and len(labels) == len(set(labels))
59 x = new['resourceDataC14N']
60 assert x['executed'] and x['qualifiedForClosedXTMNodeSet']
61 assert not x['generalXPathC14N'] and not x['XTMReaderIntegration'] and x['corpusRegistrationsAdded'] == 0
62 rows = {r['id']: r for r in x['probes']}
63 assert len(rows) == len(x['probes'])
64 # Every existing independent 22Q oracle and ORIGINAL input byte is reused.
65 for r in old['c14nPreflight']['probes']:
66 newrow = rows[r['id']]
67 assert newrow['inputHex'] == r['inputHex'] and newrow['expectedHex'] == r['expectedHex']
68 official = [r for r in rows.values() if r['id'].startswith('w3c-')]
69 assert len(official) == 6
70 assert sum(not r['publicXTMProjection'] for r in official) == 3
71 for r in rows.values():
72 actual, expected = bytes.fromhex(r['actualHex']), bytes.fromhex(r['expectedHex'])
73 assert actual == expected == r['expectedXML'].encode('utf-8') == r['actualXML'].encode('utf-8')
74 assert len(actual) == r['actualBytes'] == r['expectedBytes']
75 assert r['firstDifferentByte'] is None and r['byteIdentical'] and r['repeatedPure']
76 public = {r['id']: r for r in rows.values() if r['publicXTMProjection']}
77 assert set(public) == {r['id'] for r in x['nativeValues']}
78 for r in x['nativeValues']:
79 assert r['value'] == r['occurrenceValue'] == r['variantValue'] == public[r['id']]['actualXML']
80 assert r['datatype'] == 'http://www.w3.org/2001/XMLSchema#anyType' and r['topicCount'] == 5
81 assert r['canonicalReferencesEQ'] and r['repeatedCXTMPure'] and not r['readerIntegration']
82 assert x['controls'] and all(r['condition'] for r in x['controls'])
83 # Historical candidate still has the same eight failures, not reclassified.
84 assert new['c14nPreflight'] == old['c14nPreflight']
85 assert sum(r['normativeComparison'] == 'BYTE-DIFFERENCE' for r in new['c14nPreflight']['probes']) == 8
86 assert read(PRIOR / 'after-diagnostics.json') == read(out / 'after-diagnostics.json')
87 assert len(read(out / 'after-diagnostics.json')) == 124
88 retained = []
89 for p in sorted((out / 'retained').rglob('*')):
90 if p.is_file():
91 rel = p.relative_to(out)
92 assert p.read_bytes() == (PRIOR / rel).read_bytes(), str(rel)
93 retained.append({'report': str(rel), 'SHA256': sha(p)})
94 controls = []
95 for r in run['runs']:
96 if '-control-' not in r['label']:
97 continue
98 name = r['label'] + '.json'
99 a, b = read(PRIOR / name), read(out / name)
100 assert r['exitCode'] == 1
101 for key in a:
102 assert a[key] == b[key], (name, key)
103 assert not b['resourceDataC14N']['executed'] and not b['resourceDataC14N']['qualifiedForClosedXTMNodeSet']
104 assert not b['resourceDataC14N']['probes'] and not b['resourceDataC14N']['nativeValues']
105 controls.append({'label': r['label'], 'expectedExitCode': 1, 'previousFieldsUnchanged': True,
106 'retargeted': False, 'newTestsNotClaimedExecuted': True})
107 assert len(controls) == 113
108 prior = read(PRIOR / 'execution.json')
109 regression = (out / 'regression.log').read_text()
110 for marker, count in prior['stageChecks'].items():
111 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
112 core_markers = read(ROOT / 'reports/slice-22d/baseline/execution.json')['retainedChecks']
113 assert sum(core_markers.values()) == prior['retainedLispChecks']
114 for marker, count in core_markers.items():
115 assert int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', regression).group(1)) == count
116 assert 'CTM-XTM-EQUIVALENCE-PASS: 529 checks; 30 actual reader pairs; 3 contrasts; 0 capability gaps.' in regression
117 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
118 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out / 'xml.log').read_text()
119 assert read(out / 'ctm.validation.json')['statuses'] == {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
120 assert new['statuses'] == {'pass': 121, 'fail': 0, 'unsupported': 0, 'not-run': 0}
121 assert new['otherCorpusInputs'] == {'positiveUnexecuted': 3, 'negativeUnregisteredUnexecuted': 0}
122 assert new['cases'] == old['cases']
123 assert protected == {p: sha(ROOT / p) for p in protected}
124 summary = [{'id': r['id'], 'expectedBytes': r['expectedBytes'], 'actualBytes': r['actualBytes'],
125 'firstDifferentByte': None, 'byteIdentical': True, 'origin': r['origin']} for r in rows.values()]
126 save(out / 'comparison-summary.json', {'kind': 'XTM-node-set-C14N-1.0-closed-profile-qualification',
127 'probes': summary, 'unchanged22QVectors': len(old['c14nPreflight']['probes']),
128 'independentW3CVectors': 3, 'additionalPublicWrappersOfW3CVectors': 3,
129 'publicValueAndNativeModelRows': len(public), 'oldCandidateDifferencesPreserved': 8})
130 save(out / 'failure-controls-audit.json', controls)
131 save(out / 'retained-reports.json', retained)
132 save(out / 'diagnostic-changes.json', {'observations': 124, 'changed': 0, 'unchanged': 124,
133 'changes': [], 'newOriginalRegistrations': 0, 'retargetings': []})
134 save(out / 'integrity.json', {'protectedBaseFilesSHA256': protected,
135 'protectedBaseFileCount': len(protected),
136 'pinnedCorpusFilesVerified': sum(p.startswith('vendor/cxtm/') for p in protected),
137 'historicalReportFilesVerified': sum(p.startswith('reports/') for p in protected),
138 'allowedTrackedChanges': sorted(ALLOWED), 'newExecutionSources': sorted(NEW_SOURCES),
139 'originalXMLByteBoundaryUnchanged': True, 'allTMDMCTMXTMCXTMSourcesUnchanged': True})
140 save(out / 'execution.json', {'kind': 'slice-22r-closed-XTM-C14N-infrastructure-NOT-reader-integration',
141 'baseCommit': BASE, 'branch': subprocess.check_output(['git', 'branch', '--show-current'], cwd=ROOT, text=True).strip(),
142 'runtime': 'SBCL 2.5.4', 'startedAtUTC': run['startedAtUTC'], 'finishedAtUTC': run['finishedAtUTC'],
143 'auditedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
144 'collector': 'scripts/validate-slice-22d.py::collect (unchanged)', 'auditToolSHA256': sha(Path(__file__)),
145 'runs': run['runs'], 'standaloneXTMChecks': len(labels), 'retainedStandaloneXTMChecks': len(old['checks']),
146 'newInfrastructureChecks': len(additions), 'retainedLispChecks': prior['retainedLispChecks'],
147 'stageChecks': prior['stageChecks'], 'equivalenceChecks': prior['equivalenceChecks'],
148 'equivalenceReaderPairs': prior['equivalenceReaderPairs'], 'faultControls': len(controls),
149 'all124OriginalDiagnosesUnchanged': True, 'retainedReportsByteIdentical': len(retained),
150 'statusesBefore': prior['statusesAfter'], 'statusesAfter': prior['statusesAfter'],
151 'originalRegistrationsAdded': 0, 'xmlQualificationChecks': prior['xmlQualificationChecks'],
152 'pythonUnitTests': prior['pythonUnitTests'], 'nfcOracleComparisons': prior['nfcOracleComparisons'],
153 'qualifiedScope': 'XTM 4.16.3 descendant set; C14N 1.0; existing closed XML profile',
154 'generalXPathC14N': False, 'XTMReaderIntegration': False, 'newDependencies': 0,
155 'newNativeValueRows': len(public), 'acceptancePassed': True})
156 print(json.dumps({'acceptancePassed': True, 'XTMChecks': len(labels), 'newChecks': len(additions),
157 'positiveByteRows': len(rows), 'nativeValueRows': len(public), 'faultControls': len(controls),
158 'statusesUnchanged': prior['statusesAfter']}), flush=True)
159
160
161 def main():
162 ap = argparse.ArgumentParser(description=__doc__)
163 ap.add_argument('--output', type=Path, required=True)
164 ap.add_argument('--audit-only', action='store_true')
165 a = ap.parse_args()
166 out = a.output.resolve()
167 assert not out.is_relative_to(ROOT)
168 out.mkdir(parents=True, exist_ok=True)
169 if not a.audit_only:
170 spec = importlib.util.spec_from_file_location('oldcollector', ROOT / 'scripts/validate-slice-22d.py')
171 m = importlib.util.module_from_spec(spec)
172 spec.loader.exec_module(m)
173 m.EXECUTION_FILES = {str(p.relative_to(ROOT)) for folder in ['src', 'tests']
174 for p in (ROOT / folder).rglob('*.lisp')} | {
175 'topic-maps.asd', 'scripts/validate-slice-22d.py',
176 'reports/slice-22d/failure-controls.lisp', 'reports/slice-22r/reproduce.py'}
177 m.collect(out, ROOT / 'reports/slice-22d/baseline')
178 audit(out)
179
180
181 if __name__ == '__main__':
182 main()