]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22r/reproduce.py
2 """Reproduce Slice 22R with the existing offline collector; audit against 22Q.
3 --output must be outside the repository. --audit-only verifies saved execution,
4 never claims a new test run. No downloads or changes to existing collectors.
14 from pathlib
import Path
16 sys
.dont_write_bytecode
= True
17 ROOT
= Path(__file__
).resolve().parents
[2]
18 D
= ROOT
/ 'reports/slice-22r'
19 PRIOR
= ROOT
/ 'reports/slice-22q/after'
20 BASE
= '0cb182ebab1b90dd871048c031612acd9d302678'
21 ALLOWED
= {'src/xml/package.lisp', 'topic-maps.asd', 'tests/xtm-identity.lisp'}
22 NEW_SOURCES
= {'src/xml/resource-c14n.lisp', 'tests/xml-resource-c14n.lisp'}
26 return json
.loads(p
.read_text())
30 return hashlib
.sha256(p
.read_bytes()).hexdigest()
34 p
.write_text(json
.dumps(value
, indent
=2, ensure_ascii
=False) + '\n')
38 initial
= read(D
/ 'initial-state.json')
39 assert initial
['baseCommit'] == BASE
and initial
['initialIndexAndWorktreeClean']
40 protected
= {p
: h
for p
, h
in initial
['trackedFileSHA256'].items() if p
not in ALLOWED
}
41 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
42 run
= read(out
/ 'runs.json')
43 assert len(run
['runs']) == 119
44 assert all(r
['exitCode'] == r
['expectedExitCode'] and
45 r
['network'] == r
['repositoryWrites'] == r
['hyperdocWrites'] == 'denied' and
46 sha(out
/ (r
['label'] + '.log')) == r
['logSHA256'] for r
in run
['runs'])
47 assert run
['executionSourcesSHA256'] == {p
: sha(ROOT
/ p
) for p
in run
['executionSourcesSHA256']}
48 assert run
['rawReportSHA256'] == {p
: sha(out
/ p
) for p
in run
['rawReportSHA256']}
49 old
, new
= read(PRIOR
/ 'xtm.json'), read(out
/ 'xtm.json')
52 assert old
[key
] == new
[key
], key
53 assert [c
for c
in new
['checks'] if c
in set(old
['checks'])] == old
['checks']
54 additions
= [c
for c
in new
['checks'] if c
not in set(old
['checks'])]
55 assert additions
and all(c
.startswith(('resource-c14n:', 'resource-c14n-native:',
56 'resource-c14n-security:', 'resource-c14n-api:')) for c
in additions
)
57 labels
= re
.findall(r
'^XTM-CHECK-PASS (.+)$', (out
/ 'xtm.log').read_text(), re
.M
)
58 assert labels
== new
['checks'] and len(labels
) == len(set(labels
))
59 x
= new
['resourceDataC14N']
60 assert x
['executed'] and x
['qualifiedForClosedXTMNodeSet']
61 assert not x
['generalXPathC14N'] and not x
['XTMReaderIntegration'] and x
['corpusRegistrationsAdded'] == 0
62 rows
= {r
['id']: r
for r
in x
['probes']}
63 assert len(rows
) == len(x
['probes'])
64 # Every existing independent 22Q oracle and ORIGINAL input byte is reused.
65 for r
in old
['c14nPreflight']['probes']:
66 newrow
= rows
[r
['id']]
67 assert newrow
['inputHex'] == r
['inputHex'] and newrow
['expectedHex'] == r
['expectedHex']
68 official
= [r
for r
in rows
.values() if r
['id'].startswith('w3c-')]
69 assert len(official
) == 6
70 assert sum(not r
['publicXTMProjection'] for r
in official
) == 3
71 for r
in rows
.values():
72 actual
, expected
= bytes
.fromhex(r
['actualHex']), bytes
.fromhex(r
['expectedHex'])
73 assert actual
== expected
== r
['expectedXML'].encode('utf-8') == r
['actualXML'].encode('utf-8')
74 assert len(actual
) == r
['actualBytes'] == r
['expectedBytes']
75 assert r
['firstDifferentByte'] is None and r
['byteIdentical'] and r
['repeatedPure']
76 public
= {r
['id']: r
for r
in rows
.values() if r
['publicXTMProjection']}
77 assert set(public
) == {r
['id'] for r
in x
['nativeValues']}
78 for r
in x
['nativeValues']:
79 assert r
['value'] == r
['occurrenceValue'] == r
['variantValue'] == public
[r
['id']]['actualXML']
80 assert r
['datatype'] == 'http://www.w3.org/2001/XMLSchema#anyType' and r
['topicCount'] == 5
81 assert r
['canonicalReferencesEQ'] and r
['repeatedCXTMPure'] and not r
['readerIntegration']
82 assert x
['controls'] and all(r
['condition'] for r
in x
['controls'])
83 # Historical candidate still has the same eight failures, not reclassified.
84 assert new
['c14nPreflight'] == old
['c14nPreflight']
85 assert sum(r
['normativeComparison'] == 'BYTE-DIFFERENCE' for r
in new
['c14nPreflight']['probes']) == 8
86 assert read(PRIOR
/ 'after-diagnostics.json') == read(out
/ 'after-diagnostics.json')
87 assert len(read(out
/ 'after-diagnostics.json')) == 124
89 for p
in sorted((out
/ 'retained').rglob('*')):
91 rel
= p
.relative_to(out
)
92 assert p
.read_bytes() == (PRIOR
/ rel
).read_bytes(), str(rel
)
93 retained
.append({'report': str(rel
), 'SHA256': sha(p
)})
96 if '-control-' not in r
['label']:
98 name
= r
['label'] + '.json'
99 a
, b
= read(PRIOR
/ name
), read(out
/ name
)
100 assert r
['exitCode'] == 1
102 assert a
[key
] == b
[key
], (name
, key
)
103 assert not b
['resourceDataC14N']['executed'] and not b
['resourceDataC14N']['qualifiedForClosedXTMNodeSet']
104 assert not b
['resourceDataC14N']['probes'] and not b
['resourceDataC14N']['nativeValues']
105 controls
.append({'label': r
['label'], 'expectedExitCode': 1, 'previousFieldsUnchanged': True,
106 'retargeted': False, 'newTestsNotClaimedExecuted': True})
107 assert len(controls
) == 113
108 prior
= read(PRIOR
/ 'execution.json')
109 regression
= (out
/ 'regression.log').read_text()
110 for marker
, count
in prior
['stageChecks'].items():
111 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
112 core_markers
= read(ROOT
/ 'reports/slice-22d/baseline/execution.json')['retainedChecks']
113 assert sum(core_markers
.values()) == prior
['retainedLispChecks']
114 for marker
, count
in core_markers
.items():
115 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
116 assert 'CTM-XTM-EQUIVALENCE-PASS: 529 checks; 30 actual reader pairs; 3 contrasts; 0 capability gaps.' in regression
117 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
118 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out
/ 'xml.log').read_text()
119 assert read(out
/ 'ctm.validation.json')['statuses'] == {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
120 assert new
['statuses'] == {'pass': 121, 'fail': 0, 'unsupported': 0, 'not-run': 0}
121 assert new
['otherCorpusInputs'] == {'positiveUnexecuted': 3, 'negativeUnregisteredUnexecuted': 0}
122 assert new
['cases'] == old
['cases']
123 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
124 summary
= [{'id': r
['id'], 'expectedBytes': r
['expectedBytes'], 'actualBytes': r
['actualBytes'],
125 'firstDifferentByte': None, 'byteIdentical': True, 'origin': r
['origin']} for r
in rows
.values()]
126 save(out
/ 'comparison-summary.json', {'kind': 'XTM-node-set-C14N-1.0-closed-profile-qualification',
127 'probes': summary
, 'unchanged22QVectors': len(old
['c14nPreflight']['probes']),
128 'independentW3CVectors': 3, 'additionalPublicWrappersOfW3CVectors': 3,
129 'publicValueAndNativeModelRows': len(public
), 'oldCandidateDifferencesPreserved': 8})
130 save(out
/ 'failure-controls-audit.json', controls
)
131 save(out
/ 'retained-reports.json', retained
)
132 save(out
/ 'diagnostic-changes.json', {'observations': 124, 'changed': 0, 'unchanged': 124,
133 'changes': [], 'newOriginalRegistrations': 0, 'retargetings': []})
134 save(out
/ 'integrity.json', {'protectedBaseFilesSHA256': protected
,
135 'protectedBaseFileCount': len(protected
),
136 'pinnedCorpusFilesVerified': sum(p
.startswith('vendor/cxtm/') for p
in protected
),
137 'historicalReportFilesVerified': sum(p
.startswith('reports/') for p
in protected
),
138 'allowedTrackedChanges': sorted(ALLOWED
), 'newExecutionSources': sorted(NEW_SOURCES
),
139 'originalXMLByteBoundaryUnchanged': True, 'allTMDMCTMXTMCXTMSourcesUnchanged': True})
140 save(out
/ 'execution.json', {'kind': 'slice-22r-closed-XTM-C14N-infrastructure-NOT-reader-integration',
141 'baseCommit': BASE
, 'branch': subprocess
.check_output(['git', 'branch', '--show-current'], cwd
=ROOT
, text
=True).strip(),
142 'runtime': 'SBCL 2.5.4', 'startedAtUTC': run
['startedAtUTC'], 'finishedAtUTC': run
['finishedAtUTC'],
143 'auditedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
144 'collector': 'scripts/validate-slice-22d.py::collect (unchanged)', 'auditToolSHA256': sha(Path(__file__
)),
145 'runs': run
['runs'], 'standaloneXTMChecks': len(labels
), 'retainedStandaloneXTMChecks': len(old
['checks']),
146 'newInfrastructureChecks': len(additions
), 'retainedLispChecks': prior
['retainedLispChecks'],
147 'stageChecks': prior
['stageChecks'], 'equivalenceChecks': prior
['equivalenceChecks'],
148 'equivalenceReaderPairs': prior
['equivalenceReaderPairs'], 'faultControls': len(controls
),
149 'all124OriginalDiagnosesUnchanged': True, 'retainedReportsByteIdentical': len(retained
),
150 'statusesBefore': prior
['statusesAfter'], 'statusesAfter': prior
['statusesAfter'],
151 'originalRegistrationsAdded': 0, 'xmlQualificationChecks': prior
['xmlQualificationChecks'],
152 'pythonUnitTests': prior
['pythonUnitTests'], 'nfcOracleComparisons': prior
['nfcOracleComparisons'],
153 'qualifiedScope': 'XTM 4.16.3 descendant set; C14N 1.0; existing closed XML profile',
154 'generalXPathC14N': False, 'XTMReaderIntegration': False, 'newDependencies': 0,
155 'newNativeValueRows': len(public
), 'acceptancePassed': True})
156 print(json
.dumps({'acceptancePassed': True, 'XTMChecks': len(labels
), 'newChecks': len(additions
),
157 'positiveByteRows': len(rows
), 'nativeValueRows': len(public
), 'faultControls': len(controls
),
158 'statusesUnchanged': prior
['statusesAfter']}), flush
=True)
162 ap
= argparse
.ArgumentParser(description
=__doc__
)
163 ap
.add_argument('--output', type=Path
, required
=True)
164 ap
.add_argument('--audit-only', action
='store_true')
166 out
= a
.output
.resolve()
167 assert not out
.is_relative_to(ROOT
)
168 out
.mkdir(parents
=True, exist_ok
=True)
170 spec
= importlib
.util
.spec_from_file_location('oldcollector', ROOT
/ 'scripts/validate-slice-22d.py')
171 m
= importlib
.util
.module_from_spec(spec
)
172 spec
.loader
.exec_module(m
)
173 m
.EXECUTION_FILES
= {str(p
.relative_to(ROOT
)) for folder
in ['src', 'tests']
174 for p
in (ROOT
/ folder
).rglob('*.lisp')} |
{
175 'topic-maps.asd', 'scripts/validate-slice-22d.py',
176 'reports/slice-22d/failure-controls.lisp', 'reports/slice-22r/reproduce.py'}
177 m
.collect(out
, ROOT
/ 'reports/slice-22d/baseline')
181 if __name__
== '__main__':