]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22t/reproduce.py
2 """Reproduce/audit 22T via the existing offline harness; never downloads.
3 --audit-only inspects saved evidence and is not a new test execution.
5 import argparse
, datetime
, hashlib
, importlib
.util
, json
, re
, sys
6 from pathlib
import Path
7 sys
.dont_write_bytecode
= True
8 ROOT
= Path(__file__
).resolve().parents
[2]
9 D
= ROOT
/'reports/slice-22t'
10 PRIOR
= ROOT
/'reports/slice-22s/after'
11 BASE
= '513de98e5f9dfb62ba3c6310a150bb43c5a4c151'
12 ALLOWED
= {'tests/ctm-xtm-equivalence.lisp', 'topic-maps.asd'}
13 NEW_IDS
= {'anytype-occurrence', 'anytype-variant', 'anytype-occurrence-normalization',
14 'anytype-variant-normalization', 'anytype-namespace-meaning'}
16 def read(p
): return json
.loads(p
.read_text())
17 def sha(p
): return hashlib
.sha256(p
.read_bytes()).hexdigest()
18 def save(p
,v
): p
.write_text(json
.dumps(v
,indent
=2,ensure_ascii
=False)+'\n')
21 initial
= read(D
/'initial-state.json')
22 assert initial
['baseCommit']==BASE
and initial
['initialIndexAndWorktreeClean']
23 protected
= {p
:h
for p
,h
in initial
['trackedFileSHA256'].items() if p
not in ALLOWED
}
24 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
25 restoration
= read(D
/'restoration.json')
26 assert restoration
['restorationPassed'] and len(restoration
['acquisitions'])==2
27 assert all(r
['exitCode']==0 and r
['identityVerified'] and r
['verifiedNARHash']==r
['expectedNARHash']
28 and r
['result']['storePath']==r
['expectedStorePath'] for r
in restoration
['acquisitions'])
29 prep
,priorprep
=read(out
/'source-preparation.json'),read(PRIOR
/'source-preparation.json')
30 assert prep
['sources']==priorprep
['sources'] and prep
['patches']==priorprep
['patches']
31 runs
= read(out
/'runs.json');assert len(runs
['runs'])==119
32 for r
in runs
['runs']:
33 assert r
['exitCode']==r
['expectedExitCode'] and r
['freshProcess']
34 assert r
['network']==r
['repositoryWrites']==r
['hyperdocWrites']=='denied'
35 assert sha(out
/(r
['label']+'.log'))==r
['logSHA256']
36 assert runs
['executionSourcesSHA256']=={p
:sha(ROOT
/p
) for p
in runs
['executionSourcesSHA256']}
37 assert runs
['rawReportSHA256']=={p
:sha(out
/p
) for p
in runs
['rawReportSHA256']}
38 old
,new
=read(PRIOR
/'xtm.json'),read(out
/'xtm.json')
40 labels
=re
.findall(r
'^XTM-CHECK-PASS (.+)$',(out
/'xtm.log').read_text(),re
.M
)
41 assert labels
==new
['checks'] and len(labels
)==4327 and len(labels
)==len(set(labels
))
42 assert new
['statuses']=={'pass':121,'fail':0,'unsupported':0,'not-run':0}
43 assert new
['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
44 assert read(out
/'after-diagnostics.json')==read(PRIOR
/'after-diagnostics.json')
45 assert len(read(out
/'after-diagnostics.json'))==124
47 for r
in runs
['runs']:
48 if '-control-' not in r
['label']:continue
49 name
=r
['label']+'.json'
50 assert (out
/name
).read_bytes()==(PRIOR
/name
).read_bytes(),name
51 assert r
['exitCode']==1
52 faults
.append({'label':r
['label'],'expectedExitCode':1,'reportByteIdenticalTo22S':True,
53 'faultCountedAsOriginalPASS':False})
54 assert len(faults
)==113
55 rel
=Path('retained/xtm-mergemap.equivalence.json')
56 before
,after
=read(PRIOR
/rel
),read(out
/rel
)
57 assert after
['runtime']=='2.5.4' and after
['condition'] is None
58 assert after
['productionModelChanges'] is False and after
['originalCorpusRegistrationsAdded']==0
59 assert len(before
['checks'])==529 and len(after
['checks'])==len(set(after
['checks']))==728
60 assert [c
for c
in after
['checks'] if c
in set(before
['checks'])]==before
['checks']
61 added
=[c
for c
in after
['checks'] if c
not in set(before
['checks'])]
62 assert len(added
)==199 and all(c
.startswith(('equivalence:anytype-', 'equivalence:contrast:anytype-')) for c
in added
)
63 oldids
={r
['id'] for r
in before
['pairs']}
64 assert [r
for r
in after
['pairs'] if r
['id'] in oldids
]==before
['pairs']
65 assert len(oldids
)==33 and len({r
['id'] for r
in after
['pairs']})==len(after
['pairs'])==38
66 rows
=[r
for r
in after
['pairs'] if r
['id'] not in oldids
]
67 assert {r
['id'] for r
in rows
}==NEW_IDS
68 positives
=[r
for r
in rows
if r
['kind']=='direct-reader-equivalence-pair']
69 assert len(positives
)==4
71 assert r
['slice']=='22T' and r
['expectedDatatype']=='http://www.w3.org/2001/XMLSchema#anyType'
72 assert r
['ctmModel']['counts']==r
['xtmModel']['counts']==r
['expectedNativeCounts']
73 calls
=r
['actualReaderEntries'];n
=4 if r
in positives
else 2
74 assert len(calls
)==len(r
['nativeConstructorEvidence'])==n
75 assert {c
['entryPoint'] for c
in calls
}=={'topic-maps.ctm:deserialize-string','topic-maps.xtm:deserialize-octets'}
76 assert all(c
['documentIRI']==r
['canonicalBaseIRI'] and c
['inputHex'] for c
in calls
)
77 assert bytes
.fromhex(calls
[0]['inputHex']).decode('utf-8')==r
['ctmSource']
78 assert bytes
.fromhex(calls
[1]['inputHex']).decode('utf-8')==r
['xtmSource']
79 for proof
in r
['nativeConstructorEvidence']:
80 assert proof
['allReturnedHandlesCanonicalAndLive']
81 assert proof
['nativeItemCountsBefore']==([0,1] if r
['normalizationRequired'] else [0])
82 assert proof
['nativeItemCountsAfter']==([1,1] if r
['normalizationRequired'] else [1])
83 for lang
in ['ctm','xtm']:
84 assert len(r
[lang
+'CXTM'].encode('utf-8'))==r
[lang
+'Bytes']
86 assert r
['ctmCondition'] is None and r
['xtmCondition'] is None
87 assert r
['documentIRI']==r
['canonicalBaseIRI']
88 assert r
['nativePropertiesEqual'] and r
['ctmCanonicalEQ'] and r
['xtmCanonicalEQ']
89 assert r
['crossMapEQRequired'] is False
90 assert r
['byteExact'] and r
['firstDifferentByte'] is None and r
['ctmCXTM']==r
['xtmCXTM']
91 assert r
['independentExpectedCanonicalValue']==r
['expectedXTMCanonicalValue']
93 assert r
['id']=='anytype-namespace-meaning' and r
['byteExact'] is False
94 assert r
['nativePropertiesEqual'] is False and r
['ctmCXTM']!=r
['xtmCXTM']
95 assert r
['independentExpectedCanonicalValue']!=r
['expectedXTMCanonicalValue']
96 a
,b
=r
['ctmCXTM'].encode(),r
['xtmCXTM'].encode()
97 assert next(i
for i
,(x
,y
) in enumerate(zip(a
,b
)) if x
!=y
)==r
['firstDifferentByte']
99 for p
in sorted((out
/'retained').rglob('*')):
100 if not p
.is_file() or p
.relative_to(out
)==rel
:continue
101 prior
=PRIOR
/p
.relative_to(out
)
102 assert p
.read_bytes()==prior
.read_bytes(),str(p
.relative_to(out
))
103 retained
.append({'report':str(p
.relative_to(out
)),'SHA256':sha(p
)})
104 for name
in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
105 assert (out
/name
).read_bytes()==(PRIOR
/name
).read_bytes(),name
106 assert read(out
/'ctm.validation.json')['statuses']=={'pass':262,'fail':4,'unsupported':0,'not-run':7}
107 previous
=read(PRIOR
/'execution.json');reg
=(out
/'regression.log').read_text()
108 core
=read(ROOT
/'reports/slice-22d/baseline/execution.json')['retainedChecks']
109 assert sum(core
.values())==12100==previous
['retainedLispChecks']
110 for marker
,count
in dict(core
,**previous
['stageChecks']).items():
111 assert int(re
.search(r
'(?m)^'+marker
+r
'-PASS: (\d+)',reg
).group(1))==count
,marker
112 assert 'CTM-XTM-EQUIVALENCE-PASS: 728 checks; 34 actual reader pairs; 4 contrasts; 0 capability gaps.' in reg
113 assert 'Ran 2 tests' in reg
and 'NFC-ORACLE-PASS: 16294' in reg
114 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out
/'xml.log').read_text()
115 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
116 save(out
/'failure-controls-audit.json',faults
)
117 save(out
/'diagnostic-changes.json',{'observations':124,'changed':0,'unchanged':124,'changes':[],
118 'registrationsAdded':0,'retargetings':[]})
119 save(out
/'retained-reports.json',retained
)
120 save(out
/'equivalence-audit.json',{'retainedRowsByteIdentical':33,'retainedPositivePairs':30,
121 'retainedContrasts':3,'retainedChecksUnchanged':529,'newChecks':len(added
),'checks':728,
122 'positiveReaderPairs':34,'contrasts':4,'newProductiveReaderCalls':18,
123 'newSourceItemConstructorCalls':26,'newRows':rows
})
124 save(out
/'integrity.json',{'protectedBaseFilesSHA256':protected
,'protectedBaseFileCount':len(protected
),
125 'pinnedCorpusFilesVerified':sum(p
.startswith('vendor/cxtm/') for p
in protected
),
126 'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),
127 'allProductionSourcesUnchanged':True,'originalsBaselinesAndCaseRecordsUnchanged':True,
128 'allowedTrackedChanges':sorted(ALLOWED
),'newProductionFeatures':0})
129 save(out
/'execution.json',{'kind':'slice-22t-productive-CTM-XTM-anyType-equivalence',
130 'baseCommit':BASE
,'runtime':'SBCL 2.5.4','startedAtUTC':runs
['startedAtUTC'],
131 'finishedAtUTC':runs
['finishedAtUTC'],'auditedAtUTC':datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
132 'collector':'scripts/validate-slice-22d.py::collect with unchanged 22S failure driver',
133 'collectorConfigurationSHA256':sha(D
/'collect.py'),'auditToolSHA256':sha(Path(__file__
)),
134 'runCount':119,'runs':runs
['runs'],'standaloneXTMChecks':4327,'faultControls':113,
135 'retainedLispChecks':12100,'stageChecks':previous
['stageChecks'],'equivalenceChecks':728,
136 'equivalenceReaderPairs':34,'equivalenceContrasts':4,'newReaderPairs':4,'newNamespaceContrasts':1,
137 'newProductiveReaderCalls':18,'newNativeConstructorCalls':26,
138 'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,
139 'all124OriginalDiagnosesUnchanged':True,'retargetings':0,'originalRegistrationsAdded':0,
140 'statusesBefore':previous
['statusesAfter'],'statusesAfter':previous
['statusesAfter'],
141 'productionSourceChanges':0,'newExternalDependencies':0,
142 'pinnedSourceAndPatchIdentityRestored':True,'retainedReportsByteIdentical':len(retained
),
143 'acceptancePassed':True})
144 print(json
.dumps({'acceptancePassed':True,'equivalencePairs':34,'contrasts':4,
145 'equivalenceChecks':728,'XTMChecks':4327,'faultControls':113,'statuses':previous
['statusesAfter']}),flush
=True)
148 ap
=argparse
.ArgumentParser(description
=__doc__
)
149 ap
.add_argument('--output',type=Path
,required
=True);ap
.add_argument('--audit-only',action
='store_true')
150 args
=ap
.parse_args();out
=args
.output
.resolve();assert not out
.is_relative_to(ROOT
)
151 if not args
.audit_only
:
152 spec
=importlib
.util
.spec_from_file_location('slice22t_collector',D
/'collect.py')
153 m
=importlib
.util
.module_from_spec(spec
);spec
.loader
.exec_module(m
);m
.collect(out
)
155 if __name__
=='__main__': main()