]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22t/reproduce.py
examples: explore project work through native Topic Maps (slice 22U)
[topic-maps.git] / reports / slice-22t / reproduce.py
1 #!/usr/bin/env python3
2 """Reproduce/audit 22T via the existing offline harness; never downloads.
3 --audit-only inspects saved evidence and is not a new test execution.
4 """
5 import argparse, datetime, hashlib, importlib.util, json, re, sys
6 from pathlib import Path
7 sys.dont_write_bytecode = True
8 ROOT = Path(__file__).resolve().parents[2]
9 D = ROOT/'reports/slice-22t'
10 PRIOR = ROOT/'reports/slice-22s/after'
11 BASE = '513de98e5f9dfb62ba3c6310a150bb43c5a4c151'
12 ALLOWED = {'tests/ctm-xtm-equivalence.lisp', 'topic-maps.asd'}
13 NEW_IDS = {'anytype-occurrence', 'anytype-variant', 'anytype-occurrence-normalization',
14 'anytype-variant-normalization', 'anytype-namespace-meaning'}
15
16 def read(p): return json.loads(p.read_text())
17 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
18 def save(p,v): p.write_text(json.dumps(v,indent=2,ensure_ascii=False)+'\n')
19
20 def audit(out):
21 initial = read(D/'initial-state.json')
22 assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
23 protected = {p:h for p,h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
24 assert protected=={p:sha(ROOT/p) for p in protected}
25 restoration = read(D/'restoration.json')
26 assert restoration['restorationPassed'] and len(restoration['acquisitions'])==2
27 assert all(r['exitCode']==0 and r['identityVerified'] and r['verifiedNARHash']==r['expectedNARHash']
28 and r['result']['storePath']==r['expectedStorePath'] for r in restoration['acquisitions'])
29 prep,priorprep=read(out/'source-preparation.json'),read(PRIOR/'source-preparation.json')
30 assert prep['sources']==priorprep['sources'] and prep['patches']==priorprep['patches']
31 runs = read(out/'runs.json');assert len(runs['runs'])==119
32 for r in runs['runs']:
33 assert r['exitCode']==r['expectedExitCode'] and r['freshProcess']
34 assert r['network']==r['repositoryWrites']==r['hyperdocWrites']=='denied'
35 assert sha(out/(r['label']+'.log'))==r['logSHA256']
36 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
37 assert runs['rawReportSHA256']=={p:sha(out/p) for p in runs['rawReportSHA256']}
38 old,new=read(PRIOR/'xtm.json'),read(out/'xtm.json')
39 assert new==old
40 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
41 assert labels==new['checks'] and len(labels)==4327 and len(labels)==len(set(labels))
42 assert new['statuses']=={'pass':121,'fail':0,'unsupported':0,'not-run':0}
43 assert new['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
44 assert read(out/'after-diagnostics.json')==read(PRIOR/'after-diagnostics.json')
45 assert len(read(out/'after-diagnostics.json'))==124
46 faults=[]
47 for r in runs['runs']:
48 if '-control-' not in r['label']:continue
49 name=r['label']+'.json'
50 assert (out/name).read_bytes()==(PRIOR/name).read_bytes(),name
51 assert r['exitCode']==1
52 faults.append({'label':r['label'],'expectedExitCode':1,'reportByteIdenticalTo22S':True,
53 'faultCountedAsOriginalPASS':False})
54 assert len(faults)==113
55 rel=Path('retained/xtm-mergemap.equivalence.json')
56 before,after=read(PRIOR/rel),read(out/rel)
57 assert after['runtime']=='2.5.4' and after['condition'] is None
58 assert after['productionModelChanges'] is False and after['originalCorpusRegistrationsAdded']==0
59 assert len(before['checks'])==529 and len(after['checks'])==len(set(after['checks']))==728
60 assert [c for c in after['checks'] if c in set(before['checks'])]==before['checks']
61 added=[c for c in after['checks'] if c not in set(before['checks'])]
62 assert len(added)==199 and all(c.startswith(('equivalence:anytype-', 'equivalence:contrast:anytype-')) for c in added)
63 oldids={r['id'] for r in before['pairs']}
64 assert [r for r in after['pairs'] if r['id'] in oldids]==before['pairs']
65 assert len(oldids)==33 and len({r['id'] for r in after['pairs']})==len(after['pairs'])==38
66 rows=[r for r in after['pairs'] if r['id'] not in oldids]
67 assert {r['id'] for r in rows}==NEW_IDS
68 positives=[r for r in rows if r['kind']=='direct-reader-equivalence-pair']
69 assert len(positives)==4
70 for r in rows:
71 assert r['slice']=='22T' and r['expectedDatatype']=='http://www.w3.org/2001/XMLSchema#anyType'
72 assert r['ctmModel']['counts']==r['xtmModel']['counts']==r['expectedNativeCounts']
73 calls=r['actualReaderEntries'];n=4 if r in positives else 2
74 assert len(calls)==len(r['nativeConstructorEvidence'])==n
75 assert {c['entryPoint'] for c in calls}=={'topic-maps.ctm:deserialize-string','topic-maps.xtm:deserialize-octets'}
76 assert all(c['documentIRI']==r['canonicalBaseIRI'] and c['inputHex'] for c in calls)
77 assert bytes.fromhex(calls[0]['inputHex']).decode('utf-8')==r['ctmSource']
78 assert bytes.fromhex(calls[1]['inputHex']).decode('utf-8')==r['xtmSource']
79 for proof in r['nativeConstructorEvidence']:
80 assert proof['allReturnedHandlesCanonicalAndLive']
81 assert proof['nativeItemCountsBefore']==([0,1] if r['normalizationRequired'] else [0])
82 assert proof['nativeItemCountsAfter']==([1,1] if r['normalizationRequired'] else [1])
83 for lang in ['ctm','xtm']:
84 assert len(r[lang+'CXTM'].encode('utf-8'))==r[lang+'Bytes']
85 if r in positives:
86 assert r['ctmCondition'] is None and r['xtmCondition'] is None
87 assert r['documentIRI']==r['canonicalBaseIRI']
88 assert r['nativePropertiesEqual'] and r['ctmCanonicalEQ'] and r['xtmCanonicalEQ']
89 assert r['crossMapEQRequired'] is False
90 assert r['byteExact'] and r['firstDifferentByte'] is None and r['ctmCXTM']==r['xtmCXTM']
91 assert r['independentExpectedCanonicalValue']==r['expectedXTMCanonicalValue']
92 else:
93 assert r['id']=='anytype-namespace-meaning' and r['byteExact'] is False
94 assert r['nativePropertiesEqual'] is False and r['ctmCXTM']!=r['xtmCXTM']
95 assert r['independentExpectedCanonicalValue']!=r['expectedXTMCanonicalValue']
96 a,b=r['ctmCXTM'].encode(),r['xtmCXTM'].encode()
97 assert next(i for i,(x,y) in enumerate(zip(a,b)) if x!=y)==r['firstDifferentByte']
98 retained=[]
99 for p in sorted((out/'retained').rglob('*')):
100 if not p.is_file() or p.relative_to(out)==rel:continue
101 prior=PRIOR/p.relative_to(out)
102 assert p.read_bytes()==prior.read_bytes(),str(p.relative_to(out))
103 retained.append({'report':str(p.relative_to(out)),'SHA256':sha(p)})
104 for name in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
105 assert (out/name).read_bytes()==(PRIOR/name).read_bytes(),name
106 assert read(out/'ctm.validation.json')['statuses']=={'pass':262,'fail':4,'unsupported':0,'not-run':7}
107 previous=read(PRIOR/'execution.json');reg=(out/'regression.log').read_text()
108 core=read(ROOT/'reports/slice-22d/baseline/execution.json')['retainedChecks']
109 assert sum(core.values())==12100==previous['retainedLispChecks']
110 for marker,count in dict(core,**previous['stageChecks']).items():
111 assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',reg).group(1))==count,marker
112 assert 'CTM-XTM-EQUIVALENCE-PASS: 728 checks; 34 actual reader pairs; 4 contrasts; 0 capability gaps.' in reg
113 assert 'Ran 2 tests' in reg and 'NFC-ORACLE-PASS: 16294' in reg
114 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
115 assert protected=={p:sha(ROOT/p) for p in protected}
116 save(out/'failure-controls-audit.json',faults)
117 save(out/'diagnostic-changes.json',{'observations':124,'changed':0,'unchanged':124,'changes':[],
118 'registrationsAdded':0,'retargetings':[]})
119 save(out/'retained-reports.json',retained)
120 save(out/'equivalence-audit.json',{'retainedRowsByteIdentical':33,'retainedPositivePairs':30,
121 'retainedContrasts':3,'retainedChecksUnchanged':529,'newChecks':len(added),'checks':728,
122 'positiveReaderPairs':34,'contrasts':4,'newProductiveReaderCalls':18,
123 'newSourceItemConstructorCalls':26,'newRows':rows})
124 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'protectedBaseFileCount':len(protected),
125 'pinnedCorpusFilesVerified':sum(p.startswith('vendor/cxtm/') for p in protected),
126 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
127 'allProductionSourcesUnchanged':True,'originalsBaselinesAndCaseRecordsUnchanged':True,
128 'allowedTrackedChanges':sorted(ALLOWED),'newProductionFeatures':0})
129 save(out/'execution.json',{'kind':'slice-22t-productive-CTM-XTM-anyType-equivalence',
130 'baseCommit':BASE,'runtime':'SBCL 2.5.4','startedAtUTC':runs['startedAtUTC'],
131 'finishedAtUTC':runs['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),
132 'collector':'scripts/validate-slice-22d.py::collect with unchanged 22S failure driver',
133 'collectorConfigurationSHA256':sha(D/'collect.py'),'auditToolSHA256':sha(Path(__file__)),
134 'runCount':119,'runs':runs['runs'],'standaloneXTMChecks':4327,'faultControls':113,
135 'retainedLispChecks':12100,'stageChecks':previous['stageChecks'],'equivalenceChecks':728,
136 'equivalenceReaderPairs':34,'equivalenceContrasts':4,'newReaderPairs':4,'newNamespaceContrasts':1,
137 'newProductiveReaderCalls':18,'newNativeConstructorCalls':26,
138 'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,
139 'all124OriginalDiagnosesUnchanged':True,'retargetings':0,'originalRegistrationsAdded':0,
140 'statusesBefore':previous['statusesAfter'],'statusesAfter':previous['statusesAfter'],
141 'productionSourceChanges':0,'newExternalDependencies':0,
142 'pinnedSourceAndPatchIdentityRestored':True,'retainedReportsByteIdentical':len(retained),
143 'acceptancePassed':True})
144 print(json.dumps({'acceptancePassed':True,'equivalencePairs':34,'contrasts':4,
145 'equivalenceChecks':728,'XTMChecks':4327,'faultControls':113,'statuses':previous['statusesAfter']}),flush=True)
146
147 def main():
148 ap=argparse.ArgumentParser(description=__doc__)
149 ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true')
150 args=ap.parse_args();out=args.output.resolve();assert not out.is_relative_to(ROOT)
151 if not args.audit_only:
152 spec=importlib.util.spec_from_file_location('slice22t_collector',D/'collect.py')
153 m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);m.collect(out)
154 audit(out)
155 if __name__=='__main__': main()