]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22q/reproduce.py
perf(tmdm): index association normalization merge candidates
[topic-maps.git] / reports / slice-22q / reproduce.py
1 #!/usr/bin/env python3
2 """Run the existing offline collector, then audit Slice 22Q characterization.
3 Output must be a new directory outside the repository. This is not a C14N engine.
4 """
5 import argparse,datetime,hashlib,importlib.util,json,re,subprocess,sys
6 from pathlib import Path
7 sys.dont_write_bytecode=True
8 ROOT=Path(__file__).resolve().parents[2]
9 D=ROOT/'reports/slice-22q';PRIOR=ROOT/'reports/slice-22n/after'
10 BASE='1418be83ad013fe6d69e6036e6786be8100b9355'
11 ALLOWED={'topic-maps.asd','tests/xtm-identity.lisp'}
12 def read(p):return json.loads(p.read_text())
13 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
14 def save(p,x):p.write_text(json.dumps(x,indent=2,ensure_ascii=False)+'\n')
15 def audit(out):
16 initial=read(D/'initial-state.json');assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
17 protected={p:h for p,h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
18 assert protected=={p:sha(ROOT/p) for p in protected}
19 run=read(out/'runs.json');assert len(run['runs'])==119
20 assert all(r['exitCode']==r['expectedExitCode'] and r['network']==r['repositoryWrites']=='denied' and sha(out/(r['label']+'.log'))==r['logSHA256'] for r in run['runs'])
21 assert run['executionSourcesSHA256']=={p:sha(ROOT/p) for p in run['executionSourcesSHA256']}
22 assert run['rawReportSHA256']=={p:sha(out/p) for p in run['rawReportSHA256']}
23 old=read(PRIOR/'xtm.json');new=read(out/'xtm.json')
24 for key in old:
25 if key!='checks':assert old[key]==new[key],key
26 assert [c for c in new['checks'] if c in set(old['checks'])]==old['checks']
27 additions=[c for c in new['checks'] if c not in set(old['checks'])]
28 assert additions and all(c.startswith(('c14n-probe:','c14n-native:','c14n-api:','c14n-characterization:')) for c in additions)
29 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
30 assert labels==new['checks'] and len(labels)==len(set(labels))
31 x=new['c14nPreflight'];explore=read(D/'exploration.json')
32 assert x['executed'] and not x['W3CC14NQualified'] and not x['XTMReaderIntegration']
33 assert x['probes']==explore['probes'] and x['nativeValues']==explore['nativeValues']
34 comparisons=[]
35 for row in x['probes']:
36 actual=bytes.fromhex(row['actualHex']);expected=bytes.fromhex(row['expectedHex']);raw=bytes.fromhex(row['inputHex'])
37 assert actual==row['actualXML'].encode('utf-8') and expected==row['expectedXML'].encode('utf-8')
38 assert len(actual)==row['actualBytes'] and len(expected)==row['expectedBytes']
39 match=actual==expected;assert match==(row['normativeComparison']=='PASS-for-this-probe')
40 delta=next((i for i,(a,b) in enumerate(zip(actual,expected)) if a!=b),None)
41 if delta is None and len(actual)!=len(expected):delta=min(len(actual),len(expected))
42 assert delta==row['firstDifferentByte'] and not row['W3C-C14N-1.0-qualified']
43 comparisons.append({k:row[k] for k in ['id','expectedBytes','actualBytes','firstDifferentByte','normativeComparison']})
44 assert len({r['id'] for r in comparisons})==len(comparisons)
45 assert any(r['normativeComparison']=='BYTE-DIFFERENCE' for r in comparisons)
46 for item in x['nativeValues']:
47 assert item['occurrenceValue']==item['variantValue'] and item['datatype']=='http://www.w3.org/2001/XMLSchema#anyType'
48 assert item['canonicalParentTypeScopeEQ'] and item['repeatedCXTMIdentical'] and item['modelUnchanged'] and not item['readerIntegration']
49 assert item['topicCountBefore']==item['topicCountAfter']
50 olddiag=read(PRIOR/'after-diagnostics.json');newdiag=read(out/'after-diagnostics.json');assert olddiag==newdiag
51 retained=[]
52 for p in sorted((out/'retained').rglob('*')):
53 if p.is_file():
54 rel=p.relative_to(out);assert (PRIOR/rel).exists() and p.read_bytes()==(PRIOR/rel).read_bytes(),str(rel)
55 retained.append({'report':str(rel),'SHA256':sha(p)})
56 controls=[]
57 for r in run['runs']:
58 if '-control-' not in r['label']:continue
59 name=r['label']+'.json';a=read(PRIOR/name);b=read(out/name)
60 assert r['exitCode']==1
61 for key in a:assert a[key]==b[key],(name,key)
62 assert not b['c14nPreflight']['executed'] and not b['c14nPreflight']['probes'] and not b['c14nPreflight']['nativeValues']
63 controls.append({'label':r['label'],'expectedExitCode':1,'previousReportFieldsUnchanged':True,'newProbesNotClaimedExecuted':True})
64 assert len(controls)==113
65 prior=read(PRIOR/'execution.json');statuses=prior['statusesAfter']
66 assert new['cases']==old['cases'] and new['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
67 assert protected=={p:sha(ROOT/p) for p in protected}
68 save(out/'comparison-summary.json',{'kind':'CXML-candidate-vs-independent-XTM-C14N-oracles','W3CC14NQualified':False,'readerIntegration':False,'probes':comparisons,'matchedProbeRows':sum(r['normativeComparison']=='PASS-for-this-probe' for r in comparisons),'differingProbeRows':sum(r['normativeComparison']=='BYTE-DIFFERENCE' for r in comparisons),'nativeValueRows':len(x['nativeValues']),'nativeValuesApplyOnlyToMatchingProbeRows':True})
69 save(out/'failure-controls-audit.json',controls)
70 save(out/'retained-reports.json',retained)
71 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'pinnedCorpusFilesVerified':sum(p.startswith('vendor/cxtm/') for p in protected),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'allProductionAndOriginalSourcesUnchanged':True,'testOnlyChanges':sorted(ALLOWED|{'tests/xml-c14n-preflight.lisp'})})
72 save(out/'execution.json',{'kind':'slice-22q-preflight-characterization-NOT-C14N-qualification','baseCommit':BASE,'branch':subprocess.check_output(['git','branch','--show-current'],cwd=ROOT,text=True).strip(),'runtime':'SBCL 2.5.4','startedAtUTC':run['startedAtUTC'],'finishedAtUTC':run['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'collector':'scripts/validate-slice-22d.py::collect (unchanged)','auditToolSHA256':sha(Path(__file__)),'runs':run['runs'],'standaloneXTMChecks':len(labels),'retainedStandaloneXTMChecks':len(old['checks']),'newCharacterizationChecks':len(additions),'retainedLispChecks':prior['retainedLispChecks'],'stageChecks':prior['stageChecks'],'equivalenceChecks':prior['equivalenceChecks'],'equivalenceReaderPairs':prior['equivalenceReaderPairs'],'faultControls':len(controls),'all124OriginalDiagnosesUnchanged':True,'retainedReportsByteIdentical':len(retained),'statusesBefore':statuses,'statusesAfter':statuses,'originalRegistrationsAdded':0,'productionChanges':[],'xmlQualificationChecks':prior['xmlQualificationChecks'],'pythonUnitTests':prior['pythonUnitTests'],'nfcOracleComparisons':prior['nfcOracleComparisons'],'W3CC14NQualified':False,'XTMReaderIntegration':False,'newDependencies':0,'nativeValueRows':len(x['nativeValues']),'acceptancePassed':True})
73 print(json.dumps({'acceptancePassed':True,'W3CC14NQualified':False,'XTMChecks':len(labels),'newCharacterizationChecks':len(additions),'faultControls':len(controls),'statusesUnchanged':statuses}),flush=True)
74 def main():
75 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true');a=ap.parse_args();out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
76 if not a.audit_only:
77 spec=importlib.util.spec_from_file_location('oldcollector',ROOT/'scripts/validate-slice-22d.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
78 m.EXECUTION_FILES={str(p.relative_to(ROOT)) for folder in ['src','tests'] for p in (ROOT/folder).rglob('*.lisp')}|{'topic-maps.asd','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp','reports/slice-22q/reproduce.py'}
79 m.collect(out,ROOT/'reports/slice-22d/baseline')
80 audit(out)
81 if __name__=='__main__':main()