]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22g.py
docs(tmdm): coverage matrix, relationship readers, default name types; run report
[topic-maps.git] / scripts / validate-slice-22g.py
1 #!/usr/bin/env python3
2 """Slice 22G: reuse the existing 119-process offline collector; audit saved evidence.
3 --audit-only starts no tests. Output must be outside the repository.
4 """
5 import argparse
6 import datetime
7 import hashlib
8 import importlib.util
9 import json
10 import re
11 import subprocess
12 import sys
13 from pathlib import Path
14 sys.dont_write_bytecode=True
15 ROOT=Path(__file__).resolve().parents[1]
16 D=ROOT/'reports/slice-22g';PRIOR=ROOT/'reports/slice-22f/after'
17 BASE='b50a9d7950c5407b2ec02e1caccd76f73018b45f'
18 ALLOWED={'src/xtm/reader.lisp','topic-maps.asd','tests/xtm-identity.lisp','tests/xtm-name-scopes.lisp','tests/xtm-resource-datatypes.lisp','tests/xtm-mergemap.lisp'}
19 SOURCES=ALLOWED|{'tests/xtm-occurrences.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
20 def read(p):return json.loads(p.read_text())
21 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
22 def save(p,x):p.write_text(json.dumps(x,indent=2,ensure_ascii=False)+'\n')
23 def git(*args):return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
24 def diff(a,b,path=''):
25 if a==b:return []
26 if isinstance(a,dict) and isinstance(b,dict):return [d for k in sorted(a.keys()|b.keys()) for d in diff(a.get(k),b.get(k),path+'/'+k)]
27 if isinstance(a,list) and isinstance(b,list) and len(a)==len(b):return [d for i,(x,y) in enumerate(zip(a,b)) for d in diff(x,y,path+'/'+str(i))]
28 return [{'path':path,'before':a,'after':b}]
29 def audit(out):
30 initial=read(D/'initial-state.json');assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
31 assert set(git('diff','--name-only',BASE).splitlines())&set(initial['trackedFileSHA256'])<=ALLOWED
32 protected={p:h for p,h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
33 assert protected=={p:sha(ROOT/p) for p in protected}
34 runs=read(out/'runs.json');assert len(runs['runs'])==119
35 assert all(x['exitCode']==x['expectedExitCode'] and sha(out/(x['label']+'.log'))==x['logSHA256'] for x in runs['runs'])
36 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
37 assert runs['rawReportSHA256']=={p:sha(out/p) for p in runs['rawReportSHA256']}
38 old=read(PRIOR/'xtm.json');new=read(out/'xtm.json');targets={x['id'] for x in read(D/'originals-before.json')}
39 assert new['statuses']=={'pass':83,'fail':0,'unsupported':0,'not-run':0} and len(new['cases'])==83
40 assert new['otherCorpusInputs']=={'positiveUnexecuted':38,'negativeUnregisteredUnexecuted':3}
41 assert new['cases'][:72]==old['cases'] and {x['id'] for x in new['cases'][72:]}==targets
42 assert len({x['id'] for x in new['cases']})==83 and all(x['status']=='pass' and x['actualOutcome']=='cxtm-match' for x in new['cases'][72:])
43 assert new['checks'][:len(old['checks'])]!=[]
44 assert [x for x in new['checks'] if x in set(old['checks'])]==old['checks']
45 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
46 assert labels==new['checks'] and len(labels)==len(set(labels))
47 actualdir=out/'actual';actualdir.mkdir(exist_ok=True);originals=[]
48 sourceinfo={x['id']:x for x in read(D/'originals-before.json')}
49 assert len(new['occurrenceOriginals'])==11
50 for row in new['occurrenceOriginals']:
51 source=sourceinfo[row['id']];assert row['condition'] is None and row['status']=='pass'
52 assert row['qualifiedParserCalls']==1 and row['originalOctetsEQ'] and row['oldHandlesCanonicalAndLive']
53 assert row['model']['canonicalOccurrenceReferencesEQ'] and row['firstDifferentByte'] is None
54 raw=bytes.fromhex(row['sourceHex']);ac=row['actualCXTM'].encode();expected=(ROOT/'vendor/cxtm/xtm2/baseline'/(Path(row['id']).name+'.cxtm')).read_bytes()
55 assert raw==(ROOT/'vendor/cxtm'/row['id']).read_bytes() and hashlib.sha256(raw).hexdigest()==source['sourceSHA256']
56 assert ac==expected==row['expectedCXTM'].encode() and len(ac)==row['actualBytes']==row['expectedBytes']==source['baselineBytes']
57 p=actualdir/(Path(row['id']).name+'.cxtm');p.write_bytes(ac)
58 originals.append(dict(row,sourceSHA256=source['sourceSHA256'],baselineSHA256=sha(p),actualSHA256=sha(p),completeByteExact=True))
59 save(out/'original-contracts.json',originals)
60 controls=new['occurrenceControls'];positive=[c for c in controls if c.get('model')];negative=[c for c in controls if c.get('condition')]
61 assert len(positive)==19 and len(negative)==27 and len(controls)==46
62 assert all(c['model']['canonicalOccurrenceReferencesEQ'] for c in positive)
63 merges=[c for c in positive if c['label'].startswith('merge-')]
64 assert len(merges)==9
65 for c in merges:
66 h=c['retainedHandleEvidence'];assert c['occurrenceCountsAfterEachRead']==[1,2] and c['model']['occurrenceCount']==1
67 assert h['readHandleCount']==h['distinctReadHandles']==2 and h['canonicalGroupsInFinalMap']==[1,1] and h['replacedReadHandles']==[True,True]
68 assert {c['condition']['class'] for c in negative}=={'XTM-REJECTION','XTM-UNSUPPORTED','XTM-XML-ERROR','OCCURRENCE-ERROR','OCCURRENCE-IDENTIFIER-ERROR'}
69 save(out/'native-and-negative-controls.json',{'positiveControls':19,'negativeControls':27,'mergingDimensions':['parent','type','scope'],'identityKinds':['SI','SL','II'],'controls':controls})
70 # Compare all original diagnostics, including unregistered capability-boundary effects.
71 before={x['id']:x for x in read(PRIOR/'after-diagnostics.json')};after={x['id']:x for x in read(out/'after-diagnostics.json')};assert len(before)==len(after)==124
72 changes=[]
73 for id in sorted(before):
74 if before[id]==after[id]:continue
75 if id in targets:
76 assert after[id]['condition'] is None and after[id]['byteExact'];reason='Newly implemented supported occurrence core; full original baseline match.'
77 else:
78 assert id in {'xtm2/in/itemid-occurrence.xtm','xtm2/in/occurrence-duplicate-iid.xtm','xtm2/in/occurrence-duplicate-iid2.xtm','xtm2/in/occurrence-reifier.xtm','xtm2/in/occurrence-duplicate-reifier.xtm','xtm2/invalid/reifier-collision.xtm'}
79 assert after[id]['condition']['class']=='XTM-UNSUPPORTED' and after[id]['condition']['code']=='unsupported-construct'
80 assert 'occurrence' in after[id]['condition']['detail'];reason='Precise excluded occurrence II/reification boundary, not a registered PASS.'
81 changes.append({'id':id,'reason':reason,'before':before[id],'after':after[id],'differences':diff(before[id],after[id])})
82 assert {x['id'] for x in changes}==targets|{'xtm2/in/itemid-occurrence.xtm','xtm2/in/occurrence-duplicate-iid.xtm','xtm2/in/occurrence-duplicate-iid2.xtm','xtm2/in/occurrence-reifier.xtm','xtm2/in/occurrence-duplicate-reifier.xtm','xtm2/invalid/reifier-collision.xtm'}
83 save(out/'diagnostic-changes.json',{'originalDiagnoses':124,'changed':len(changes),'unchanged':124-len(changes),'changes':changes})
84 faults=[]
85 for x in runs['runs']:
86 if '-control-' not in x['label']:continue
87 a=read(PRIOR/(x['label']+'.json'));b=read(out/(x['label']+'.json'))
88 assert a['cases']==b['cases'] and a['statuses']==b['statuses'],x['label']
89 assert x['exitCode']==1 and len(a['checks'])==len(b['checks'])
90 differences=diff(a,b)
91 permitted={'/scope','/occurrenceControls','/occurrenceOriginals','/scopeControls/35/sourceText',
92 '/scopeControls/35/condition/column','/scopeControls/35/condition/detail','/scopeControls/35/condition/message',
93 '/mapReifierControls/31/condition/detail','/mapReifierControls/31/condition/message',
94 '/variantReifierControls/35/condition/detail','/variantReifierControls/35/condition/message'}
95 assert all(d['path'] in permitted for d in differences),x['label']
96 faults.append({'label':x['label'],'exitCode':1,'expectedASDFFailureObserved':True,'caseRecordsAndOutcomesIdentical':True,'checkCoveragePreserved':True,'behavioralDifferences':differences})
97 assert len(faults)==113
98 save(out/'failure-controls.json',{'retained':113,'new':0,'controls':faults,'noFaultCountedAsPASS':True})
99 # Preserve all non-XTM reports. The old equivalence gap has a more precise message.
100 retained=[]
101 for p in sorted((PRIOR/'retained').glob('*.json')):
102 rel=p.relative_to(PRIOR)
103 if p.name=='xtm-mergemap.equivalence.json':continue
104 assert p.read_bytes()==(out/rel).read_bytes(),str(rel)
105 retained.append({'report':str(rel),'sha256':sha(p),'byteIdentical':True})
106 for name in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
107 assert (PRIOR/name).read_bytes()==(out/name).read_bytes();retained.append({'report':name,'sha256':sha(out/name),'byteIdentical':True})
108 assert len(retained)==48
109 save(out/'retained-reports.json',retained)
110 eqold=read(PRIOR/'retained/xtm-mergemap.equivalence.json');eq=read(out/'retained/xtm-mergemap.equivalence.json')
111 assert eq['condition'] is None and eq['checks']==eqold['checks'] and len(eq['checks'])==487
112 assert eq['pairs'][:30]==eqold['pairs'][:30] and len(eq['pairs'])==33
113 eqdiff=diff(eqold,eq)
114 assert all(x['path'].startswith('/pairs/30/xtmCondition/') and x['path'].split('/')[-1] in {'message','detail'} for x in eqdiff)
115 save(out/'equivalence-preservation.json',{'checks':487,'readerPairs':27,'originalPairSourcesModelsAndFullCXTMUnchanged':True,'differences':eqdiff,'scope':'Only the existing unsupported occurrence-reification gap message is refined.'})
116 prior=read(PRIOR/'execution.json');log=(out/'regression.log').read_text()
117 for marker,count in dict(prior['retainedChecks'],**prior['stageChecks']).items():assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',log).group(1))==count
118 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log
119 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
120 lock=read(ROOT/'references/corpus-lock.json')
121 for x in lock['files']:
122 p=ROOT/'vendor/cxtm'/x['path'];assert p.stat().st_size==x['size'] and sha(p)==x['sha256']
123 assert protected=={p:sha(ROOT/p) for p in protected}
124 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'originalSourcesBaselinesHistoricalReportsUnchanged':True,'onlyProductionChange':'src/xtm/reader.lisp'})
125 save(out/'retargeted-observations.json',{'retargetings':read(D/'retargetings.json'),'existingTestReportDifferences':diff(old,{k:v for k,v in new.items() if k not in {'occurrenceOriginals','occurrenceControls'}})})
126 save(out/'execution.json',{'kind':'slice-22g-XTM-occurrence-core','baseCommit':BASE,'branch':git('branch','--show-current'),'runtime':'SBCL 2.5.4',
127 'startedAtUTC':runs['startedAtUTC'],'finishedAtUTC':runs['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'runs':runs['runs'],
128 'collector':'scripts/validate-slice-22d.py::collect (unchanged)','auditToolSHA256':sha(Path(__file__)),'standaloneXTMChecks':len(labels),'retainedStandaloneXTMChecks':len(old['checks']),
129 'newXTMChecks':len(labels)-len(old['checks']),'retainedLispChecks':12100,'retainedChecks':prior['retainedChecks'],'stageChecks':prior['stageChecks'],
130 'equivalenceChecks':487,'equivalenceReaderPairs':27,'faultControls':113,'originalDiagnoses':124,'diagnosticChanges':len(changes),
131 'statusesBefore':prior['statusesAfter'],'statusesAfter':{'XTM':{'pass':83,'fail':0,'not-run':41},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':345,'fail':4,'not-run':48}},
132 'originalRegistrationsAdded':11,'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,'pinnedCorpusFilesVerified':len(lock['files']),
133 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'knownCTMFailuresUnchanged':prior['knownCTMFailuresUnchanged'],
134 'productionChanges':['src/xtm/reader.lisp'],'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
135 print(json.dumps({'acceptancePassed':True,'XTMChecks':len(labels),'newOriginalPASS':11,'diagnosticChanges':len(changes),'faultControls':113}))
136 def main():
137 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true');a=ap.parse_args();out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
138 if not a.audit_only:
139 spec=importlib.util.spec_from_file_location('v22d',ROOT/'scripts/validate-slice-22d.py');v=importlib.util.module_from_spec(spec);spec.loader.exec_module(v);v.EXECUTION_FILES=SOURCES;v.collect(out,ROOT/'reports/slice-22d/baseline')
140 audit(out)
141 if __name__=='__main__':main()