]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22n.py
docs(tmdm): coverage matrix, relationship readers, default name types; run report
[topic-maps.git] / scripts / validate-slice-22n.py
1 #!/usr/bin/env python3
2 """Slice 22N offline execution and acceptance audit using the existing collector.
3 Use an unused --output outside the repository; --audit-only rechecks saved runs.
4 """
5 import argparse,datetime,hashlib,importlib.util,json,re,subprocess,sys
6 from pathlib import Path
7 sys.dont_write_bytecode=True
8 ROOT=Path(__file__).resolve().parents[1]
9 D=ROOT/'reports/slice-22n';PRIOR=ROOT/'reports/slice-22m/after'
10 BASE='acc9eb65b0cce71409ad84841f082f5ca125c839'
11 ALLOWED={'tests/xtm-associations.lisp', 'tests/xtm-map-reification.lisp', 'tests/xtm-name-scopes.lisp', 'tests/ctm-xtm-equivalence.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-association-reification.lisp', 'topic-maps.asd', 'src/xtm/reader.lisp', 'tests/xtm-occurrence-reification.lisp'}
12 SOURCES=ALLOWED|{'tests/xtm-role-reifier-native.lisp','tests/xtm-role-reification.lisp','reports/slice-22n/api-preflight.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
13 def read(p):return json.loads(p.read_text())
14 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
15 def save(p,x):p.write_text(json.dumps(x,indent=2,ensure_ascii=False)+'\n')
16 def git(*args):return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
17 def diff(a,b,path=''):
18 if a==b:return []
19 if isinstance(a,dict) and isinstance(b,dict):return [d for k in sorted(a.keys()|b.keys()) for d in diff(a.get(k),b.get(k),path+'/'+k)]
20 if isinstance(a,list) and isinstance(b,list) and len(a)==len(b):return [d for i,(x,y) in enumerate(zip(a,b)) for d in diff(x,y,path+'/'+str(i))]
21 return [{'path':path,'before':a,'after':b}]
22 def audit(out):
23 initial=read(D/'initial-state.json');assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
24 assert set(git('diff','--name-only',BASE).splitlines())&set(initial['trackedFileSHA256'])<=ALLOWED
25 protected={p:h for p,h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
26 assert protected=={p:sha(ROOT/p) for p in protected}
27 run=read(out/'runs.json');assert len(run['runs'])==119
28 assert all(x['exitCode']==x['expectedExitCode'] and x['network']==x['repositoryWrites']=='denied' and sha(out/(x['label']+'.log'))==x['logSHA256'] for x in run['runs'])
29 assert run['executionSourcesSHA256']=={p:sha(ROOT/p) for p in run['executionSourcesSHA256']}
30 assert run['rawReportSHA256']=={p:sha(out/p) for p in run['rawReportSHA256']}
31 old=read(PRIOR/'xtm.json');new=read(out/'xtm.json');before={x['id']:x for x in old['cases']};after={x['id']:x for x in new['cases']}
32 inputs={x['id']:x for x in read(D/'originals-before.json')};targets=set(inputs)
33 assert old['statuses']=={'pass':115,'fail':0,'unsupported':0,'not-run':0}
34 assert new['statuses']=={'pass':121,'fail':0,'unsupported':0,'not-run':0} and new['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
35 assert old['cases']==new['cases'][:len(old['cases'])] and set(after)-set(before)==targets
36 assert len(after)==len(new['cases']) and all(after[id]['status']=='pass' for id in targets)
37 assert [c for c in new['checks'] if c in set(old['checks'])]==old['checks']
38 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
39 assert labels==new['checks'] and len(labels)==len(set(labels))==3743
40 permitted={'scope','statuses','otherCorpusInputs','checks','cases','scopeControls','mapReifierControls','associationControls','occurrenceReifierControls','associationReifierControls','mergemapControls'}
41 for key in old:
42 if key not in permitted:assert old[key]==new[key],key
43 retargets=[]
44 for key,count in {'scopeControls':1,'mapReifierControls':1,'associationControls':4,'occurrenceReifierControls':2,'associationReifierControls':1}.items():
45 assert len(old[key])==len(new[key]);changed=[(a,b) for a,b in zip(old[key],new[key]) if a!=b];assert len(changed)==count,(key,changed)
46 for a,b in changed:
47 assert a['label']==b['label'] and a['condition']['class']=='XTM-UNSUPPORTED' and b['condition']['class']=='XTM-REJECTION'
48 assert a['condition']['code']=='unsupported-construct' and b['condition']['code']=='invalid-percent-escape'
49 assert a['condition']['detail']=='XTM role reification is not implemented in this subset.'
50 assert b['condition']['detail']=='Expected two hexadecimal digits.'
51 expected=re.sub(r"(<role reifier=')([^']+)(')",lambda m:m.group(1)+m.group(2)+'%GG'+m.group(3),a['sourceText'],count=1)
52 assert b['sourceText']==expected
53 for k in ['sourceKey','documentIRI','line']:assert a['condition'][k]==b['condition'][k]
54 retargets.append({'reportKey':key,'label':a['label'],'before':a,'after':b,'differences':diff(a,b)})
55 assert new['mergemapControls'][1:]==old['mergemapControls']
56 imported=new['mergemapControls'][0];assert imported['label']=='Role-Reification-import' and imported['sourceMapsPreserved'] and imported['qualifiedParserCalls']==2
57 assert len(imported['resolverCalls'])==1 and imported['model']['canonicalNativeReferencesEQ']
58 for fixture in read(D/'retargetings.json'):
59 if 'afterSource' in fixture:assert fixture['afterSource'] in (ROOT/fixture['file']).read_text()
60 save(out/'retargeted-control-observations.json',retargets)
61 originals=[];(out/'actual').mkdir(exist_ok=True)
62 assert {x['id'] for x in new['roleReifierOriginals']}==targets
63 for row in new['roleReifierOriginals']:
64 source=inputs[row['id']];raw=bytes.fromhex(row['sourceHex'])
65 assert raw==(ROOT/'vendor/cxtm'/row['id']).read_bytes() and hashlib.sha256(raw).hexdigest()==source['sourceSHA256']
66 assert row['status']=='pass' and row['originalOctetsEQ'] and row['qualifiedParserCalls']==1
67 if '/invalid/' in row['id']:
68 c=row['condition'];assert row['intendedCauseReached'] and c['class']=='REIFICATION-ERROR' and c['code']=='reifier-conflict'
69 assert c['detail']=='Topic already reifies a different information item.' and c['line'] is c['column'] is c['sourceKey'] is c['documentIRI'] is None
70 assert len(row['actualNativeAssignments'])==1
71 frame=row['actualNativeAssignments'][0]
72 assert frame['assignedItemKind']==frame['priorReifiedItemKind']=='ASSOCIATION-ROLE'
73 assert frame['actualMapAssociationCount']==frame['actualMapRoleCount']==2
74 assert frame['sameRoleTypeAndPlayerEQ'] and frame['differentParentsEQ'] and frame['nativeAssignmentUnchanged']
75 assert frame['assignedParentTypeII']==[row['documentIRI']+'#assoctype2'] and frame['priorParentTypeII']==[row['documentIRI']+'#assoctype']
76 assert after[row['id']]['actualOutcome']=='ctm-rejection'
77 originals.append(dict(row,sourceSHA256=source['sourceSHA256'],baselineSHA256=None))
78 else:
79 assert row['condition'] is None and row['firstDifferentByte'] is None and row['oldHandlesCanonicalAndLive'] and row['model']['canonicalRoleReifierInverseEQ']
80 actual=row['actualCXTM'].encode();expected=(ROOT/'vendor/cxtm/xtm2/baseline'/(Path(row['id']).name+'.cxtm')).read_bytes()
81 assert actual==expected==row['expectedCXTM'].encode() and len(actual)==row['actualBytes']==row['expectedBytes']==source['baselineBytes']
82 assert hashlib.sha256(expected).hexdigest()==source['baselineSHA256'] and after[row['id']]['actualOutcome']=='cxtm-match'
83 p=out/'actual'/(Path(row['id']).name+'.cxtm');p.write_bytes(actual)
84 originals.append(dict(row,actualSHA256=sha(p),completeByteExact=True,sourceSHA256=source['sourceSHA256'],baselineSHA256=source['baselineSHA256']))
85 save(out/'original-contracts.json',originals)
86 # Compare all 124 actual Reader diagnoses and registered statuses.
87 oldd={x['id']:x for x in read(PRIOR/'after-diagnostics.json')};newd={x['id']:x for x in read(out/'after-diagnostics.json')}
88 inventory=sorted('xtm2/'+sub+'/'+p.name for sub in ['in','invalid'] for p in (ROOT/'vendor/cxtm/xtm2'/sub).glob('*.xtm'))
89 assert len(inventory)==124 and set(inventory)==set(oldd)==set(newd)
90 changes=[];matrix=[]
91 for id in inventory:
92 a,b=oldd[id],newd[id];ds=diff(a,b)
93 if ds:
94 assert id in targets,id
95 if '/in/' in id:assert b['byteExact'] and b['condition'] is None
96 else:assert b['condition']['class']=='REIFICATION-ERROR' and b['condition']['code']=='reifier-conflict'
97 changes.append({'id':id,'reason':'new Role Reification reaches intended original result','differences':ds})
98 elif id in targets:raise AssertionError(id)
99 bs=before.get(id,{}).get('status','not-run');ns=after.get(id,{}).get('status','not-run');assert (bs!=ns)==(id in targets)
100 matrix.append({'id':id,'before':bs,'after':ns,'registeredStatusChanged':bs!=ns,'diagnosisChanged':bool(ds),'beforeDiagnosis':a,'afterDiagnosis':b})
101 assert all(newd[id]==oldd[id] for id in before) and targets=={x['id'] for x in changes}
102 save(out/'diagnostic-changes.json',{'originalDiagnoses':124,'changed':len(changes),'unchanged':124-len(changes),'changes':changes})
103 save(out/'all-124-original-statuses.json',{'cases':matrix,'registeredChanges':6,'previous115OriginalPASSUnchanged':True})
104 faults=[]
105 for x in run['runs']:
106 if '-control-' not in x['label']:continue
107 a=read(PRIOR/(x['label']+'.json'));b=read(out/(x['label']+'.json'));ds=diff(a,b)
108 assert a['cases']==b['cases'] and a['statuses']==b['statuses'] and a['checks']==b['checks'] and x['exitCode']==1
109 aa=dict(a);bb=dict(b)
110 for k in ['scope','roleReifierNative','roleReifierOriginals','roleReifierControls']:aa.pop(k,None);bb.pop(k,None)
111 for rt in retargets:
112 key=rt['reportKey'];aa[key]=[rt['after'] if row==rt['before'] else row for row in aa[key]]
113 assert aa==bb,(x['label'],diff(aa,bb))
114 assert not b['roleReifierNative'] and not b['roleReifierOriginals'] and not b['roleReifierControls']
115 faults.append({'label':x['label'],'expectedASDFFailureObserved':True,'exitCode':1,'caseRecordsOutcomesAndChecksIdentical':True,'faultCountedAsOriginalPASS':False,'reportDifferences':ds})
116 assert len(faults)==113
117 save(out/'failure-controls.json',{'retained':113,'new':0,'controls':faults,'allTargetCasesAndChecksUnchanged':True,'failureDriverUnchanged':True})
118 retained=[]
119 for p in sorted((PRIOR/'retained').glob('*.json')):
120 rel=p.relative_to(PRIOR)
121 if p.name.endswith('.equivalence.json'):continue
122 assert p.read_bytes()==(out/rel).read_bytes(),str(rel)
123 retained.append({'report':str(rel),'sha256':sha(p),'byteIdentical':True})
124 for name in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
125 assert (out/name).read_bytes()==(PRIOR/name).read_bytes();retained.append({'report':name,'sha256':sha(out/name),'byteIdentical':True})
126 assert len(retained)==48;save(out/'retained-reports.json',retained)
127 eqpath=next((out/'retained').glob('*.equivalence.json'));eqa=read(PRIOR/eqpath.relative_to(out));eqb=read(eqpath)
128 oldpairs={x['id']:x for x in eqa['pairs']};pairs={x['id']:x for x in eqb['pairs']};gap='association-role-scope-reifiers'
129 assert oldpairs.keys()==pairs.keys() and all(oldpairs[id]==pairs[id] for id in oldpairs if id!=gap)
130 a,b=oldpairs[gap],pairs[gap];assert a['ctmSource']==b['ctmSource'] and a['xtmSource']==b['xtmSource']
131 assert b['kind']=='direct-reader-equivalence-pair' and b['nativePropertiesEqual'] and b['ctmCanonicalEQ'] and b['xtmCanonicalEQ']
132 assert b['byteExact'] and b['firstDifferentByte'] is None and b['ctmCondition'] is b['xtmCondition'] is None and b['ctmBytes']==b['xtmBytes']
133 assert a['ctmCXTM']==b['ctmCXTM']==b['xtmCXTM']
134 oldgapchecks=[c for c in eqa['checks'] if 'gap:'+gap in c];assert len(oldgapchecks)==2
135 remaining=[c for c in eqa['checks'] if c not in oldgapchecks]
136 assert [c for c in eqb['checks'] if c in set(remaining)]==remaining
137 assert sum(x['kind']=='direct-reader-equivalence-pair' for x in eqb['pairs'])==30 and not any(x['kind']=='reader-capability-gap-not-equivalence-PASS' for x in eqb['pairs'])
138 save(out/'equivalence-progress.json',{'allPrevious29PairsAnd3ContrastsUnchanged':True,'resolvedGapOriginalSourcesUnchanged':True,'checks':len(eqb['checks']),'actualPairs':30,'remainingGaps':0,'before':a,'after':b,'replacedGapChecks':oldgapchecks})
139 # Native gate really executed on the original Reader, before its edit.
140 gate=read(D/'native-before-execution.json');native=read(D/'native-before/report.json')
141 assert gate['exitCode']==0 and gate['beforeAnyProductionChange'] and gate['network']==gate['repositoryWrites']=='denied'
142 assert gate['executionSourceSHA256']['src/xtm/reader.lisp']==initial['trackedFileSHA256']['src/xtm/reader.lisp']
143 for p,h in gate['executionSourceSHA256'].items():
144 if p!='src/xtm/reader.lisp':assert sha(ROOT/p)==h
145 assert sha(D/'native-before/run.log')==gate['logSHA256'] and sha(D/'native-before/report.json')==gate['reportSHA256']
146 assert len(native['checks'])==15 and not native['ReaderInvoked'] and native['originalRegistrationsAdded']==0
147 assert native['observations']==new['roleReifierNative'] and all(c in new['checks'] for c in native['checks'])
148 assert len(new['roleReifierControls'])==40
149 save(out/'native-preflight-confirmation.json',{'executedBeforeReaderChange':True,'nativeChecksBefore':15,'sameNativeChecksAndObservationsRepeatedInASDF':True,'nativeExecution':gate})
150 prior=read(PRIOR/'execution.json');log=(out/'regression.log').read_text()
151 for marker,count in dict(prior['retainedChecks'],**prior['stageChecks']).items():assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',log).group(1))==count
152 assert f"CTM-XTM-EQUIVALENCE-PASS: {len(eqb['checks'])} checks; 30 actual reader pairs;" in log
153 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
154 lock=read(ROOT/'references/corpus-lock.json')
155 for f in lock['files']:
156 p=ROOT/'vendor/cxtm'/f['path'];assert p.stat().st_size==f['size'] and sha(p)==f['sha256']
157 assert protected=={p:sha(ROOT/p) for p in protected}
158 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
159 'originalsBaselinesHistoricalReportsAndCorpusLockUnchanged':True,'productionChanges':['src/xtm/reader.lisp'],'TMDM_CTM_CXTM_XMLUnchanged':True})
160 save(out/'execution.json',{'kind':'slice-22n-XTM-role-reification','baseCommit':BASE,'branch':git('branch','--show-current'),'runtime':'SBCL 2.5.4',
161 'startedAtUTC':run['startedAtUTC'],'finishedAtUTC':run['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'runs':run['runs'],
162 'collector':'scripts/validate-slice-22d.py::collect (unchanged)','auditToolSHA256':sha(Path(__file__)),'standaloneXTMChecks':len(labels),'retainedStandaloneXTMChecks':len(old['checks']),
163 'newXTMChecks':len(labels)-len(old['checks']),'retainedLispChecks':12100,'retainedChecks':prior['retainedChecks'],'stageChecks':prior['stageChecks'],
164 'equivalenceChecks':len(eqb['checks']),'equivalenceReaderPairs':30,'faultControls':113,'allOriginalStatusesCompared':124,'diagnosticChanges':len(changes),'registeredStatusChanges':6,
165 'statusesBefore':prior['statusesAfter'],'statusesAfter':{'XTM':{'pass':121,'fail':0,'not-run':3},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':383,'fail':4,'not-run':10}},
166 'originalRegistrationsAdded':6,'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,'pinnedCorpusFilesVerified':len(lock['files']),
167 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'knownCTMFailuresUnchanged':prior['knownCTMFailuresUnchanged'],
168 'productionChanges':['src/xtm/reader.lisp'],'nativePreflightBeforeReaderChange':gate,'nativeGateChecks':15,'totalFreshProcessesIncludingBeforeGate':120,'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
169 print(json.dumps({'acceptancePassed':True,'XTMChecks':len(labels),'newOriginalPASS':6,'allOriginalStatusesCompared':124,'diagnosticChanges':len(changes),'faultControls':113}))
170 def main():
171 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true');a=ap.parse_args();out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
172 if not a.audit_only:
173 spec=importlib.util.spec_from_file_location('v22d',ROOT/'scripts/validate-slice-22d.py');v=importlib.util.module_from_spec(spec);spec.loader.exec_module(v);v.EXECUTION_FILES=SOURCES;v.collect(out,ROOT/'reports/slice-22d/baseline')
174 audit(out)
175 if __name__=='__main__':main()