]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22h.py
examples: explore project work through native Topic Maps (slice 22U)
[topic-maps.git] / scripts / validate-slice-22h.py
1 #!/usr/bin/env python3
2 """Register five already supported XTM mergeMap originals via the existing offline collector.
3 --audit-only verifies saved executions and starts no tests. Output is outside the repository.
4 """
5 import argparse
6 import datetime
7 import hashlib
8 import importlib.util
9 import json
10 import re
11 import subprocess
12 import sys
13 from pathlib import Path
14 sys.dont_write_bytecode=True
15 ROOT=Path(__file__).resolve().parents[1]
16 D=ROOT/'reports/slice-22h';PRIOR=ROOT/'reports/slice-22g/after'
17 BASE='7f4aa941293e555ccb9d96f0c05c651b0549a866'
18 ALLOWED={'topic-maps.asd','tests/xtm-identity.lisp','tests/xtm-mergemap.lisp'}
19 SOURCES=ALLOWED|{'tests/xtm-mergemap-originals.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
20 def read(p):return json.loads(p.read_text())
21 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
22 def save(p,x):p.write_text(json.dumps(x,indent=2,ensure_ascii=False)+'\n')
23 def git(*args):return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
24 def diff(a,b,path=''):
25 if a==b:return []
26 if isinstance(a,dict) and isinstance(b,dict):return [d for k in sorted(a.keys()|b.keys()) for d in diff(a.get(k),b.get(k),path+'/'+k)]
27 if isinstance(a,list) and isinstance(b,list) and len(a)==len(b):return [d for i,(x,y) in enumerate(zip(a,b)) for d in diff(x,y,path+'/'+str(i))]
28 return [{'path':path,'before':a,'after':b}]
29 def audit(out):
30 initial=read(D/'initial-state.json');assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
31 assert set(git('diff','--name-only',BASE).splitlines())&set(initial['trackedFileSHA256'])<=ALLOWED
32 protected={p:h for p,h in initial['trackedFileSHA256'].items() if p not in ALLOWED}
33 assert protected=={p:sha(ROOT/p) for p in protected}
34 run=read(out/'runs.json');assert len(run['runs'])==119
35 assert all(x['exitCode']==x['expectedExitCode'] and x['network']==x['repositoryWrites']=='denied' and sha(out/(x['label']+'.log'))==x['logSHA256'] for x in run['runs'])
36 assert run['executionSourcesSHA256']=={p:sha(ROOT/p) for p in run['executionSourcesSHA256']}
37 assert run['rawReportSHA256']=={p:sha(out/p) for p in run['rawReportSHA256']}
38 old=read(PRIOR/'xtm.json');new=read(out/'xtm.json');before={x['id']:x for x in old['cases']};after={x['id']:x for x in new['cases']}
39 inputs={x['id']:x for x in read(D/'originals-before.json')};targets=set(inputs)
40 assert new['statuses']=={'pass':88,'fail':0,'unsupported':0,'not-run':0} and new['otherCorpusInputs']=={'positiveUnexecuted':33,'negativeUnregisteredUnexecuted':3}
41 assert old['cases']==new['cases'][:len(old['cases'])] and set(after)-set(before)==targets
42 assert len(after)==len(new['cases']) and all(after[id]['status']=='pass' and after[id]['actualOutcome']=='cxtm-match' for id in targets)
43 assert [c for c in new['checks'] if c in set(old['checks'])]==old['checks']
44 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
45 assert labels==new['checks'] and len(labels)==len(set(labels))==2965
46 for key in old:
47 if key not in {'scope','statuses','otherCorpusInputs','checks','cases'}:assert old[key]==new[key],key
48 originals=[];(out/'actual').mkdir(exist_ok=True)
49 assert {x['id'] for x in new['registeredMergemapOriginals']}==targets
50 for row in new['registeredMergemapOriginals']:
51 source=inputs[row['id']];assert row['condition'] is None and row['status']=='pass' and row['firstDifferentByte'] is None
52 raw=bytes.fromhex(row['sourceHex']);childraw=bytes.fromhex(row['childHex'])
53 assert raw==(ROOT/'vendor/cxtm'/row['id']).read_bytes() and childraw==(ROOT/'vendor/cxtm'/source['resourceID']).read_bytes()
54 assert hashlib.sha256(raw).hexdigest()==source['sourceSHA256'] and hashlib.sha256(childraw).hexdigest()==source['resourceSHA256']
55 actual=row['actualCXTM'].encode();expected=(ROOT/'vendor/cxtm/xtm2/baseline'/(Path(row['id']).name+'.cxtm')).read_bytes()
56 assert actual==expected==row['expectedCXTM'].encode() and len(actual)==row['actualBytes']==row['expectedBytes']==source['baselineBytes']
57 assert hashlib.sha256(expected).hexdigest()==source['baselineSHA256']
58 traces=row['qualifiedParserTrace'];assert len(traces)==2
59 assert traces[0]['documentIRI']==row['documentIRI'] and traces[0]['sourceKey']==row['sourceKey'] and traces[0]['originalRootOctetsEQ']
60 assert traces[1]['documentIRI']==row['childDocumentIRI'] and traces[1]['sourceKey']==row['childSourceKey'] and traces[1]['originalChildOctetsEQ']
61 assert traces[0]['bytes']==len(raw) and traces[1]['bytes']==len(childraw)
62 trace=row['importTrace'];assert trace['sourceMapsPreserved'] and trace['qualifiedParserCalls']==2 and trace['model']==row['targetModel'] and trace['model']['canonicalNativeReferencesEQ']
63 assert trace['resolverCalls']==[{'resourceIRI':row['childDocumentIRI'],'purpose':'mergemap','fromDocumentIRI':row['documentIRI'],'notation':'http://psi.topicmaps.org/iso13250/xtm'}]
64 p=out/'actual'/(Path(row['id']).name+'.cxtm');p.write_bytes(actual)
65 originals.append(dict(row,sourceSHA256=source['sourceSHA256'],childSHA256=source['resourceSHA256'],baselineSHA256=source['baselineSHA256'],actualSHA256=sha(p),completeByteExact=True))
66 save(out/'original-contracts.json',originals)
67 # Existing no-resolver exploratory diagnoses are unchanged. They are not the new original executions.
68 assert (out/'after-diagnostics.json').read_bytes()==(PRIOR/'after-diagnostics.json').read_bytes()
69 diagnoses={x['id']:x for x in read(out/'after-diagnostics.json')};assert len(diagnoses)==124
70 inventory=sorted('xtm2/'+sub+'/'+p.name for sub in ['in','invalid'] for p in (ROOT/'vendor/cxtm/xtm2'/sub).glob('*.xtm'))
71 assert len(inventory)==124 and set(inventory)==set(diagnoses)
72 matrix=[]
73 for id in inventory:
74 b=before.get(id,{}).get('status','not-run');a=after.get(id,{}).get('status','not-run')
75 assert (b!=a)==(id in targets)
76 matrix.append({'id':id,'before':b,'after':a,'changed':b!=a,'beforeCaseRecord':before.get(id),'afterCaseRecord':after.get(id),
77 'readerDiagnosisWithoutResolverUnchanged':True,'noResolverCondition':diagnoses[id]['condition'],
78 'newOriginalWithPinnedResolver':next((o for o in originals if o['id']==id),None)})
79 assert sum(x['after']=='pass' for x in matrix)==88 and sum(x['after']=='not-run' for x in matrix)==36
80 save(out/'all-124-original-statuses.json',{'cases':matrix,'registeredChanges':5,'allOtherRegisteredStatusesUnchanged':True,'noResolverDiagnosticChanges':0})
81 save(out/'diagnostic-changes.json',{'originalDiagnoses':124,'changed':0,'unchanged':124,'byteIdenticalToSlice22G':True,'note':'The legacy diagnostic driver omits a resolver. Five new original executions explicitly configure the pinned resolver and are recorded separately.'})
82 faults=[]
83 for x in run['runs']:
84 if '-control-' not in x['label']:continue
85 a=read(PRIOR/(x['label']+'.json'));b=read(out/(x['label']+'.json'));changes=diff(a,b)
86 assert a['cases']==b['cases'] and a['statuses']==b['statuses'] and a['checks']==b['checks'] and x['exitCode']==1
87 assert all(d['path'] in {'/scope','/registeredMergemapOriginals'} for d in changes)
88 assert not b['registeredMergemapOriginals']
89 faults.append({'label':x['label'],'expectedASDFFailureObserved':True,'exitCode':1,'caseRecordsOutcomesAndChecksIdentical':True,'faultCountedAsOriginalPASS':False,'reportDifferences':changes})
90 assert len(faults)==113
91 save(out/'failure-controls.json',{'retained':113,'new':0,'controls':faults,'allTargetCasesAndChecksUnchanged':True})
92 retained=[]
93 for p in sorted((PRIOR/'retained').glob('*.json')):
94 rel=p.relative_to(PRIOR);assert p.read_bytes()==(out/rel).read_bytes(),str(rel)
95 retained.append({'report':str(rel),'sha256':sha(p),'byteIdentical':True})
96 for name in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
97 assert (out/name).read_bytes()==(PRIOR/name).read_bytes();retained.append({'report':name,'sha256':sha(out/name),'byteIdentical':True})
98 assert len(retained)==49;save(out/'retained-reports.json',retained)
99 prior=read(PRIOR/'execution.json');log=(out/'regression.log').read_text()
100 for marker,count in dict(prior['retainedChecks'],**prior['stageChecks']).items():assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',log).group(1))==count
101 assert 'CTM-XTM-EQUIVALENCE-PASS: 487 checks; 27 actual reader pairs;' in log
102 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log and 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
103 lock=read(ROOT/'references/corpus-lock.json')
104 for f in lock['files']:
105 p=ROOT/'vendor/cxtm'/f['path'];assert p.stat().st_size==f['size'] and sha(p)==f['sha256']
106 assert protected=={p:sha(ROOT/p) for p in protected}
107 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
108 'productionSourcesOriginalsBaselinesHistoricalReportsAndCorpusLockUnchanged':True,'productionChanges':[]})
109 save(out/'execution.json',{'kind':'slice-22h-five-existing-XTM-mergeMap-originals','baseCommit':BASE,'branch':git('branch','--show-current'),'runtime':'SBCL 2.5.4',
110 'startedAtUTC':run['startedAtUTC'],'finishedAtUTC':run['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'runs':run['runs'],
111 'collector':'scripts/validate-slice-22d.py::collect (unchanged)','auditToolSHA256':sha(Path(__file__)),'standaloneXTMChecks':len(labels),'retainedStandaloneXTMChecks':len(old['checks']),
112 'newXTMChecks':len(labels)-len(old['checks']),'retainedLispChecks':12100,'retainedChecks':prior['retainedChecks'],'stageChecks':prior['stageChecks'],
113 'equivalenceChecks':487,'equivalenceReaderPairs':27,'faultControls':113,'allOriginalStatusesCompared':124,'diagnosticChanges':0,'registeredStatusChanges':5,
114 'statusesBefore':prior['statusesAfter'],'statusesAfter':{'XTM':{'pass':88,'fail':0,'not-run':36},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':350,'fail':4,'not-run':43}},
115 'originalRegistrationsAdded':5,'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,'pinnedCorpusFilesVerified':len(lock['files']),
116 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'knownCTMFailuresUnchanged':prior['knownCTMFailuresUnchanged'],
117 'productionChanges':[],'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
118 print(json.dumps({'acceptancePassed':True,'XTMChecks':len(labels),'newOriginalPASS':5,'allOriginalStatusesCompared':124,'faultControls':113}))
119 def main():
120 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true');a=ap.parse_args();out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
121 if not a.audit_only:
122 spec=importlib.util.spec_from_file_location('v22d',ROOT/'scripts/validate-slice-22d.py');v=importlib.util.module_from_spec(spec);spec.loader.exec_module(v);v.EXECUTION_FILES=SOURCES;v.collect(out,ROOT/'reports/slice-22d/baseline')
123 audit(out)
124 if __name__=='__main__':main()