]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22g/final-audit.py
perf(tmdm): index association normalization merge candidates
[topic-maps.git] / reports / slice-22g / final-audit.py
1 #!/usr/bin/env python3
2 """Audit committed-candidate Slice 22G scope and saved execution; starts no tests."""
3 from pathlib import Path
4 import ast
5 import datetime
6 import hashlib
7 import json
8 import re
9 import subprocess
10 import sys
11 sys.dont_write_bytecode=True
12 ROOT=Path(__file__).resolve().parents[2];D=ROOT/'reports/slice-22g';A=D/'after';P=ROOT/'reports/slice-22f/after'
13 BASE='b50a9d7950c5407b2ec02e1caccd76f73018b45f'
14 ALLOWED={'src/xtm/reader.lisp','topic-maps.asd','tests/xtm-identity.lisp','tests/xtm-name-scopes.lisp','tests/xtm-resource-datatypes.lisp','tests/xtm-mergemap.lisp'}
15 NEW={'tests/xtm-occurrences.lisp','scripts/validate-slice-22g.py','docs/slice-22g-xtm-occurrence-core.md'}
16 def read(p):return json.loads(p.read_text())
17 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
18 def git(*args):return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
19 s=read(D/'initial-state.json');assert s['baseCommit']==BASE and s['initialIndexAndWorktreeClean']
20 assert set(s['trackedFileSHA256'])==set(git('ls-tree','-r','--name-only',BASE).splitlines())
21 protected={p:h for p,h in s['trackedFileSHA256'].items() if p not in ALLOWED}
22 assert protected=={p:sha(ROOT/p) for p in protected}
23 changed=set(git('diff','--name-only',BASE).splitlines());untracked=set(git('ls-files','--others','--exclude-standard').splitlines())
24 assert all(p in ALLOWED|NEW or p.startswith('reports/slice-22g/') for p in changed|untracked)
25 assert {p for p in changed if p.startswith('src/')}=={'src/xtm/reader.lisp'}
26 e=read(A/'execution.json');runs=read(A/'runs.json');assert e['acceptancePassed'] and e['auditToolSHA256']==sha(ROOT/'scripts/validate-slice-22g.py')
27 assert len(runs['runs'])==119 and all(r['exitCode']==r['expectedExitCode'] and r['freshProcess'] and r['network']==r['repositoryWrites']==r['hyperdocWrites']=='denied' and sha(A/(r['label']+'.log'))==r['logSHA256'] for r in runs['runs'])
28 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
29 assert runs['rawReportSHA256']=={p:sha(A/p) for p in runs['rawReportSHA256']}
30 assert e['standaloneXTMChecks']==2869 and e['newXTMChecks']==165 and e['retainedStandaloneXTMChecks']==2704 and e['retainedLispChecks']==12100
31 assert e['statusesAfter']=={'XTM':{'pass':83,'fail':0,'not-run':41},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':345,'fail':4,'not-run':48}}
32 old=read(P/'xtm.json');new=read(A/'xtm.json');assert old['cases']==new['cases'][:72]
33 assert len(new['cases'])==len({r['id'] for r in new['cases']})==83
34 assert [c for c in new['checks'] if c in set(old['checks'])]==old['checks']
35 assert len(new['checks'])==len(set(new['checks']))==2869
36 assert re.findall(r'^XTM-CHECK-PASS (.+)$',(A/'xtm.log').read_text(),re.M)==new['checks']
37 originals=read(A/'original-contracts.json');inputs=read(D/'originals-before.json');assert len(originals)==len(inputs)==11
38 assert {r['id'] for r in originals}=={r['id'] for r in new['cases'][72:]}=={r['id'] for r in inputs}
39 for o in originals:
40 p=ROOT/'vendor/cxtm'/o['id'];b=ROOT/'vendor/cxtm/xtm2/baseline'/(p.name+'.cxtm');actual=A/'actual'/(p.name+'.cxtm')
41 assert sha(p)==o['sourceSHA256'] and p.read_bytes()==bytes.fromhex(o['sourceHex'])
42 assert actual.read_bytes()==b.read_bytes()==o['actualCXTM'].encode()==o['expectedCXTM'].encode()
43 assert sha(actual)==sha(b)==o['actualSHA256']==o['baselineSHA256']
44 assert o['status']=='pass' and o['condition'] is None and o['firstDifferentByte'] is None and o['completeByteExact']
45 assert o['qualifiedParserCalls']==1 and o['originalOctetsEQ'] and o['oldHandlesCanonicalAndLive'] and o['model']['canonicalOccurrenceReferencesEQ']
46 assert o['actualBytes']==o['expectedBytes']==actual.stat().st_size
47 controls=read(A/'native-and-negative-controls.json');assert controls['positiveControls']==19 and controls['negativeControls']==27
48 for c in controls['controls']:
49 if c.get('model'):assert c['model']['canonicalOccurrenceReferencesEQ']
50 if c['label'].startswith('merge-'):
51 h=c['retainedHandleEvidence'];assert h['distinctReadHandles']==2 and h['canonicalGroupsInFinalMap']==[1,1] and h['replacedReadHandles']==[True,True] and c['occurrenceCountsAfterEachRead']==[1,2]
52 delta=read(A/'diagnostic-changes.json');assert delta['originalDiagnoses']==124 and delta['changed']==17 and delta['unchanged']==107
53 assert len({r['id'] for r in delta['changes']}&{r['id'] for r in originals})==11
54 faults=read(A/'failure-controls.json');assert faults['retained']==113 and faults['new']==0 and faults['noFaultCountedAsPASS'] and len(faults['controls'])==113
55 for f in faults['controls']:
56 assert f['exitCode']==1 and f['caseRecordsAndOutcomesIdentical'] and f['checkCoveragePreserved'] and f['expectedASDFFailureObserved']
57 assert read(A/(f['label']+'.json'))['cases']==read(P/(f['label']+'.json'))['cases']
58 retained=read(A/'retained-reports.json');assert len(retained)==48
59 for f in retained:assert sha(A/f['report'])==f['sha256'] and (A/f['report']).read_bytes()==(P/f['report']).read_bytes()
60 eq=read(A/'equivalence-preservation.json');assert eq['checks']==487 and eq['readerPairs']==27 and eq['originalPairSourcesModelsAndFullCXTMUnchanged']
61 assert read(A/'retained/xtm-mergemap.equivalence.json')['pairs'][:30]==read(P/'retained/xtm-mergemap.equivalence.json')['pairs'][:30]
62 ret=read(D/'retargetings.json');assert len(ret)==4
63 for x in ret:
64 assert x['baseFileSHA256']==s['trackedFileSHA256'][x['file']] and x['finalFileSHA256']==sha(ROOT/x['file'])
65 assert x['before'] in subprocess.check_output(['git','show',BASE+':'+x['file']],cwd=ROOT,text=True)
66 assert x['after'] in (ROOT/x['file']).read_text() and x['expectedClass']=='XTM-UNSUPPORTED' and x['expectedCode']=='unsupported-construct'
67 lock=read(ROOT/'references/corpus-lock.json');assert len(lock['files'])==2006
68 for x in lock['files']:
69 p=ROOT/'vendor/cxtm'/x['path'];assert p.stat().st_size==x['size'] and sha(p)==x['sha256']
70 assert len(protected)==8459 and sum(p.startswith('reports/') for p in protected)==6178
71 for p in ['scripts/validate-slice-22g.py','reports/slice-22g/final-audit.py']:ast.parse((ROOT/p).read_text(),filename=p)
72 doc=ROOT/'docs/slice-22g-xtm-occurrence-core.md'
73 for link in re.findall(r'\]\(([^)]+)\)',doc.read_text()):
74 if '://' not in link:assert (doc.parent/link).resolve().exists() or (doc.parent/link).resolve()==D/'final-audit.json',link
75 manifest={str(p.relative_to(ROOT)):sha(p) for p in sorted(D.rglob('*')) if p.is_file() and p.name!='final-audit.json'}
76 manifest.update({p:sha(ROOT/p) for p in ALLOWED|NEW})
77 audit={'kind':'final-slice-22g-saved-execution-and-scope-audit-not-new-test-run','baseCommit':BASE,'observedHEAD':git('rev-parse','HEAD'),'branch':git('branch','--show-current'),
78 'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'initialIndexAndWorktreeClean':True,'allowedTrackedChanges':sorted(ALLOWED),
79 'productionChanges':['src/xtm/reader.lisp'],'newOriginalPASS':11,'retainedOriginalPASS':72,'standaloneXTMChecks':2869,'newXTMChecks':165,
80 'nativePositiveControls':19,'negativeControls':27,'nativeLateMergeProbes':9,'executorControls':113,'originalDiagnosesChanged':17,'originalDiagnosesUnchanged':107,
81 'retainedEquivalenceChecks':487,'retainedEquivalenceReaderPairs':27,'statuses':e['statusesAfter'],'pinnedCorpusFilesVerified':2006,'historicalReportFilesVerified':6178,
82 'protectedBaseFilesVerified':len(protected),'originalSourcesBaselinesHistoricalReportsAndCorpusLockUnchanged':True,'TMDMCTMCXTMXMLUnchanged':True,
83 'hyperdocOrOtherWorkspaceChanges':False,'newDependencyDownloads':0,'persistentImagesTouched':False,'pushOrDeployment':False,
84 'artifactSHA256':manifest,'selfExcludedFromManifest':True,'acceptancePassed':True}
85 (D/'final-audit.json').write_text(json.dumps(audit,indent=2,ensure_ascii=False)+'\n')
86 print(json.dumps({k:v for k,v in audit.items() if k not in ['artifactSHA256','allowedTrackedChanges']}))