]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22h/final-audit.py
feat(tmdm): readers for the TMDM 7.2/7.3 relationships; tests for both default name...
[topic-maps.git] / reports / slice-22h / final-audit.py
1 #!/usr/bin/env python3
2 """Audit saved Slice 22H executions and artifact scope; starts no tests."""
3 from pathlib import Path
4 import ast
5 import datetime
6 import hashlib
7 import json
8 import re
9 import subprocess
10 import sys
11 sys.dont_write_bytecode=True
12 ROOT=Path(__file__).resolve().parents[2];D=ROOT/'reports/slice-22h';A=D/'after';P=ROOT/'reports/slice-22g/after'
13 BASE='7f4aa941293e555ccb9d96f0c05c651b0549a866'
14 ALLOWED={'topic-maps.asd','tests/xtm-identity.lisp','tests/xtm-mergemap.lisp'}
15 NEW={'tests/xtm-mergemap-originals.lisp','scripts/validate-slice-22h.py','docs/slice-22h-xtm-mergemap-originals.md'}
16 def read(p):return json.loads(p.read_text())
17 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
18 def git(*args):return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
19 s=read(D/'initial-state.json');assert s['baseCommit']==BASE and s['initialIndexAndWorktreeClean']
20 assert set(s['trackedFileSHA256'])==set(git('ls-tree','-r','--name-only',BASE).splitlines())
21 protected={p:h for p,h in s['trackedFileSHA256'].items() if p not in ALLOWED}
22 assert protected=={p:sha(ROOT/p) for p in protected}
23 changed=set(git('diff','--name-only',BASE).splitlines());untracked=set(git('ls-files','--others','--exclude-standard').splitlines())
24 assert all(p in ALLOWED|NEW or p.startswith('reports/slice-22h/') for p in changed|untracked)
25 assert not any(p.startswith('src/') for p in changed|untracked)
26 e=read(A/'execution.json');runs=read(A/'runs.json');assert e['acceptancePassed'] and e['auditToolSHA256']==sha(ROOT/'scripts/validate-slice-22h.py')
27 assert len(runs['runs'])==119
28 assert all(x['exitCode']==x['expectedExitCode'] and x['freshProcess'] and x['network']==x['repositoryWrites']==x['hyperdocWrites']=='denied' and sha(A/(x['label']+'.log'))==x['logSHA256'] for x in runs['runs'])
29 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
30 assert runs['rawReportSHA256']=={p:sha(A/p) for p in runs['rawReportSHA256']}
31 assert e['standaloneXTMChecks']==2965 and e['newXTMChecks']==96 and e['retainedStandaloneXTMChecks']==2869 and e['retainedLispChecks']==12100
32 assert e['statusesAfter']=={'XTM':{'pass':88,'fail':0,'not-run':36},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':350,'fail':4,'not-run':43}}
33 old=read(P/'xtm.json');new=read(A/'xtm.json');assert old['cases']==new['cases'][:len(old['cases'])]
34 assert len(new['cases'])==len({c['id'] for c in new['cases']})
35 assert [c for c in new['checks'] if c in set(old['checks'])]==old['checks']
36 assert len(new['checks'])==len(set(new['checks']))==2965
37 assert re.findall(r'^XTM-CHECK-PASS (.+)$',(A/'xtm.log').read_text(),re.M)==new['checks']
38 inputs={x['id']:x for x in read(D/'originals-before.json')};originals=read(A/'original-contracts.json')
39 assert {o['id'] for o in originals}==set(inputs)=={c['id'] for c in new['cases']} - {c['id'] for c in old['cases']}
40 for o in originals:
41 i=inputs[o['id']];source=ROOT/'vendor/cxtm'/o['id'];child=ROOT/'vendor/cxtm'/i['resourceID'];baseline=ROOT/'vendor/cxtm/xtm2/baseline'/(source.name+'.cxtm');actual=A/'actual'/(source.name+'.cxtm')
42 assert source.read_bytes()==bytes.fromhex(o['sourceHex']) and sha(source)==i['sourceSHA256']==o['sourceSHA256']
43 assert child.read_bytes()==bytes.fromhex(o['childHex']) and sha(child)==i['resourceSHA256']==o['childSHA256']
44 assert actual.read_bytes()==baseline.read_bytes()==o['actualCXTM'].encode()==o['expectedCXTM'].encode()
45 assert sha(actual)==sha(baseline)==i['baselineSHA256']==o['baselineSHA256']==o['actualSHA256']
46 assert actual.stat().st_size==o['actualBytes']==o['expectedBytes'] and o['condition'] is None and o['firstDifferentByte'] is None and o['status']=='pass'
47 assert o['importTrace']['sourceMapsPreserved'] and o['targetModel']['canonicalNativeReferencesEQ'] and o['sourceModel']['canonicalNativeReferencesEQ']
48 p=o['qualifiedParserTrace'];assert len(p)==2 and p[0]['originalRootOctetsEQ'] and p[1]['originalChildOctetsEQ']
49 assert p[0]['documentIRI']==o['documentIRI'] and p[1]['documentIRI']==o['childDocumentIRI']
50 assert p[0]['sourceKey']==o['sourceKey'] and p[1]['sourceKey']==o['childSourceKey']
51 assert o['importTrace']['resolverCalls']==[{'resourceIRI':o['childDocumentIRI'],'purpose':'mergemap','fromDocumentIRI':o['documentIRI'],'notation':'http://psi.topicmaps.org/iso13250/xtm'}]
52 ledger=read(A/'all-124-original-statuses.json');assert len(ledger['cases'])==124 and ledger['registeredChanges']==5
53 assert {c['id'] for c in ledger['cases'] if c['changed']}==set(inputs)
54 assert sum(c['after']=='pass' for c in ledger['cases'])==88 and sum(c['after']=='not-run' for c in ledger['cases'])==36
55 assert all(c['before']=='not-run' and c['after']=='pass' and c['newOriginalWithPinnedResolver']['condition'] is None for c in ledger['cases'] if c['changed'])
56 assert (A/'after-diagnostics.json').read_bytes()==(P/'after-diagnostics.json').read_bytes() and len(read(A/'after-diagnostics.json'))==124
57 faults=read(A/'failure-controls.json');assert faults['retained']==113 and faults['new']==0 and len(faults['controls'])==113
58 for f in faults['controls']:
59 assert f['exitCode']==1 and f['expectedASDFFailureObserved'] and f['caseRecordsOutcomesAndChecksIdentical'] and not f['faultCountedAsOriginalPASS']
60 assert all(d['path'] in {'/scope','/registeredMergemapOriginals'} for d in f['reportDifferences'])
61 assert read(A/(f['label']+'.json'))['cases']==read(P/(f['label']+'.json'))['cases']
62 retained=read(A/'retained-reports.json');assert len(retained)==49
63 for f in retained:assert sha(A/f['report'])==f['sha256'] and (A/f['report']).read_bytes()==(P/f['report']).read_bytes()
64 for key in ['occurrenceOriginals','occurrenceControls','mergemapOriginalObservations']:assert old[key]==new[key]
65 notes=read(D/'test-integration-notes.json')
66 for n in notes['changedExcludedCaseGuards']:
67 assert n['baseFileSHA256']==s['trackedFileSHA256'][n['file']] and n['finalFileSHA256']==sha(ROOT/n['file'])
68 assert n['before'] in subprocess.check_output(['git','show',BASE+':'+n['file']],cwd=ROOT,text=True) and n['after'] in (ROOT/n['file']).read_text()
69 lock=read(ROOT/'references/corpus-lock.json');assert len(lock['files'])==2006
70 for f in lock['files']:
71 p=ROOT/'vendor/cxtm'/f['path'];assert p.stat().st_size==f['size'] and sha(p)==f['sha256']
72 assert len(protected)==8777 and sum(p.startswith('reports/') for p in protected)==6490
73 for f in ['scripts/validate-slice-22h.py','reports/slice-22h/final-audit.py']:ast.parse((ROOT/f).read_text(),filename=f)
74 doc=ROOT/'docs/slice-22h-xtm-mergemap-originals.md'
75 for link in re.findall(r'\]\(([^)]+)\)',doc.read_text()):
76 if '://' not in link:assert (doc.parent/link).resolve().exists() or (doc.parent/link).resolve()==D/'final-audit.json',link
77 manifest={str(p.relative_to(ROOT)):sha(p) for p in sorted(D.rglob('*')) if p.is_file() and p.name!='final-audit.json'}
78 manifest.update({p:sha(ROOT/p) for p in ALLOWED|NEW})
79 audit={'kind':'final-slice-22h-saved-execution-and-scope-audit-not-new-test-run','baseCommit':BASE,'observedHEAD':git('rev-parse','HEAD'),'branch':git('branch','--show-current'),
80 'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'initialIndexAndWorktreeClean':True,'allowedTrackedChanges':sorted(ALLOWED),'productionChanges':[],
81 'newOriginalPASS':5,'retainedOriginalPASS':83,'allOriginalStatusesCompared':124,'readerDiagnosticChanges':0,'standaloneXTMChecks':2965,'newXTMChecks':96,
82 'executorControls':113,'retainedEquivalenceChecks':487,'retainedEquivalenceReaderPairs':27,'statuses':e['statusesAfter'],'pinnedCorpusFilesVerified':2006,
83 'historicalReportFilesVerified':6490,'protectedBaseFilesVerified':len(protected),'productionSourcesOriginalsBaselinesHistoricalReportsAndCorpusLockUnchanged':True,
84 'excludedMapItemIdentityAndCycleOriginalsStillNotRun':True,'hyperdocOrOtherWorkspaceChanges':False,'newDependencyDownloads':0,'persistentImagesTouched':False,'pushOrDeployment':False,
85 'artifactSHA256':manifest,'selfExcludedFromManifest':True,'acceptancePassed':True}
86 (D/'final-audit.json').write_text(json.dumps(audit,indent=2,ensure_ascii=False)+'\n')
87 print(json.dumps({k:v for k,v in audit.items() if k not in ['artifactSHA256','allowedTrackedChanges']}))