]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22f/final-audit.py
Document XTM normative coverage and remaining qualification gaps
[topic-maps.git] / reports / slice-22f / final-audit.py
1 #!/usr/bin/env python3
2 """Audit saved Slice 22F executions and final artifact scope; starts no tests."""
3 from pathlib import Path
4 import ast
5 import datetime
6 import hashlib
7 import json
8 import re
9 import subprocess
10 import sys
11 sys.dont_write_bytecode=True
12 ROOT=Path(__file__).resolve().parents[2];D=ROOT/'reports/slice-22f';A=D/'after';P=ROOT/'reports/slice-22e/after'
13 BASE='aeebbc7c6fdbf26642db0b7b5a358a3d04d62cca'
14 ALLOWED={'topic-maps.asd'}
15 NEW={'tests/ctm-xtm-equivalence.lisp','scripts/validate-slice-22f.py','docs/slice-22f-ctm-xtm-equivalence.md'}
16 def read(p):return json.loads(p.read_text())
17 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
18 def git(*args,cwd=ROOT):return subprocess.check_output(['git',*args],cwd=cwd,text=True).strip()
19 s=read(D/'initial-state.json');assert s['baseCommit']==BASE and s['initialIndexAndWorktreeClean']
20 assert set(s['trackedFileSHA256'])==set(git('ls-tree','-r','--name-only',BASE).splitlines())
21 protected={p:h for p,h in s['trackedFileSHA256'].items() if p not in ALLOWED}
22 assert protected=={p:sha(ROOT/p) for p in protected}
23 changed=set(git('diff','--name-only',BASE).splitlines());untracked=set(git('ls-files','--others','--exclude-standard').splitlines())
24 assert all(p in ALLOWED|NEW or p.startswith('reports/slice-22f/') for p in changed|untracked)
25 assert not any(p.startswith('src/') for p in changed|untracked)
26 e=read(A/'execution.json');runs=read(A/'runs.json')
27 assert e['acceptancePassed'] and e['auditToolSHA256']==sha(ROOT/'scripts/validate-slice-22f.py')
28 assert len(runs['runs'])==119
29 assert all(x['exitCode']==x['expectedExitCode'] and x['network']==x['repositoryWrites']==x['hyperdocWrites']=='denied' and x['freshProcess'] and sha(A/(x['label']+'.log'))==x['logSHA256'] for x in runs['runs'])
30 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
31 assert runs['rawReportSHA256']=={p:sha(A/p) for p in runs['rawReportSHA256']}
32 assert e['equivalenceChecks']==487 and e['actualReaderPairs']==27 and e['originalCorpusRegistrationsAdded']==0
33 assert e['retainedStandaloneXTMChecks']==2704 and e['retainedLispChecks']==12100
34 assert e['statusesBefore']==e['statusesAfter']=={'XTM':{'pass':72,'fail':0,'not-run':52},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':334,'fail':4,'not-run':59}}
35 assert (A/'xtm.json').read_bytes()==(P/'xtm.json').read_bytes()
36 assert (A/'after-diagnostics.json').read_bytes()==(P/'after-diagnostics.json').read_bytes()
37 assert len(read(A/'after-diagnostics.json'))==124
38 faults=read(A/'failure-controls.json');assert faults['retained']==113 and faults['new']==0 and faults['allFaultReportsByteIdentical']
39 assert len(faults['controls'])==113
40 for f in faults['controls']:
41 assert f['expectedASDFFailureObserved'] and f['exitCode']==1 and f['behavioralReportByteIdentical']
42 p=f['label']+'.json';assert sha(A/p)==f['sha256'] and (A/p).read_bytes()==(P/p).read_bytes()
43 unchanged=read(A/'retained-reports.json');assert len(unchanged)==48
44 for f in unchanged:assert sha(A/f['report'])==f['sha256'] and (A/f['report']).read_bytes()==(P/f['report']).read_bytes()
45 q=read(A/'retained/xtm-mergemap.equivalence.json');summary=read(A/'equivalence-summary.json')
46 assert q['condition'] is None and q['originalCorpusRegistrationsAdded']==0
47 labels=re.findall(r'^XTM-MERGEMAP-CHECK-PASS (equivalence:.+)$',(A/'regression.log').read_text(),re.M)
48 assert labels==q['checks'] and len(labels)==len(set(labels))==487
49 pairs=[p for p in q['pairs'] if p['kind']=='direct-reader-equivalence-pair']
50 assert len(pairs)==len(summary['pairs'])==27
51 for p,sm in zip(pairs,summary['pairs']):
52 assert p['id']==sm['id'] and p['ctmModel']==p['xtmModel'] and p['nativePropertiesEqual'] and p['ctmCanonicalEQ'] and p['xtmCanonicalEQ'] and not p['crossMapEQRequired']
53 assert p['ctmCondition'] is p['xtmCondition'] is None and p['byteExact'] and p['firstDifferentByte'] is None
54 for syntax in ['ctm','xtm']:
55 data=p[syntax+'CXTM'].encode('utf-8');file=A/'actual'/(p['id']+'.'+syntax+'.cxtm')
56 assert file.read_bytes()==data and len(data)==p[syntax+'Bytes'] and sha(file)==sm[syntax+'SHA256']
57 assert hashlib.sha256(p[syntax+'Source'].encode('utf-8')).hexdigest()==sm['source'+syntax.upper()+'SHA256']
58 assert (A/'actual'/(p['id']+'.ctm.cxtm')).read_bytes()==(A/'actual'/(p['id']+'.xtm.cxtm')).read_bytes()
59 assert p['ctmTrace']['documentIRI']==p['xtmTrace']['documentIRI']==p['documentIRI']==p['canonicalBaseIRI']
60 assert p['ctmTrace']['sourceKey']!=p['xtmTrace']['sourceKey']
61 for p in summary['contrasts']:
62 a=p['ctmCXTM'].encode();b=p['xtmCXTM'].encode()
63 assert p['ctmModel']!=p['xtmModel'] and len(a)==p['ctmBytes'] and len(b)==p['xtmBytes'] and a!=b
64 assert next(i for i,(x,y) in enumerate(zip(a,b)) if x!=y)==p['firstDifferentByte']
65 assert len(summary['contrasts'])==len(summary['gaps'])==3
66 for p in summary['gaps']:
67 assert p['ctmModel'] and p['ctmCondition'] is None and p['xtmModel'] is None and not p['successfulReaderPair']
68 assert p['xtmCondition']['class']=='XTM-UNSUPPORTED' and p['xtmCondition']['code']=='unsupported-construct'
69 imp=pairs[-1];assert imp['id']=='content-import'
70 for lang in ['ctm','xtm']:
71 trace=imp[lang+'Trace'];assert len(trace['resolverCalls'])==1 and len(trace['imports'])==1
72 assert trace['imports'][0]=={'sourceMapItemIdentifiers':['urn:child-map'],'sourceMapReifierPresent':True,'sourceMapPreserved':True}
73 assert all(x['originalResourceOctetsEQ'] for x in trace['qualifiedXMLParserCalls'] if x['documentIRI']==imp['resourceIRI'])
74 lock=read(ROOT/'references/corpus-lock.json');assert len(lock['files'])==2006
75 for f in lock['files']:
76 p=ROOT/'vendor/cxtm'/f['path'];assert p.stat().st_size==f['size'] and sha(p)==f['sha256']
77 assert len(protected)==8109 and sum(p.startswith('reports/') for p in protected)==5826
78 hd=Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch')
79 assert git('rev-parse','HEAD',cwd=hd)==s['hyperdocHEAD'] and git('status','--porcelain=v1',cwd=hd)==s['hyperdocStatus']
80 for p in ['scripts/validate-slice-22f.py','reports/slice-22f/final-audit.py']:ast.parse((ROOT/p).read_text(),filename=p)
81 # Literal whitespace is immutable output evidence, not source formatting.
82 ws=subprocess.run(['git','diff','--cached','--check'],cwd=ROOT,text=True,capture_output=True)
83 expected='reports/slice-22f/after/actual/unicode-whitespace-escaping.ctm.cxtm:7: trailing whitespace.\n+<value> 馬🦊 &amp; &lt; &gt; \nreports/slice-22f/after/actual/unicode-whitespace-escaping.ctm.cxtm:8: space before tab in indent.\n+ \t " \\ </value>\nreports/slice-22f/after/actual/unicode-whitespace-escaping.xtm.cxtm:7: trailing whitespace.\n+<value> 馬🦊 &amp; &lt; &gt; \nreports/slice-22f/after/actual/unicode-whitespace-escaping.xtm.cxtm:8: space before tab in indent.\n+ \t " \\ </value>\n'
84 assert ws.returncode==2 and ws.stdout==expected and not ws.stderr
85 clean=subprocess.run(['git','diff','--cached','--check','--','.',
86 ':(exclude)reports/slice-22f/after/actual/unicode-whitespace-escaping.ctm.cxtm',
87 ':(exclude)reports/slice-22f/after/actual/unicode-whitespace-escaping.xtm.cxtm'],cwd=ROOT,text=True,capture_output=True)
88 assert clean.returncode==0 and not clean.stdout and not clean.stderr
89 whitespace={'kind':'literal-whitespace-evidence-not-formatting-change','actualGitCheckExitCode':ws.returncode,'actualMessages':ws.stdout,
90 'scope':'Exactly four findings in two byte-identical actual CXTM outputs. Spaces and tab belong to the native Unicode Name value.',
91 'actualBytesPreserved':True,'allOtherStagedFilesWhitespaceCheckPassed':True}
92 (D/'whitespace-check.json').write_text(json.dumps(whitespace,indent=2,ensure_ascii=False)+'\n')
93 doc=ROOT/'docs/slice-22f-ctm-xtm-equivalence.md'
94 for link in re.findall(r'\]\(([^)]+)\)',doc.read_text()):
95 if '://' not in link:assert (doc.parent/link).resolve().exists() or (doc.parent/link).resolve()==D/'final-audit.json',link
96 manifest={str(p.relative_to(ROOT)):sha(p) for p in sorted(D.rglob('*')) if p.is_file() and p.name!='final-audit.json'}
97 manifest.update({p:sha(ROOT/p) for p in ALLOWED|NEW})
98 audit={'kind':'final-slice-22f-artifact-and-scope-audit-not-new-test-execution','baseCommit':BASE,'observedHEAD':git('rev-parse','HEAD'),'branch':git('branch','--show-current'),
99 'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'initialIndexAndWorktreeClean':True,'allowedTrackedChanges':sorted(ALLOWED),
100 'productionChanges':[],'newRegisteredPASS':0,'actualReaderPairs':27,'equivalenceChecks':487,'fullByteExactPairs':27,'intraMapCanonicalEQBothReaders':True,
101 'intentionalSemanticDifferences':3,'documentedReaderCapabilityGaps':3,'standaloneXTMChecks':2704,'retainedCoreChecks':12100,
102 'executorControls':113,'unchangedOriginalDiagnoses':124,'statuses':e['statusesAfter'],'pinnedCorpusFilesVerified':2006,'historicalReportFilesVerified':5826,
103 'protectedBaseFilesVerified':len(protected),'historicalReportsOriginalsBaselinesCorpusLockUnchanged':True,'openSlice22CDecisionsUnchanged':True,
104 'hyperdocHEAD':s['hyperdocHEAD'],'hyperdocStatusUnchanged':True,'newDependencyDownloads':0,'persistentImagesTouched':False,'pushOrDeployment':False,
105 'artifactSHA256':manifest,'selfExcludedFromManifest':True,'acceptancePassed':True}
106 (D/'final-audit.json').write_text(json.dumps(audit,indent=2,ensure_ascii=False)+'\n')
107 print(json.dumps({k:v for k,v in audit.items() if k not in ['artifactSHA256','allowedTrackedChanges']}))