]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22f.py
perf(tmdm): index topic normalization merge candidates
[topic-maps.git] / scripts / validate-slice-22f.py
1 #!/usr/bin/env python3
2 """Validate direct CTM/XTM semantic pairs via existing ASDF/offline executors.
3 --audit-only verifies saved executions; it starts no test processes.
4 """
5 import argparse
6 import datetime
7 import hashlib
8 import importlib.util
9 import json
10 from pathlib import Path
11 import re
12 import sys
13 import subprocess
14 sys.dont_write_bytecode=True
15 ROOT=Path(__file__).resolve().parents[1]
16 D=ROOT/'reports/slice-22f';PRIOR=ROOT/'reports/slice-22e/after'
17 BASE='aeebbc7c6fdbf26642db0b7b5a358a3d04d62cca'
18 ALLOWED={'topic-maps.asd'}
19 SOURCES={'topic-maps.asd','tests/ctm-xtm-equivalence.lisp','reports/slice-22f/fresh-equivalence.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
20 def read(p):return json.loads(p.read_text())
21 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
22 def save(p,x):p.write_text(json.dumps(x,indent=2,ensure_ascii=False)+'\n')
23 def git(*a,cwd=ROOT):return subprocess.check_output(['git',*a],cwd=cwd,text=True).strip()
24 def legacy():
25 spec=importlib.util.spec_from_file_location('slice22d',ROOT/'scripts/validate-slice-22d.py');v=importlib.util.module_from_spec(spec);spec.loader.exec_module(v);return v
26
27 def audit(out,initial):
28 state=read(initial);assert state['baseCommit']==BASE and state['initialIndexAndWorktreeClean']
29 assert set(git('diff','--name-only',BASE).splitlines())&set(state['trackedFileSHA256'])<=ALLOWED
30 protected={p:h for p,h in state['trackedFileSHA256'].items() if p not in ALLOWED}
31 assert protected=={p:sha(ROOT/p) for p in protected}
32 run=read(out/'runs.json');assert len(run['runs'])==119
33 assert all(x['exitCode']==x['expectedExitCode'] and sha(out/(x['label']+'.log'))==x['logSHA256'] for x in run['runs'])
34 assert run['executionSourcesSHA256']=={p:sha(ROOT/p) for p in run['executionSourcesSHA256']}
35 assert run['rawReportSHA256']=={p:sha(out/p) for p in run['rawReportSHA256']}
36 # Existing standalone XTM behavior and registry are completely unchanged.
37 assert (out/'xtm.json').read_bytes()==(PRIOR/'xtm.json').read_bytes()
38 assert read(out/'xtm.json')['statuses']=={'pass':72,'fail':0,'unsupported':0,'not-run':0}
39 assert (out/'after-diagnostics.json').read_bytes()==(PRIOR/'after-diagnostics.json').read_bytes()
40 assert len(read(out/'after-diagnostics.json'))==124
41 faults=[]
42 for x in run['runs']:
43 if '-control-' not in x['label']:continue
44 p=x['label']+'.json';assert (out/p).read_bytes()==(PRIOR/p).read_bytes(),p
45 faults.append({'label':x['label'],'exitCode':x['exitCode'],'expectedASDFFailureObserved':True,'behavioralReportByteIdentical':True,'sha256':sha(out/p)})
46 assert len(faults)==113
47 save(out/'failure-controls.json',{'retained':113,'new':0,'controls':faults,'allFaultReportsByteIdentical':True})
48 unchanged=[]
49 for old in [*sorted((PRIOR/'retained').glob('*.json')),*[PRIOR/f'ctm.{s}.json' for s in ['validation','cxtm','encoding']]]:
50 rel=old.relative_to(PRIOR);assert old.read_bytes()==(out/rel).read_bytes(),str(rel)
51 unchanged.append({'report':str(rel),'sha256':sha(old),'byteIdentical':True})
52 assert len(unchanged)==48
53 save(out/'retained-reports.json',unchanged)
54 prior=read(PRIOR/'execution.json');log=(out/'regression.log').read_text()
55 for marker,count in dict(prior['retainedChecks'],**prior['stageChecks']).items():
56 assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',log).group(1))==count
57 assert 'Ran 2 tests' in log and 'NFC-ORACLE-PASS: 16294' in log
58 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
59 eq=read(out/'retained/xtm-mergemap.equivalence.json');assert eq['condition'] is None and eq['originalCorpusRegistrationsAdded']==0
60 labels=re.findall(r'^XTM-MERGEMAP-CHECK-PASS (equivalence:.+)$',log,re.M)
61 assert labels==eq['checks'] and len(labels)==len(set(labels))==487
62 assert 'CTM-XTM-EQUIVALENCE-PASS: 487 checks; 27 actual reader pairs; 3 contrasts; 3 explicit capability gaps.' in log
63 pairs=[p for p in eq['pairs'] if p['kind']=='direct-reader-equivalence-pair']
64 contrasts=[p for p in eq['pairs'] if p['kind']=='intentional-semantic-difference']
65 gaps=[p for p in eq['pairs'] if p['kind']=='reader-capability-gap-not-equivalence-PASS']
66 assert len(pairs)==27 and len(contrasts)==3 and len(gaps)==3
67 allids=[p['id'] for p in eq['pairs']];assert len(allids)==len(set(allids))==33
68 (out/'actual').mkdir(exist_ok=True);summary=[]
69 for p in pairs:
70 assert p['nativePropertiesEqual'] and p['ctmCanonicalEQ'] and p['xtmCanonicalEQ'] and not p['crossMapEQRequired']
71 assert p['ctmCondition'] is None and p['xtmCondition'] is None and p['byteExact'] and p['firstDifferentByte'] is None
72 assert p['ctmTrace']['entryPoint']=='topic-maps.ctm:deserialize-string' and p['xtmTrace']['entryPoint']=='topic-maps.xtm:deserialize-octets'
73 assert p['ctmTrace']['sourceKey']!=p['xtmTrace']['sourceKey']
74 assert p['ctmTrace']['documentIRI']==p['xtmTrace']['documentIRI']==p['documentIRI']==p['canonicalBaseIRI']
75 assert p['ctmModel']==p['xtmModel']
76 ac=p['ctmCXTM'].encode();bc=p['xtmCXTM'].encode();assert ac==bc and len(ac)==p['ctmBytes']==p['xtmBytes']
77 name=p['id'];(out/'actual'/(name+'.ctm.cxtm')).write_bytes(ac);(out/'actual'/(name+'.xtm.cxtm')).write_bytes(bc)
78 summary.append({'id':name,'documentIRI':p['documentIRI'],'nativeCounts':p['ctmModel']['counts'],'ctmBytes':len(ac),'xtmBytes':len(bc),
79 'ctmSHA256':hashlib.sha256(ac).hexdigest(),'xtmSHA256':hashlib.sha256(bc).hexdigest(),'sourceCTMSHA256':hashlib.sha256(p['ctmSource'].encode()).hexdigest(),
80 'sourceXTMSHA256':hashlib.sha256(p['xtmSource'].encode()).hexdigest(),'firstDifferentByte':None,'completeByteExact':True})
81 for p in contrasts:
82 a=p['ctmCXTM'].encode();b=p['xtmCXTM'].encode();assert a!=b and not p['nativePropertiesEqual'] and not p['byteExact']
83 assert p['firstDifferentByte']==next(i for i,(x,y) in enumerate(zip(a,b)) if x!=y)
84 for p in gaps:
85 assert p['ctmCondition'] is None and p['ctmModel'] and p['xtmModel'] is None and not p['successfulReaderPair']
86 assert p['xtmCondition']['class']=='XTM-UNSUPPORTED' and p['xtmCondition']['code']=='unsupported-construct'
87 save(out/'equivalence-summary.json',{'checks':487,'actualPairs':27,'intentionalDifferences':3,'readerCapabilityGaps':3,'pairs':summary,'contrasts':contrasts,'gaps':gaps,'originalCorpusRegistrationsAdded':0})
88 save(out/'diagnostic-changes.json',{'originalDiagnoses':124,'changed':0,'byteIdenticalToSlice22E':True,'registeredStatusesChanged':False})
89 lock=read(ROOT/'references/corpus-lock.json')
90 for e in lock['files']:
91 p=ROOT/'vendor/cxtm'/e['path'];assert p.stat().st_size==e['size'] and sha(p)==e['sha256']
92 assert protected=={p:sha(ROOT/p) for p in protected}
93 hd=Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch');assert git('rev-parse','HEAD',cwd=hd)==state['hyperdocHEAD'] and git('status','--porcelain=v1',cwd=hd)==state['hyperdocStatus']
94 save(out/'integrity.json',{'baseCommit':BASE,'protectedBaseFilesSHA256':protected,'executionSourcesSHA256':run['executionSourcesSHA256'],
95 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
96 'productionSourcesUnchanged':True,'originalSourcesBaselinesCorpusLockHistoricalReportsUnchanged':True,'hyperdocUnchanged':True})
97 save(out/'execution.json',{'kind':'slice-22f-direct-CTM-XTM-semantic-equivalence','baseCommit':BASE,'branch':git('branch','--show-current'),
98 'runtime':'SBCL 2.5.4','startedAtUTC':run['startedAtUTC'],'finishedAtUTC':run['finishedAtUTC'],'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),
99 'runs':run['runs'],'collector':'scripts/validate-slice-22d.py::collect (unchanged)','auditToolSHA256':sha(Path(__file__)),
100 'equivalenceChecks':487,'actualReaderPairs':27,'intentionalSemanticDifferences':3,'documentedReaderCapabilityGaps':3,
101 'retainedStandaloneXTMChecks':2704,'retainedLispChecks':12100,'retainedChecks':prior['retainedChecks'],'stageChecks':prior['stageChecks'],
102 'xmlQualificationChecks':146,'faultControls':113,'newFaultControls':0,'originalDiagnoses':124,'diagnosticChanges':0,
103 'statusesBefore':prior['statusesAfter'],'statusesAfter':prior['statusesAfter'],'originalCorpusRegistrationsAdded':0,
104 'pythonUnitTests':2,'nfcOracleComparisons':16294,'retainedReportsByteIdentical':48,
105 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
106 'knownCTMFailuresUnchanged':prior['knownCTMFailuresUnchanged'],'productionChanges':[],'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
107 print(json.dumps({'acceptancePassed':True,'readerPairs':27,'equivalenceChecks':487,'faultControls':113,'statuses':prior['statusesAfter']}))
108
109 def main():
110 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--initial',type=Path,default=D/'initial-state.json');ap.add_argument('--audit-only',action='store_true');a=ap.parse_args()
111 out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
112 if not a.audit_only:
113 v=legacy();v.EXECUTION_FILES=SOURCES;v.collect(out,ROOT/'reports/slice-22d/baseline')
114 audit(out,a.initial)
115 if __name__=='__main__':main()