]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22e.py
Support XTM item identity on occurrences associations and roles
[topic-maps.git] / scripts / validate-slice-22e.py
1 #!/usr/bin/env python3
2 """Run/audit Slice 22E through the unchanged 22D offline executor and ASDF tests.
3 --output must be outside the repository. --audit-only verifies saved executions,
4 without pretending they are fresh runs. Local pinned dependencies only.
5 """
6 import argparse
7 import datetime
8 import hashlib
9 import importlib.util
10 import json
11 from pathlib import Path
12 import re
13 import subprocess
14 import sys
15 sys.dont_write_bytecode=True
16 ROOT=Path(__file__).resolve().parents[1]
17 TOOLS=ROOT/'reports/slice-22e'
18 BASE='791969ca59922d61e967198eaee4c1fe172b5a0d'
19 PRIOR=ROOT/'reports/slice-22d/after'
20 ALLOWED={'src/xtm/package.lisp','src/xtm/reader.lisp','tests/xtm-identity.lisp','tests/xtm-map-reification.lisp','tests/ctm-xtm-mergemap.lisp','topic-maps.asd'}
21 NEW={'tests/xtm-mergemap.lisp','scripts/validate-slice-22e.py'}
22 TARGET='xtm2/in/mergemap-itemid.xtm'
23
24 def read(p):return json.loads(p.read_text())
25 def save(p,v):p.write_text(json.dumps(v,indent=2,ensure_ascii=False)+'\n')
26 def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
27 def git(*a,cwd=ROOT):return subprocess.check_output(['git',*a],cwd=cwd,text=True).strip()
28 def now():return datetime.datetime.now(datetime.timezone.utc).isoformat()
29 def previous_validator():
30 spec=importlib.util.spec_from_file_location('slice22d',ROOT/'scripts/validate-slice-22d.py')
31 v=importlib.util.module_from_spec(spec);spec.loader.exec_module(v)
32 return v
33
34 def audit(out,baseline,initial):
35 v=previous_validator();start=read(initial)
36 assert start['baseCommit']==BASE and start['indexClean'] and start['worktreeClean']
37 assert set(git('diff','--name-only',BASE).splitlines())&set(start['trackedFileSHA256'])<=ALLOWED
38 protected={p:h for p,h in start['trackedFileSHA256'].items() if p not in ALLOWED}
39 assert protected=={p:sha(ROOT/p) for p in protected}
40 runs=read(out/'runs.json');assert len(runs['runs'])==119
41 assert all(r['exitCode']==r['expectedExitCode'] and sha(out/(r['label']+'.log'))==r['logSHA256'] for r in runs['runs'])
42 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
43 assert runs['rawReportSHA256']=={p:sha(out/p) for p in runs['rawReportSHA256']}
44 before=read(baseline/'xtm.json');xtm=read(out/'xtm.json')
45 assert before==read(PRIOR/'xtm.json') and len(before['checks'])==2522
46 assert read(baseline/'diagnostics.json')==read(PRIOR/'after-diagnostics.json')
47 assert xtm['cases']==before['cases'] and len(xtm['cases'])==72
48 assert xtm['statuses']==before['statuses']=={'pass':72,'fail':0,'unsupported':0,'not-run':0}
49 assert xtm['otherCorpusInputs']==before['otherCorpusInputs']
50 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
51 assert labels==xtm['checks'] and len(labels)==len(set(labels))
52 assert [x for x in labels if x in set(before['checks'])]==before['checks']
53 assert all(x.startswith(('mm:','mm-error:','mm-original:')) for x in set(labels)-set(before['checks']))
54 retargets=[]
55 for k,value in before.items():
56 if k=='checks':continue
57 if k=='mapReifierControls':
58 assert len(value)==len(xtm[k])
59 for i,(a,b) in enumerate(zip(value,xtm[k])):
60 if a!=b:
61 assert a['label']==b['label']=='mergeMap'
62 assert a['condition']['class']==b['condition']['class']=='XTM-UNSUPPORTED'
63 assert a['condition']['code']=='unsupported-construct' and b['condition']['code']=='resource-resolver-required'
64 assert all(d['path'] in {'/condition/code','/condition/detail','/condition/message','/expectedCode'} for d in v.changes(a,b))
65 retargets.append({'field':k,'index':i,'before':a,'after':b})
66 else:assert xtm[k]==value,k
67 save(out/'retargeted-observations.json',retargets)
68 olddiag=read(baseline/'diagnostics.json');newdiag=read(out/'after-diagnostics.json')
69 assert len(olddiag)==len(newdiag)==124
70 changes=[{'id':a['id'],'before':a,'after':b} for a,b in zip(olddiag,newdiag) if a!=b]
71 merge_ids={x['id'] for x in olddiag if x.get('condition') and x['condition'].get('detail')=='XTM mergeMap is not implemented in this subset.'}
72 assert len(merge_ids)==7 and {x['id'] for x in changes}==merge_ids
73 assert all(b'<mergeMap ' in (ROOT/'vendor/cxtm'/id).read_bytes() for id in merge_ids)
74 for x in changes:
75 assert x['before']['condition']['code']=='unsupported-construct'
76 assert x['after']['condition']['class']=='XTM-UNSUPPORTED' and x['after']['condition']['code']=='resource-resolver-required'
77 assert not x['after']['byteExact']
78 save(out/'diagnostic-changes.json',{'kind':'all-124-originals-without-resolver','changed':7,'unchanged':117,'changes':changes,
79 'separateExplicitResolverObservations':'original-observations.json','registeredStatusesChanged':False})
80 faults=[]
81 for r in runs['runs']:
82 label=r['label']
83 if '-control-' not in label:continue
84 a=read(PRIOR/(label+'.json'));b=read(out/(label+'.json'))
85 assert a['cases']==b['cases'] and a['statuses']==b['statuses'],label
86 ca=dict(v.condition_nodes(a));cb=dict(v.condition_nodes(b));assert ca.keys()==cb.keys(),label
87 ds=[]
88 for path,c in ca.items():
89 d=cb[path];assert c['class']==d['class'],(label,path,c,d)
90 if c.get('code')!=d.get('code'):
91 assert path.startswith('/mapReifierControls/') and path.endswith('/condition')
92 assert c.get('code')=='unsupported-construct' and d.get('code')=='resource-resolver-required'
93 if c!=d:ds.append({'path':path,'before':c,'after':d})
94 assert all(x['status']!='pass' for x in b['cases'] if x['status'] in ('fail','unsupported'))
95 faults.append({'label':label,'expectedASDFFailureObserved':True,'caseRecordsAndOutcomesIdentical':True,
96 'previousCorrectPASSRetained':sum(x['status']=='pass' for x in b['cases']),
97 'conditionClassesUnchanged':True,'conditionChanges':ds,'faultCountedAsPass':False})
98 assert len(faults)==113
99 save(out/'failure-controls.json',{'retained':113,'new':0,'controls':faults,'allExpectedFailureExitsObserved':True})
100 unchanged=[];adapter=[]
101 for p in [*sorted((PRIOR/'retained').glob('*.json')),*[PRIOR/f'ctm.{s}.json' for s in ['validation','cxtm','encoding']]]:
102 rel=p.relative_to(PRIOR);q=out/rel
103 if p.read_bytes()!=q.read_bytes():
104 assert str(rel)=='retained/xtm-mergemap.integration.json'
105 adapter=v.changes(read(p),read(q));assert len(adapter)==1,adapter
106 assert adapter[0]['before']=='UNSUPPORTED-CONSTRUCT' and adapter[0]['after']=='RESOURCE-RESOLVER-REQUIRED',adapter
107 else:unchanged.append({'report':str(rel),'sha256':sha(p),'byteIdentical':True})
108 assert len(unchanged)==47
109 save(out/'retained-reports.json',unchanged);save(out/'adapter-retargeting.json',{'differences':adapter,'adapterProductionCodeChanged':False})
110 regression=(out/'regression.log').read_text();prior=read(PRIOR/'execution.json')
111 for marker,count in dict(prior['retainedChecks'],**prior['stageChecks']).items():
112 assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',regression).group(1))==count
113 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
114 xml=(out/'xml.log').read_text();assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
115 assert len(set(re.findall(r'^CXML-CHECK-PASS (.+)$',xml,re.M)))==146
116 assert read(out/'ctm.validation.json')['statuses']==read(PRIOR/'ctm.validation.json')['statuses']
117 originals=xtm['mergemapOriginalObservations'];assert len(originals)==2
118 item=originals[0];assert item['id']==TARGET and item['condition'] is None and item['registeredStatus']=='not-run' and not item['registeredAsPASS']
119 raw=ROOT/'vendor/cxtm'/TARGET;child=ROOT/'vendor/cxtm/xtm2/in/mergemap-itemid.sub';expected=ROOT/'vendor/cxtm/xtm2/baseline/mergemap-itemid.xtm.cxtm'
120 assert raw.read_bytes()==bytes.fromhex(item['sourceHex']) and child.read_bytes()==bytes.fromhex(item['childHex'])
121 actual=item['actualCXTM'].encode();assert expected.read_bytes()==item['expectedCXTM'].encode() and actual!=expected.read_bytes()
122 mismatch=next(i for i,(a,b) in enumerate(zip(actual,expected.read_bytes())) if a!=b)
123 assert mismatch==item['firstDifferentByte'] and item['onlyBaselineRootMapItemIdentifiersDiffer']
124 assert item['targetMapItemIdentifiers']==[] and item['sourceMapItemIdentifiers']==[item['childIRI']+'#the-topic-map']
125 item=dict(item,sourceSHA256=sha(raw),childSHA256=sha(child),baselineSHA256=sha(expected),actualSHA256=hashlib.sha256(actual).hexdigest())
126 save(out/'original-observations.json',[item,originals[1]])
127 (out/'actual').mkdir(exist_ok=True);(out/'actual/mergemap-itemid.xtm.cxtm').write_bytes(actual)
128 assert originals[1]['condition']['code']=='cyclic-mergemap' and originals[1]['registeredStatus']=='not-run'
129 lock=read(ROOT/'references/corpus-lock.json')
130 for e in lock['files']:
131 p=ROOT/'vendor/cxtm'/e['path'];assert p.stat().st_size==e['size'] and sha(p)==e['sha256']
132 assert protected=={p:sha(ROOT/p) for p in protected}
133 hd=Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch');assert git('rev-parse','HEAD',cwd=hd)==start['hyperdocHEAD'] and git('status','--porcelain=v1',cwd=hd)==start['hyperdocStatus']
134 save(out/'integrity.json',{'baseCommit':BASE,'protectedBaseFilesSHA256':protected,'executionSourcesSHA256':runs['executionSourcesSHA256'],
135 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(x.startswith('reports/') for x in protected),
136 'historicalReportsOriginalsBaselinesCorpusLockUnchanged':True,'hyperdocUnchanged':True,'productionChanges':['src/xtm/package.lisp','src/xtm/reader.lisp']})
137 save(out/'execution.json',{'kind':'slice-22e-noncyclic-xtm-mergemap','baseCommit':BASE,'branch':git('branch','--show-current'),
138 'runtime':'SBCL 2.5.4','startedAtUTC':runs['startedAtUTC'],'finishedAtUTC':runs['finishedAtUTC'],'auditedAtUTC':now(),
139 'runs':runs['runs'],'collector':'scripts/validate-slice-22d.py::collect (unchanged)','collectorSHA256':sha(ROOT/'scripts/validate-slice-22d.py'),
140 'auditToolSHA256':sha(Path(__file__)),'standaloneXTMChecks':len(labels),'retainedXTMChecks':2522,'newXTMChecks':len(labels)-2522,
141 'registeredXTMOriginals':72,'newPositiveOriginals':0,'originalObservationsNotRegistrations':2,
142 'retainedLispChecks':12100,'retainedChecks':prior['retainedChecks'],'stageChecks':prior['stageChecks'],'xmlQualificationChecks':146,
143 'faultControls':113,'newFaultControls':0,'originalDiagnoses':124,'diagnosticChangesWithinSlice':7,'diagnosesOutsideScopeUnchanged':117,
144 'statusesBefore':prior['statusesAfter'],'statusesAfter':prior['statusesAfter'],'pythonUnitTests':2,'nfcOracleComparisons':16294,
145 'nonXTMReportsByteIdentical':47,'adapterControlCodeRetargetings':1,'pinnedCorpusFilesVerified':len(lock['files']),
146 'historicalReportFilesVerified':sum(x.startswith('reports/') for x in protected),'knownCTMFailuresUnchanged':prior['knownCTMFailuresUnchanged'],
147 'mapItemMetadataImported':False,'normativeCycleSemanticsImplemented':False,'anyTypeImplemented':False,'anyTypePlannedSlice':'22I',
148 'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
149 print(json.dumps({'acceptancePassed':True,'xtmChecks':len(labels),'statuses':prior['statusesAfter'],'faultControls':113,'actualBytes':len(actual),'baselineBytes':len(expected.read_bytes()),'firstDifferentByte':mismatch}))
150
151 def main():
152 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True)
153 ap.add_argument('--baseline',type=Path,default=TOOLS/'baseline');ap.add_argument('--initial',type=Path,default=TOOLS/'initial-state.json');ap.add_argument('--audit-only',action='store_true');a=ap.parse_args()
154 out=a.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
155 if not a.audit_only:
156 v=previous_validator();v.EXECUTION_FILES=ALLOWED|{'tests/xtm-mergemap.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
157 # Its historical 108 labels plus the five datatype controls are exactly
158 # the 113 accepted controls, using the unchanged 22D executor driver.
159 v.collect(out,ROOT/'reports/slice-22d/baseline')
160 audit(out,a.baseline,a.initial)
161 if __name__=='__main__':main()