]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22d.py
Implement XTM occurrence core and register eleven originals (Slice 22G)
[topic-maps.git] / scripts / validate-slice-22d.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 22D using the existing ASDF, XML and corpus harnesses.
3 Writes only to --output outside the repository. --audit-only rechecks saved runs;
4 it never represents them as new executions. Dependencies come from local pins.
5 """
6 import argparse
7 import datetime
8 import hashlib
9 import importlib.util
10 import json
11 import os
12 from pathlib import Path
13 import re
14 import subprocess
15 import sys
16 import tempfile
17
18 sys.dont_write_bytecode = True
19 ROOT = Path(__file__).resolve().parents[1]
20 BASE = 'b5c5ea5cc84dd31104045d032c25880b8bd62839'
21 TOOLS = ROOT / 'reports/slice-22d'
22 TARGET = 'xtm2/in/variant-datatype-unknown.xtm'
23 RUNTIME = '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
24 POLICY = f'(version 1)(allow default)(deny network*)(deny file-write* (subpath "{ROOT}"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
25 CTM = {'pass':262,'fail':4,'unsupported':0,'not-run':7}
26 ALLOWED = {'src/xtm/reader.lisp','topic-maps.asd'} | {r['file'] for r in json.loads((TOOLS/'retargetings.json').read_text())}
27 EXECUTION_FILES = ALLOWED | {'tests/xtm-resource-datatypes.lisp','reports/slice-22d/failure-controls.lisp'}
28 MODES = {'mismatch':('cxtm-mismatch',None,None), 'xml':('execution-error','XTM-XML-ERROR','xml-not-well-formed'),
29 'xtm':('execution-error','XTM-REJECTION','invalid-id'), 'unsupported':('unsupported','XTM-UNSUPPORTED','resource-data-datatype'),
30 'tmdm':('execution-error','REIFICATION-ERROR','reifier-conflict')}
31
32 def read(p): return json.loads(p.read_text())
33 def save(p,v): p.write_text(json.dumps(v,indent=2,ensure_ascii=False)+'\n')
34 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
35 def git(*args): return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
36 def now(): return datetime.datetime.now(datetime.timezone.utc).isoformat()
37 def prepare_pinned(target):
38 spec=importlib.util.spec_from_file_location('slice22b',ROOT/'scripts/validate-slice-22b.py')
39 module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
40 return module.prepare_pinned(target)
41 def old_label(s):
42 return s.replace('resourceData-anyType','resourceData-integer').replace('anyType-text-still-unsupported','integer-still-unsupported')
43 def changes(a,b,path=''):
44 if a==b:return []
45 if isinstance(a,dict) and isinstance(b,dict):
46 return [d for k in sorted(a.keys()|b.keys()) for d in changes(a.get(k),b.get(k),path+'/'+k)]
47 if isinstance(a,list) and isinstance(b,list) and len(a)==len(b):
48 return [d for i,(x,y) in enumerate(zip(a,b)) for d in changes(x,y,path+'/'+str(i))]
49 return [{'path':path,'before':a,'after':b}]
50
51
52 def condition_nodes(value,path=''):
53 if isinstance(value,dict):
54 for k,v in value.items():
55 if k=='condition' and isinstance(v,dict):yield path+'/'+k,v
56 else:yield from condition_nodes(v,path+'/'+k)
57 elif isinstance(value,list):
58 for i,v in enumerate(value):yield from condition_nodes(v,path+'/'+str(i))
59
60 def verify_baseline(baseline):
61 ex=read(baseline/'execution.json'); assert ex['acceptancePassed']
62 assert ex['statusesAfter']['XTM']=={'pass':71,'fail':0,'not-run':53}
63 assert ex['statusesAfter']['CTM']=={'pass':262,'fail':4,'not-run':7}
64 assert ex['standaloneXTMChecks']==2396 and ex['retainedLispChecks']==12100
65 assert len(ex['runs'])==114 and ex['faultControls']==108
66 assert all(r['exitCode']==r['expectedExitCode'] for r in ex['runs'])
67 # Preserve the provenance of the reused 22B runner; its older base/statusBefore
68 # fields describe that runner, not the actual 22D pre-change checkout.
69 previous=ROOT/'reports/slice-22b/after'
70 rows=[]
71 for p in sorted(baseline.rglob('*.json')):
72 rel=p.relative_to(baseline); old=previous/rel
73 if str(rel) in {'execution.json','integrity.json','source-preparation.json','baseline-comparison.json','retained-fault-reports.json'}: continue
74 if old.exists():
75 assert read(p)==read(old),str(rel)
76 rows.append({'report':str(rel),'sha256':sha(p),'historical22BBehaviorIdentical':True})
77 assert read(baseline/'xtm.json')['cases']==read(previous/'xtm.json')['cases']
78 return rows
79
80 def collect(output,baseline):
81 assert not (output/'runs.json').exists(),'Use an unused output directory, or --audit-only.'
82 assert subprocess.check_output([RUNTIME,'--version'],text=True).strip()=='SBCL 2.5.4'
83 env={k:v for k,v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
84 env.update(PYTHONDONTWRITEBYTECODE='1',GIT_OPTIONAL_LOCKS='0',SBCL=RUNTIME)
85 sources={p:sha(ROOT/p) for p in sorted(EXECUTION_FILES)}
86 started=now();runs=[];retained=output/'retained';retained.mkdir()
87 def run(label,cmd,env,expected=0):
88 command=['/usr/bin/sandbox-exec','-p',POLICY]+cmd;t=now()
89 r=subprocess.run(command,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,timeout=300)
90 log=output/(label+'.log');log.write_bytes(r.stdout)
91 runs.append({'label':label,'command':command,'startedAtUTC':t,'finishedAtUTC':now(),
92 'exitCode':r.returncode,'expectedExitCode':expected,'logSHA256':sha(log),'network':'denied',
93 'repositoryWrites':'denied','hyperdocWrites':'denied','freshProcess':True})
94 save(output/'runs.json',{'startedAtUTC':started,'runs':runs,'executionSourcesSHA256':sources})
95 assert r.returncode==expected,(label,r.stdout.decode()[-2500:])
96 print(f'{label}: expected exit {expected}',flush=True)
97 return r.stdout.decode()
98 with tempfile.TemporaryDirectory(prefix='topic-maps-22d-sources-') as tmp:
99 # The source-preparation subprocess is also network/repository-write denied.
100 text=run('source-preparation',['python3',str(Path(__file__)),'--prepare',tmp+'/sources'],env)
101 provenance=json.loads(text);save(output/'source-preparation.json',provenance)
102 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'],TOPIC_MAPS_REPORT=str(retained/'core.json'),
103 TOPIC_MAPS_XTM_REPORT=str(output/'xtm.json'),TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained/'xtm-mergemap'),
104 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained/'slice20a'),TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained/'slice20b'),
105 TOPIC_MAPS_PROLOG_REPORT=str(retained/'slice20c'),TOPIC_MAPS_EMPTY2_REPORT=str(retained/'slice20e'),
106 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained/'slice20f'),TOPIC_MAPS_TRIPLE_REPORT=str(retained/'slice20g'),TOPIC_MAPS_ENCODING_REPORT=str(output/'ctm'))
107 prefix=[RUNTIME,'--noinform','--no-sysinit','--no-userinit','--script']
108 run('xml',prefix+['tests/cxml-probe.lisp'],env)
109 run('variant-reifier-api-preflight',prefix+['reports/slice-21o/api-preflight.lisp'],env)
110 run('xtm',prefix+['scripts/fresh-xtm.lisp'],env)
111 run('regression',[str(ROOT/'scripts/test.sh')],env)
112 run('diagnostics',prefix+['reports/slice-21o/diagnostics.lisp'],dict(env,TOPIC_MAPS_21O_ROOT=str(ROOT),
113 TOPIC_MAPS_21O_DIAGNOSTICS=str(output/'after-diagnostics.json'),TOPIC_MAPS_21O_EXTRA=tmp+'/reader-only-extra.json'))
114 labels=[r['label'] for r in read(baseline/'execution.json')['runs'] if '-control-' in r['label']]
115 assert len(labels)==108
116 labels += ['datatype-control-'+mode for mode in MODES]
117 for label in labels:
118 family,mode=label.split('-control-')
119 run(label,prefix+[str(TOOLS/'failure-controls.lisp')],dict(env,TOPIC_MAPS_XTM_REPORT=str(output/(label+'.json')),
120 TOPIC_MAPS_21O_FAMILY=family,TOPIC_MAPS_21O_FAULT=mode),1)
121 assert sources=={p:sha(ROOT/p) for p in sources}
122 r=read(output/'runs.json');r['finishedAtUTC']=now()
123 r['rawReportSHA256']={str(p.relative_to(output)):sha(p) for p in sorted(output.rglob('*.json')) if p.name!='runs.json'}
124 save(output/'runs.json',r)
125
126 def audit(output,baseline,initial):
127 start=read(initial);assert start['baseCommit']==BASE and start['indexClean'] and start['worktreeClean']
128 assert set(git('diff','--name-only',BASE).splitlines())&set(start['trackedFileSHA256'])<=ALLOWED
129 protected={p:h for p,h in start['trackedFileSHA256'].items() if p not in ALLOWED}
130 assert protected=={p:sha(ROOT/p) for p in protected}
131 save(output/'baseline-comparison.json',verify_baseline(baseline))
132 runs=read(output/'runs.json');assert len(runs['runs'])==119
133 assert all(r['exitCode']==r['expectedExitCode'] and sha(output/(r['label']+'.log'))==r['logSHA256'] for r in runs['runs'])
134 assert runs['executionSourcesSHA256']=={p:sha(ROOT/p) for p in runs['executionSourcesSHA256']}
135 assert runs['rawReportSHA256']=={p:sha(output/p) for p in runs['rawReportSHA256']}
136 previous=read(baseline/'xtm.json');xtm=read(output/'xtm.json')
137 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(output/'xtm.log').read_text(),re.M)
138 assert labels==xtm['checks'] and len(labels)==len(set(labels))
139 normalized=[old_label(l) for l in labels]
140 assert [l for l in normalized if l in set(previous['checks'])]==previous['checks']
141 assert xtm['statuses']=={'pass':72,'fail':0,'unsupported':0,'not-run':0}
142 assert xtm['otherCorpusInputs']=={'positiveUnexecuted':49,'negativeUnregisteredUnexecuted':3}
143 assert xtm['cases'][:71]==previous['cases'] and len(xtm['cases'])==len({c['id'] for c in xtm['cases']})==72
144 assert xtm['cases'][-1]['id']==TARGET and xtm['cases'][-1]['status']=='pass'
145 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 64 original CXTM PASS; 8 negative originals PASS' in (output/'xtm.log').read_text()
146 # Every prior original model/output stays identical. Control retargetings are
147 # recorded separately, including source and exact condition/position changes.
148 meta={'scope','statuses','otherCorpusInputs','checks','cases'}
149 control_diffs=[]
150 for k,v in previous.items():
151 if k in meta: continue
152 if k.endswith('Controls'):
153 assert len(v)==len(xtm[k]),k
154 for i,(a,b) in enumerate(zip(v,xtm[k])):
155 if a!=b:
156 ds=changes(a,b)
157 assert all(d['path'] in {'/label','/sourceText','/sourceKey','/condition/code','/condition/column','/condition/detail','/condition/message','/condition/sourceKey'} for d in ds),(k,i,ds)
158 if a.get('condition') and b.get('condition'):assert a['condition']['class']==b['condition']['class']
159 control_diffs.append({'field':k,'index':i,'before':a,'after':b,'differences':ds})
160 else:assert xtm[k]==v,k
161 save(output/'retargeted-control-observations.json',control_diffs)
162 before_diags=read(baseline/'after-diagnostics.json');after_diags=read(output/'after-diagnostics.json')
163 assert len(before_diags)==len(after_diags)==124
164 delta=[{'id':a['id'],'before':a,'after':b} for a,b in zip(before_diags,after_diags) if a!=b]
165 assert len(delta)==1 and delta[0]['id']==TARGET
166 assert delta[0]['after']['byteExact'] and delta[0]['after']['condition'] is None
167 save(output/'diagnostic-changes.json',{'readerObservations':124,'changed':1,'unchanged':123,'changes':delta})
168 faults=[]
169 for r in runs['runs']:
170 label=r['label']
171 if '-control-' not in label:continue
172 result=read(output/(label+'.json'));family,mode=label.split('-control-')
173 if family=='datatype':
174 assert result['cases'][:71]==xtm['cases'][:71] and len(result['cases'])==72
175 failed=result['cases'][-1];outcome,cls,code=MODES[mode]
176 assert failed['id']==TARGET and failed['actualOutcome']==outcome and failed['status']!='pass'
177 assert result['statuses']['pass']==71
178 original=result['datatypeOriginals'][0];c=original['condition']
179 if cls: assert c['class']==cls and c['code']==code,(mode,c)
180 else: assert c is None and original['firstDifferentByte']==0
181 faults.append({'label':label,'retained':False,'target':TARGET,'condition':c,'actualOutcome':outcome,
182 'caseStatus':failed['status'],'retainedOriginalPASS':71,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
183 else:
184 old=read(baseline/(label+'.json'))
185 assert len(old['cases'])==len(result['cases']),label
186 for a,b in zip(old['cases'],result['cases']):
187 assert {k:v for k,v in a.items() if k!='diagnostics'}=={k:v for k,v in b.items() if k!='diagnostics'},(label,a,b)
188 if a['status']=='pass': assert a==b
189 # Stop-on-error execution retains the same successful prefix. Later
190 # cases are not executed; the normal run verifies all prior 71 PASS.
191 assert result['statuses']==old['statuses'],label
192 old_conditions=dict(condition_nodes(old));new_conditions=dict(condition_nodes(result))
193 assert old_conditions.keys()==new_conditions.keys(),label
194 condition_diffs=[]
195 for path,c in old_conditions.items():
196 d=new_conditions[path]
197 assert c['class']==d['class'],(label,path,c,d)
198 if c.get('code')!=d.get('code'):
199 # This documented synthetic source was changed from invalid
200 # default-string XML to valid but unimplemented anyType XML.
201 assert path=='/variantControls/52/condition' and c.get('code')=='resource-data-markup' and d.get('code')=='resource-data-datatype'
202 if c!=d:condition_diffs.append({'path':path,'before':c,'after':d})
203 faults.append({'label':label,'retained':True,'sameCaseStatusesAndOutcomes':True,
204 'conditionClassesUnchanged':True,'conditionCodesUnchangedExceptDocumentedEmbeddedXMLRetargeting':True,'conditionChanges':condition_diffs,
205 'priorCorrectPASSRetained':sum(c['status']=='pass' for c in result['cases']),
206 'expectedASDFFailureObserved':True,'faultCountedAsPass':False,
207 'caseDiagnosticChanges':changes(old['cases'],result['cases'])})
208 assert len(faults)==113
209 save(output/'failure-controls.json',{'retained':108,'new':5,'controls':faults,'allExpectedFailureExitsObserved':True})
210 unchanged=[]
211 for old in [*sorted((baseline/'retained').glob('*.json')),*[baseline/f'ctm.{s}.json' for s in ['validation','cxtm','encoding']]]:
212 rel=old.relative_to(baseline)
213 if str(rel)=='retained/xtm-mergemap.integration.json' and old.read_bytes()!=(output/rel).read_bytes():
214 ds=changes(read(old),read(output/rel))
215 save(output/'mergemap-retargeting.json',{'report':str(rel),'differences':ds})
216 assert all('Only textual xsd:string and xsd:anyURI' in str(d['before']) or 'XMLSchema#integer' in str(d['before']) for d in ds),ds
217 continue
218 assert old.read_bytes()==(output/rel).read_bytes(),str(rel)
219 unchanged.append({'report':str(rel),'sha256':sha(old),'byteIdentical':True})
220 assert len(unchanged) in (47,48)
221 save(output/'retained-reports.json',unchanged)
222 regression=(output/'regression.log').read_text();ex=read(baseline/'execution.json')
223 for marker,count in dict(ex['retainedChecks'],**ex['stageChecks']).items():
224 assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',regression).group(1))==count
225 assert 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
226 xml=(output/'xml.log').read_text();assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
227 assert len(set(re.findall(r'^CXML-CHECK-PASS (.+)$',xml,re.M)))==146
228 assert read(output/'ctm.validation.json')['statuses']==CTM
229 original=xtm['datatypeOriginals'][0];source=ROOT/'vendor/cxtm'/TARGET
230 baseline_file=ROOT/'vendor/cxtm/xtm2/baseline/variant-datatype-unknown.xtm.cxtm'
231 actual=original['actualCXTM'].encode();expected=baseline_file.read_bytes()
232 assert actual==expected==original['expectedCXTM'].encode() and len(actual)==original['actualBytes']==original['expectedBytes']==579
233 assert source.read_bytes()==bytes.fromhex(original['sourceHex'])==original['sourceText'].encode()
234 assert original['sourceBytes']==source.stat().st_size and original['condition'] is None and original['firstDifferentByte'] is None
235 assert original['qualifiedParserCalls']==1 and original['status']=='pass' and original['sourceKey']==TARGET
236 save(output/'original-contract.json',dict(original,sourceSHA256=sha(source),baselineSHA256=sha(baseline_file),
237 actualSHA256=hashlib.sha256(actual).hexdigest(),byteExact=True,executedInASDFTestOp=True))
238 (output/'actual').mkdir(exist_ok=True);(output/'actual/variant-datatype-unknown.xtm.cxtm').write_bytes(actual)
239 matrix=read(ROOT/'reports/slice-22a/matrix.json');rows=[]
240 prior_registered={c['id']:c['status'] for c in previous['cases']}
241 for c in matrix['cases']:
242 before=prior_registered.get(c['id'],c['registeredStatus']);after='pass' if c['id']==TARGET else before
243 rows.append({'id':c['id'],'slice22A':c['registeredStatus'],'slice22C':before,'slice22D':after,'changedIn22D':before!=after})
244 assert sum(r['changedIn22D'] for r in rows)==1
245 save(output/'closure-delta.json',{'sourceMatrix':'reports/slice-22a/matrix.json','sourceMatrixSHA256':sha(ROOT/'reports/slice-22a/matrix.json'),
246 'changedCase':TARGET,'newPASS':1,'remainingOpenCases':63,'slice22CDecisionsUnchanged':True,'twelveOpenCDecisionsUnchanged':True,'cases':rows})
247 lock=read(ROOT/'references/corpus-lock.json')
248 for entry in lock['files']:
249 p=ROOT/'vendor/cxtm'/entry['path'];assert p.stat().st_size==entry['size'] and sha(p)==entry['sha256']
250 assert protected=={p:sha(ROOT/p) for p in protected}
251 save(output/'integrity.json',{'baseCommit':BASE,'protectedBaseFilesSHA256':protected,'executionSourcesSHA256':runs['executionSourcesSHA256'],
252 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
253 'productionChanges':['src/xtm/reader.lisp'],'historicalReportsUnchanged':True,'originalSourcesBaselinesAndLockUnchanged':True})
254 after={'XTM':{'pass':72,'fail':0,'not-run':52},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':334,'fail':4,'not-run':59}}
255 save(output/'execution.json',{'kind':'slice-22d-textual-resource-datatypes','baseCommit':BASE,'branch':git('branch','--show-current'),
256 'runtime':'SBCL 2.5.4','auditToolSHA256':sha(Path(__file__)),'startedAtUTC':runs['startedAtUTC'],'finishedAtUTC':runs['finishedAtUTC'],'auditedAtUTC':now(),'runs':runs['runs'],
257 'standaloneXTMChecks':len(labels),'retainedXTMChecks':2396,'newXTMChecks':len(labels)-2396,'retainedLispChecks':12100,
258 'retainedChecks':ex['retainedChecks'],'stageChecks':ex['stageChecks'],'xmlQualificationChecks':146,
259 'retainedOriginals':71,'newPositiveOriginals':1,'faultControls':113,'retainedFaultControls':108,'newFaultControls':5,
260 'diagnosticOriginalObservations':124,'diagnosticChanges':1,'statusesBefore':ex['statusesAfter'],'statusesAfter':after,
261 'pythonUnitTests':2,'nfcOracleComparisons':16294,'retainedNonXTMReportsByteIdentical':len(unchanged),
262 'knownCTMFailuresUnchanged':[c['id'] for c in read(output/'ctm.validation.json')['cases'] if c['status']=='fail'],
263 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
264 'productionChanges':['src/xtm/reader.lisp'],'historicalReportsUnchanged':True,'originalSourcesAndBaselinesUnchanged':True,
265 'newDependencyDownloads':0,'persistentLispImagesTouched':False,'acceptancePassed':True})
266 print(json.dumps({'acceptancePassed':True,'statuses':after,'xtmChecks':len(labels),'faultControls':113,'output':str(output)}))
267
268 def main():
269 ap=argparse.ArgumentParser(description=__doc__)
270 ap.add_argument('--output',type=Path);ap.add_argument('--baseline',type=Path,default=TOOLS/'baseline')
271 ap.add_argument('--initial',type=Path,default=TOOLS/'initial-state.json');ap.add_argument('--prepare',type=Path)
272 ap.add_argument('--audit-only',action='store_true');args=ap.parse_args()
273 if args.prepare:print(json.dumps(prepare_pinned(args.prepare)));return
274 assert args.output;output=args.output.resolve();assert not output.is_relative_to(ROOT)
275 output.mkdir(parents=True,exist_ok=True)
276 verify_baseline(args.baseline)
277 if not args.audit_only:collect(output,args.baseline)
278 audit(output,args.baseline,args.initial)
279
280 if __name__=='__main__':main()