]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22b.py
Implement XTM association reification with native conformance evidence
[topic-maps.git] / scripts / validate-slice-22b.py
1 #!/usr/bin/env python3
2 """Validate one XTM original registration through the existing offline harness.
3 Outputs are outside the repository. No production edits or new test architecture.
4 The pre-change execution is retained in reports/slice-22b/baseline; --baseline may
5 select another verified fresh baseline. The default reruns the complete current
6 regression and all 108 executor controls against the immutable Slice-22A evidence.
7 """
8 import argparse
9 import datetime
10 import hashlib
11 import importlib.util
12 import json
13 import os
14 from pathlib import Path
15 import re
16 import shutil
17 import subprocess
18 import sys
19 import tempfile
20
21 sys.dont_write_bytecode = True
22 ROOT = Path(__file__).resolve().parents[1]
23 BASE = 'b12ece02faa039e816fca3750acd9eceb6be1513'
24 PRIOR = ROOT / 'reports/slice-22a/baseline'
25 TOOLS = ROOT / 'reports/slice-22b'
26 TARGET = 'xtm2/in/variant-duplicate-reifier2.xtm'
27 RUNTIME = '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
28 ALLOWED = {'tests/xtm-identity.lisp', 'tests/xtm-variant-reification.lisp', 'topic-maps.asd'}
29 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "'+str(ROOT)+'"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
30 CTM = {'pass':262,'fail':4,'unsupported':0,'not-run':7}
31
32
33 def read(p): return json.loads(p.read_text())
34 def save(p,v): p.write_text(json.dumps(v,indent=2,ensure_ascii=False)+'\n')
35 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
36 def git(*args): return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
37 def prepare_pinned(target):
38 spec=importlib.util.spec_from_file_location('slice21o',ROOT/'scripts/validate-slice-21o.py')
39 module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
40 return module.prepare_pinned(target)
41
42
43 def verify_before(path):
44 ex=read(path/'execution.json')
45 assert ex['acceptancePassed'] and ex['xtmStatusesAfter']=={'pass':70,'fail':0,'unsupported':0,'not-run':54}
46 assert ex['ctmStatusesAfter']==CTM and len(ex['runs'])==109
47 assert all(r['exitCode']==r['expectedExitCode'] for r in ex['runs'])
48 rows=[]
49 for item in read(ROOT/'reports/slice-22a/baseline-comparison.json'):
50 name=item['freshReport'].removeprefix('baseline/')
51 a,b=PRIOR/name,path/name
52 assert a.read_bytes()==b.read_bytes(),name
53 rows.append({'slice22AReport':str(a.relative_to(ROOT)),'freshBeforeReport':'baseline/'+name,
54 'sha256':sha(b),'byteIdentical':True})
55 assert len(rows)==154
56 return rows
57
58
59 def main():
60 ap=argparse.ArgumentParser(description=__doc__)
61 ap.add_argument('--sbcl',default=RUNTIME);ap.add_argument('--output',type=Path)
62 ap.add_argument('--baseline',type=Path,default=TOOLS/'baseline');ap.add_argument('--prepare',type=Path)
63 args=ap.parse_args()
64 if args.prepare: print(json.dumps(prepare_pinned(args.prepare)));return
65 assert args.output,'--output outside the repository is required'
66 output=args.output.resolve();assert not output.is_relative_to(ROOT)
67 output.mkdir(parents=True,exist_ok=True);assert not (output/'execution.json').exists(),'Choose an unused output directory.'
68 runtime=shutil.which(args.sbcl);assert runtime and subprocess.check_output([runtime,'--version'],text=True).strip()=='SBCL 2.5.4'
69 subprocess.run(['git','merge-base','--is-ancestor',BASE,'HEAD'],cwd=ROOT,check=True)
70 tracked=git('ls-tree','-r','--name-only',BASE).splitlines()
71 changed=set(git('diff','--name-only',BASE).splitlines()) & set(tracked);assert changed<=ALLOWED,changed
72 before_hashes={p:sha(ROOT/p) for p in tracked};protected={p:v for p,v in before_hashes.items() if p not in ALLOWED}
73 sources={p:sha(ROOT/p) for p in [*sorted(ALLOWED),'scripts/validate-slice-22b.py','reports/slice-22b/failure-controls.lisp']}
74 save(output/'baseline-comparison.json',verify_before(args.baseline.resolve()))
75 previous=read(PRIOR/'xtm.json');assert len(previous['cases'])==70 and TARGET not in {c['id'] for c in previous['cases']}
76 started=datetime.datetime.now(datetime.timezone.utc).isoformat();runs=[]
77 env={k:v for k,v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
78 env.update(PYTHONDONTWRITEBYTECODE='1',GIT_OPTIONAL_LOCKS='0',SBCL=runtime)
79 def run(label,command,env,exit_code=0):
80 command=['/usr/bin/sandbox-exec','-p',POLICY]+command
81 t=datetime.datetime.now(datetime.timezone.utc).isoformat()
82 result=subprocess.run(command,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,timeout=300)
83 (output/(label+'.log')).write_bytes(result.stdout)
84 runs.append({'label':label,'command':command,'startedAtUTC':t,'finishedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),
85 'exitCode':result.returncode,'expectedExitCode':exit_code,'network':'denied','repositoryWrites':'denied','hyperdocWrites':'denied','freshProcess':True})
86 assert result.returncode==exit_code,(label,result.stdout.decode()[-3500:])
87 return result.stdout.decode()
88 retained=output/'retained';retained.mkdir()
89 with tempfile.TemporaryDirectory(prefix='topic-maps-22b-sources-') as tmp:
90 provenance=json.loads(run('source-preparation',['python3',str(Path(__file__)), '--prepare',tmp+'/sources'],env));save(output/'source-preparation.json',provenance)
91 env.update(CL_SOURCE_REGISTRY=provenance['sourceRegistry'],TOPIC_MAPS_REPORT=str(retained/'core.json'),
92 TOPIC_MAPS_XTM_REPORT=str(output/'xtm.json'),TOPIC_MAPS_XTM_MERGEMAP_REPORT=str(retained/'xtm-mergemap'),
93 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT=str(retained/'slice20a'),TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT=str(retained/'slice20b'),
94 TOPIC_MAPS_PROLOG_REPORT=str(retained/'slice20c'),TOPIC_MAPS_EMPTY2_REPORT=str(retained/'slice20e'),
95 TOPIC_MAPS_CTM_INTEGER_REPORT=str(retained/'slice20f'),TOPIC_MAPS_TRIPLE_REPORT=str(retained/'slice20g'),TOPIC_MAPS_ENCODING_REPORT=str(output/'ctm'))
96 prefix=[runtime,'--noinform','--no-sysinit','--no-userinit','--script']
97 xml=run('xml',prefix+['tests/cxml-probe.lisp'],env)
98 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
99 assert len(set(re.findall(r'^CXML-CHECK-PASS (.+)$',xml,re.M)))==146
100 preflight=run('variant-reifier-api-preflight',prefix+['reports/slice-21o/api-preflight.lisp'],env)
101 assert 'VARIANT-REIFIER-API-PREFLIGHT-PASS:' in preflight
102 xtm_log=run('xtm',prefix+['scripts/fresh-xtm.lisp'],env);xtm=read(output/'xtm.json')
103 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',xtm_log,re.M);assert labels==xtm['checks'] and len(labels)==len(set(labels))
104 assert [l for l in labels if l in set(previous['checks'])]==previous['checks']
105 assert len(set(labels)-set(previous['checks']))==12
106 assert f'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 63 original CXTM PASS; 8 negative originals PASS' in xtm_log
107 assert xtm['statuses']=={'pass':71,'fail':0,'unsupported':0,'not-run':0}
108 assert xtm['otherCorpusInputs']=={'positiveUnexecuted':50,'negativeUnregisteredUnexecuted':3}
109 assert xtm['cases'][:70]==previous['cases'] and len(xtm['cases'])==len({c['id'] for c in xtm['cases']})==71
110 assert xtm['cases'][-1]['id']==TARGET and xtm['cases'][-1]['status']=='pass' and xtm['cases'][-1]['actualOutcome']=='cxtm-match'
111 allowed_fields={'scope','statuses','otherCorpusInputs','checks','cases','variantReifierOriginals'}
112 for k,v in previous.items():
113 if k not in allowed_fields:assert xtm[k]==v,k
114 assert xtm['scope']==previous['scope'].replace('62 positive','63 positive')
115 assert len(xtm['variantReifierOriginals'])==3 and xtm['variantReifierOriginals'][:2]==previous['variantReifierOriginals']
116 regression=run('regression',[str(ROOT/'scripts/test.sh')],env)
117 # This unchanged reader-only diagnostic driver performs no registration.
118 # Its historical extra flag is process-local and is not a registry report.
119 run('diagnostics',prefix+['reports/slice-21o/diagnostics.lisp'],dict(env,TOPIC_MAPS_21O_ROOT=str(ROOT),
120 TOPIC_MAPS_21O_DIAGNOSTICS=str(output/'after-diagnostics.json'),TOPIC_MAPS_21O_EXTRA=tmp+'/reader-only-extra.json'))
121 assert read(output/'after-diagnostics.json')==read(PRIOR/'after-diagnostics.json')
122 assert len(read(output/'after-diagnostics.json'))==124
123 fault_controls=[];retained_faults=[]
124 old_fault_runs=[r for r in read(args.baseline/'execution.json')['runs'] if '-control-' in r['label']]
125 assert len(old_fault_runs)==103
126 for entry in old_fault_runs:
127 label=entry['label'];family,mode=label.split('-control-')
128 run(label,prefix+['reports/slice-21o/failure-controls.lisp'],dict(env,TOPIC_MAPS_XTM_REPORT=str(output/(label+'.json')),
129 TOPIC_MAPS_21O_FAMILY=family,TOPIC_MAPS_21O_FAULT=mode),1)
130 old,new=read(args.baseline/(label+'.json')),read(output/(label+'.json'))
131 assert new['scope']==old['scope'].replace('62 positive','63 positive')
132 assert {k:v for k,v in old.items() if k!='scope'}=={k:v for k,v in new.items() if k!='scope'},label
133 retained_faults.append({'label':label,'allBehavioralFieldsUnchanged':True,'onlyMetadataChange':'/scope: 62 -> 63 positive capacity',
134 'beforeSHA256':sha(args.baseline/(label+'.json')),'afterSHA256':sha(output/(label+'.json'))})
135 fault_controls.append({'label':label,'retained':True,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
136 modes={'mismatch':('cxtm-mismatch',None,None),'xml':('execution-error','XTM-XML-ERROR','xml-not-well-formed'),
137 'xtm':('execution-error','XTM-REJECTION','invalid-id'),'unsupported':('unsupported','XTM-UNSUPPORTED','resource-data-datatype'),
138 'tmdm':('execution-error','REIFICATION-ERROR','reifier-conflict')}
139 for mode,(outcome,cls,code) in modes.items():
140 label='variant-reifier-followup-control-'+mode
141 run(label,prefix+[str(TOOLS/'failure-controls.lisp')],dict(env,TOPIC_MAPS_XTM_REPORT=str(output/(label+'.json')),TOPIC_MAPS_22B_FAULT=mode),1)
142 result=read(output/(label+'.json'));assert result['cases'][:70]==xtm['cases'][:70] and len(result['cases'])==71
143 failed=result['cases'][-1];assert failed['id']==TARGET and failed['actualOutcome']==outcome and failed['status']!='pass'
144 assert result['statuses']['pass']==70
145 actual=next(c for c in result['variantReifierOriginals'] if c['id']==TARGET);c=actual['condition']
146 if cls:assert c['class']==cls and c['code']==code,(mode,c)
147 else:assert c is None and actual['firstDifferentByte']==0
148 fault_controls.append({'label':label,'retained':False,'target':TARGET,'mode':mode,'actualOutcome':outcome,'condition':c,
149 'caseStatus':failed['status'],'retainedOriginalPASS':70,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
150 unchanged=[]
151 for old in sorted((PRIOR/'retained').glob('*.json')):
152 assert old.read_bytes()==(retained/old.name).read_bytes(),old.name
153 unchanged.append({'previousReport':str(old.relative_to(ROOT)),'freshReport':'retained/'+old.name,'sha256':sha(old),'byteIdentical':True})
154 for suffix in ['validation','cxtm','encoding']:
155 name=f'ctm.{suffix}.json';assert (PRIOR/name).read_bytes()==(output/name).read_bytes()
156 unchanged.append({'previousReport':str((PRIOR/name).relative_to(ROOT)),'freshReport':name,'sha256':sha(output/name),'byteIdentical':True})
157 totals=read(PRIOR/'execution.json')['retainedChecks'];stages=read(PRIOR/'execution.json')['stageChecks']
158 for marker,count in dict(totals,**stages).items():assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',regression).group(1))==count
159 assert sum(totals.values())==12100 and 'Ran 2 tests' in regression and 'NFC-ORACLE-PASS: 16294' in regression
160 assert read(output/'ctm.validation.json')['statuses']==CTM
161 new=xtm['variantReifierOriginals'][-1];source=ROOT/'vendor/cxtm'/TARGET;baseline=ROOT/'vendor/cxtm/xtm2/baseline/variant-duplicate-reifier2.xtm.cxtm'
162 actual=new['actualCXTM'].encode('utf-8');raw=source.read_bytes();expected=baseline.read_bytes()
163 assert raw==bytes.fromhex(new['sourceHex'])==new['sourceText'].encode('utf-8') and len(raw)==new['sourceBytes']==543
164 assert sha(source)=='87b740c6ec7d11b485b223cb5c28807bce9259d04c2d586ea557e6cd10c3faf4'
165 assert actual==expected==new['expectedCXTM'].encode('utf-8') and len(actual)==new['actualBytes']==new['expectedBytes']==702
166 assert sha(baseline)==hashlib.sha256(actual).hexdigest()=='014773fca95a2b7bb70038f8e118d402d311a2804f54f105ff05ee53c7eb0a60'
167 assert new['firstDifferentByte'] is None and new['condition'] is None and new['status']=='pass'
168 assert new['documentIRI']==new['model']['identity']['identity']['nameModel']['identity']['topics'][0]['itemIdentifiers'][0].removesuffix('#topic')
169 assert new['sourceKey']==TARGET and new['primaryQualifiedParserCalls']==1
170 assert new['model']==read(PRIOR/'further-variant-reification.json')['model']
171 assert new['nativeHandleObservation']=={'sourceNameCalls':2,'sameRawNameHandleEQ':True,'preNormalizationPairs':1,'distinctRetainedVariantHandles':2,
172 'retainedNameCanonicalEQ':True,'retainedVariantCanonicalEQ':True,'variantReplacementObserved':True,'canonicalParentScopeReifierInverseEQ':True}
173 new=dict(new,sourceSHA256=sha(source),baselineSHA256=sha(baseline),actualSHA256=hashlib.sha256(actual).hexdigest(),byteExact=True,executedInASDFTestOp=True)
174 (output/'actual').mkdir();(output/'actual/variant-duplicate-reifier2.xtm.cxtm').write_bytes(actual)
175 save(output/'original-contract.json',new)
176 save(output/'diagnostic-changes.json',{'readerObservations':124,'changed':0,'byteIdenticalToSlice22A':(output/'after-diagnostics.json').read_bytes()==(PRIOR/'after-diagnostics.json').read_bytes(),
177 'beforeSHA256':sha(PRIOR/'after-diagnostics.json'),'afterSHA256':sha(output/'after-diagnostics.json'),'changes':[]})
178 save(output/'retained-reports.json',unchanged);save(output/'retained-fault-reports.json',retained_faults)
179 save(output/'failure-controls.json',{'kind':'executor-fault-controls-not-corpus-results','controls':fault_controls,'retained':103,'new':5,'allExpectedFailureExitsObserved':True})
180 matrix=read(ROOT/'reports/slice-22a/matrix.json');rows=[]
181 for c in matrix['cases']:
182 after='pass' if c['id']==TARGET else c['registeredStatus']
183 rows.append({'id':c['id'],'before':c['registeredStatus'],'after':after,'changed':after!=c['registeredStatus'],
184 'readerConditionUnchanged':True,'newCapability':False,'newNormativeDecision':False})
185 assert sum(c['changed'] for c in rows)==1
186 save(output/'closure-delta.json',{'kind':'slice-22b-delta-to-immutable-slice-22a-matrix','sourceMatrix':'reports/slice-22a/matrix.json',
187 'sourceMatrixSHA256':sha(ROOT/'reports/slice-22a/matrix.json'),'changedCase':TARGET,'newPASS':1,'otherOpenCasesUnchanged':64,'remainingOpenCases':64,
188 'newCapability':False,'newNormativeDecision':False,'cases':rows})
189 lock=read(ROOT/'references/corpus-lock.json')
190 for entry in lock['files']:
191 p=ROOT/'vendor/cxtm'/entry['path'];assert p.stat().st_size==entry['size'] and sha(p)==entry['sha256']
192 for group in [protected,sources]:assert group=={p:sha(ROOT/p) for p in group}
193 assert set(git('diff','--name-only',BASE).splitlines()) & set(tracked)<=ALLOWED
194 save(output/'integrity.json',{'baseCommit':BASE,'protectedBaseFilesSHA256':protected,'executionSourcesSHA256':sources,
195 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),'pinnedCorpusFilesVerified':len(lock['files']),
196 'productionSourcesTestsExceptThreeRegistrationFilesAndHistoricalEvidenceUnchanged':True})
197 before={'XTM':{'pass':70,'fail':0,'not-run':54},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':332,'fail':4,'not-run':61}}
198 after={'XTM':{'pass':71,'fail':0,'not-run':53},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':333,'fail':4,'not-run':60}}
199 save(output/'execution.json',{'kind':'slice-22b-one-positive-original-registration','baseCommit':BASE,'branch':git('branch','--show-current'),
200 'runtime':'SBCL 2.5.4','startedAtUTC':started,'finishedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'runs':runs,
201 'baselineBehavioralReportsReproduced':154,'retainedLispChecks':12100,'retainedChecks':totals,'stageChecks':stages,'xmlQualificationChecks':146,
202 'standaloneXTMChecks':len(labels),'retainedXTMChecks':len(previous['checks']),'newXTMChecks':12,'retainedOriginals':70,'retainedNegativeOriginals':8,
203 'newPositiveOriginals':1,'faultControls':108,'newFaultControls':5,'diagnosticOriginalObservations':124,'diagnosticChanges':0,
204 'statusesBefore':before,'statusesAfter':after,'pythonUnitTests':2,'nfcOracleComparisons':16294,'retainedNonXTMReportsByteIdentical':len(unchanged),
205 'knownCTMFailuresUnchanged':[c['id'] for c in read(output/'ctm.validation.json')['cases'] if c['status']=='fail'],
206 'pinnedCorpusFilesVerified':len(lock['files']),'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
207 'productionChanges':[],'historicalReportsUnchanged':True,'originalSourcesAndBaselinesUnchanged':True,'hyperdocLoadedOrChanged':False,
208 'newDependencyDownloads':0,'persistentLispImagesTouched':False,'acceptancePassed':True})
209 print(json.dumps({'acceptancePassed':True,'statuses':after,'xtmChecks':len(labels),'faultControls':108,'output':str(output)}))
210
211 if __name__=='__main__':main()