]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22s/reproduce.py
Integrate qualified XML anyType values into the XTM reader
[topic-maps.git] / reports / slice-22s / reproduce.py
1 #!/usr/bin/env python3
2 """Reproduce/audit 22S through the existing offline harness; no downloads.
3 Use --audit-only for SAVED runs. Output must be outside the repository.
4 """
5 import argparse, datetime, hashlib, importlib.util, json, re, sys
6 from pathlib import Path
7 sys.dont_write_bytecode = True
8 ROOT = Path(__file__).resolve().parents[2]
9 D = ROOT/'reports/slice-22s'
10 PRIOR = ROOT/'reports/slice-22r/after'
11 BASE = 'c1fa847c85017d4edf5c2439c772d903261e5a12'
12
13 def read(p): return json.loads(p.read_text())
14 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
15 def save(p, v): p.write_text(json.dumps(v, indent=2, ensure_ascii=False)+'\n')
16 def differences(a,b,path=''):
17 if a==b: return []
18 if isinstance(a,dict) and isinstance(b,dict):
19 return [d for k in sorted(set(a)|set(b)) for d in differences(a.get(k),b.get(k),path+'/'+k)]
20 if isinstance(a,list) and isinstance(b,list) and len(a)==len(b):
21 return [d for i,(x,y) in enumerate(zip(a,b)) for d in differences(x,y,path+'/'+str(i))]
22 return [{'path':path,'before':a,'after':b}]
23
24 def check_control_row(a,b,where):
25 if a==b: return None
26 ca,cb=a.get('condition'),b.get('condition')
27 assert ca and cb and ca['class']==cb['class']=='XTM-UNSUPPORTED',where
28 assert ca['code']=='resource-data-datatype' and cb['code'] in {'resource-resolver-required','cyclic-mergemap'},where
29 assert all(a.get(k)==b.get(k) for k in set(a)|set(b) if k not in {'condition','sourceText','expectedCode'}),where
30 assert all(ca.get(k)==cb.get(k) for k in set(ca)|set(cb) if k not in {'code','detail','message','line','column'}),where
31 return {'where':where,'purpose':'retain real Unsupported non-PASS and source diagnostic checks',
32 'before':a,'after':b,'differences':differences(a,b)}
33
34 def compare_existing(a,b,where):
35 changed=[]
36 for k,v in a.items():
37 if k=='checks':
38 assert [c for c in b[k] if c in set(v)]==v,(where,k)
39 elif k=='cases':
40 assert len(v)==len(b[k])
41 for old,new in zip(v,b[k]):
42 assert {q:z for q,z in old.items() if q!='diagnostics'}=={q:z for q,z in new.items() if q!='diagnostics'},(where,k)
43 if old!=new:
44 # Fault Unsupported payload changes code/text/position only.
45 ds=differences(old,new)
46 assert all('/diagnostics/' in d['path'] for d in ds),(where,ds)
47 changed.append({'where':where+'/cases/'+old['id'],'differences':ds})
48 elif k.endswith('Controls'):
49 assert len(v)==len(b[k]),(where,k)
50 for i,(old,new) in enumerate(zip(v,b[k])):
51 row=check_control_row(old,new,where+'/'+k+'/'+str(i))
52 if row:changed.append(row)
53 elif v!=b[k] and isinstance(v,list):
54 assert len(v)==len(b[k]),(where,k)
55 for i,(old,new) in enumerate(zip(v,b[k])):
56 row=check_control_row(old,new,where+'/'+k+'/'+str(i))
57 if row:changed.append(row)
58 else: assert v==b[k],(where,k)
59 return changed
60
61 def audit(out):
62 initial=read(D/'initial-state.json');scope=read(D/'change-scope.json')
63 assert initial['baseCommit']==BASE and initial['initialIndexAndWorktreeClean']
64 allowed=set(scope['allowedTrackedChanges'])
65 assert all(p in {'src/xtm/reader.lisp','topic-maps.asd','tests/ctm-xtm-mergemap.lisp'} or p.startswith('tests/xtm-') for p in allowed)
66 protected={p:h for p,h in initial['trackedFileSHA256'].items() if p not in allowed}
67 assert protected=={p:sha(ROOT/p) for p in protected}
68 pre=read(D/'integration-preflight-run.json');proof=read(D/'integration-preflight.json')
69 assert pre['exitCode']==0 and pre['readerMatchesBase'] and proof['qualificationPassed']
70 assert pre['readerSHA256']==initial['trackedFileSHA256']['src/xtm/reader.lisp']==scope['preflightReaderSHA256']
71 assert scope['preflightToolSHA256']==sha(D/'integration-preflight.lisp')
72 run=read(out/'runs.json');assert len(run['runs'])==119
73 assert all(r['exitCode']==r['expectedExitCode'] and r['network']==r['repositoryWrites']==r['hyperdocWrites']=='denied'
74 and sha(out/(r['label']+'.log'))==r['logSHA256'] for r in run['runs'])
75 assert run['executionSourcesSHA256']=={p:sha(ROOT/p) for p in run['executionSourcesSHA256']}
76 assert run['rawReportSHA256']=={p:sha(out/p) for p in run['rawReportSHA256']}
77 old,new=read(PRIOR/'xtm.json'),read(out/'xtm.json')
78 retargets=compare_existing(old,new,'normal')
79 assert new['cases']==old['cases']
80 labels=re.findall(r'^XTM-CHECK-PASS (.+)$',(out/'xtm.log').read_text(),re.M)
81 assert labels==new['checks'] and len(labels)==len(set(labels))
82 added=[c for c in labels if c not in set(old['checks'])]
83 assert added and all(c.startswith(('anytype:','anytype-error:','anytype-replay:')) for c in added)
84 x=new['anyTypeReader'];assert x['executed'] and x['XTMReaderAnyTypeIntegration'] and x['originalRegistrationsAdded']==0
85 assert len(x['retargetings'])==len(x['formerSourceReplays']) and all(r['condition'] is None for r in x['formerSourceReplays'])
86 for row in x['positiveProbes']:
87 assert sorted(row['expectedValues'])==sorted(row['actualValues']),row['id']
88 assert bytes.fromhex(row['actualCXTMHex'])==row['actualCXTM'].encode('utf-8')
89 assert row['canonicalReferencesEQ'] and row['repeatedCXTMPure'] and row['inputHex']
90 assert all(row['condition'] for row in x['negativeProbes'])
91 assert new['resourceDataC14N']==old['resourceDataC14N'] and new['c14nPreflight']==old['c14nPreflight']
92 assert len(new['resourceDataC14N']['probes'])==44 and len(new['resourceDataC14N']['nativeValues'])==41
93 before,after=read(PRIOR/'after-diagnostics.json'),read(out/'after-diagnostics.json')
94 assert before==after and len(after)==124
95 fault=[]
96 for r in run['runs']:
97 if '-control-' not in r['label']:continue
98 name=r['label']+'.json';a,b=read(PRIOR/name),read(out/name)
99 changes=compare_existing(a,b,r['label'])
100 assert a['checks']==b['checks'] and a['statuses']==b['statuses']
101 assert r['exitCode']==1 and not b['anyTypeReader']['executed'] and not b['anyTypeReader']['positiveProbes']
102 fault.append({'label':r['label'],'expectedExitCode':1,'caseStatusesAndOutcomesUnchanged':True,
103 'previousCheckLabelsUnchanged':True,'faultCountedAsOriginalPASS':False,'newReaderProbesNotClaimedExecuted':True,
104 'retargetedDiagnostics':changes})
105 assert len(fault)==113
106 retained=[];retained_deltas=[]
107 for p in sorted((out/'retained').rglob('*')):
108 if not p.is_file():continue
109 rel=p.relative_to(out);prior=PRIOR/rel
110 if p.read_bytes()==prior.read_bytes():retained.append({'report':str(rel),'SHA256':sha(p)})
111 else:
112 assert str(rel)=='retained/xtm-mergemap.integration.json',str(rel)
113 ds=differences(read(prior),read(p))
114 assert ds==[{'path':'/rejections/13/code','before':'RESOURCE-DATA-DATATYPE','after':'RESOURCE-RESOLVER-REQUIRED'},
115 {'path':'/rejections/14/code','before':'RESOURCE-DATA-DATATYPE','after':'RESOURCE-RESOLVER-REQUIRED'}],ds
116 retained_deltas.append({'report':str(rel),'differences':ds,
117 'purpose':'two CTM/XTM child-resource Unsupported controls now require explicit XTM resolver; CTM production unchanged'})
118 for rel in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
119 assert (out/rel).read_bytes()==(PRIOR/rel).read_bytes(),rel
120 prev=read(PRIOR/'execution.json');reg=(out/'regression.log').read_text()
121 core=read(ROOT/'reports/slice-22d/baseline/execution.json')['retainedChecks']
122 assert sum(core.values())==prev['retainedLispChecks']
123 for marker,count in dict(core,**prev['stageChecks']).items():
124 assert int(re.search(r'(?m)^'+marker+r'-PASS: (\d+)',reg).group(1))==count,marker
125 assert 'CTM-XTM-EQUIVALENCE-PASS: 529 checks; 30 actual reader pairs; 3 contrasts; 0 capability gaps.' in reg
126 assert 'Ran 2 tests' in reg and 'NFC-ORACLE-PASS: 16294' in reg
127 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out/'xml.log').read_text()
128 assert new['statuses']==old['statuses']=={'pass':121,'fail':0,'unsupported':0,'not-run':0}
129 assert new['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
130 assert read(out/'ctm.validation.json')['statuses']=={'pass':262,'fail':4,'unsupported':0,'not-run':7}
131 assert protected=={p:sha(ROOT/p) for p in protected}
132 save(out/'retargetings.json',{'controlRows':retargets,'runtimeSourceRetargetings':x['retargetings'],
133 'formerSourceReplays':x['formerSourceReplays'],'retainedReportChanges':retained_deltas,
134 'executorSourceBefore':'reports/slice-22d/failure-controls.lisp','executorSourceAfter':'reports/slice-22s/failure-controls.lisp',
135 'executorChange':'Unsupported fixture is valid mergeMap with missing explicit resolver; 113 existing target/mode controls retained',
136 'legacyLabelsAreHistoricalNotCurrentFeatureClaims':True})
137 save(out/'diagnostic-changes.json',{'observations':124,'changed':0,'unchanged':124,'changes':[],'registrationsAdded':0})
138 save(out/'failure-controls-audit.json',fault);save(out/'retained-reports.json',retained)
139 save(out/'integrity.json',{'protectedBaseFilesSHA256':protected,'protectedBaseFileCount':len(protected),
140 'pinnedCorpusFilesVerified':sum(p.startswith('vendor/cxtm/') for p in protected),
141 'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
142 'allXMLC14NTMDMCTMCXTMSourcesUnchanged':True,'allowedTrackedChanges':sorted(allowed),
143 'newProductionFeature':'only XTM Reader anyType integration'})
144 save(out/'execution.json',{'kind':'slice-22s-productive-XTM-anyType-integration','baseCommit':BASE,
145 'runtime':'SBCL 2.5.4','startedAtUTC':run['startedAtUTC'],'finishedAtUTC':run['finishedAtUTC'],
146 'auditedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),'runs':run['runs'],
147 'collector':'scripts/validate-slice-22d.py::collect with explicit 22S fault-driver path',
148 'collectorConfigurationSHA256':sha(D/'collect.py'),'auditToolSHA256':sha(Path(__file__)),
149 'standaloneXTMChecks':len(labels),'retainedStandaloneXTMChecks':len(old['checks']),'newReaderChecks':len(added),
150 'positiveReaderProbes':len(x['positiveProbes']),'negativeReaderProbes':len(x['negativeProbes']),
151 'retargetedSourceReplays':len(x['formerSourceReplays']),'retargetedRegularControlRows':len(retargets),
152 'retainedC14NByteRows':44,'retainedNativeTMDMValueRows':41,'retainedLispChecks':prev['retainedLispChecks'],
153 'stageChecks':prev['stageChecks'],'equivalenceChecks':529,'equivalenceReaderPairs':30,'equivalenceContrasts':3,
154 'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,'faultControls':113,
155 'retainedReportsByteIdentical':len(retained),'retargetedRetainedReports':len(retained_deltas),
156 'all124OriginalDiagnosesUnchanged':True,'statusesBefore':prev['statusesAfter'],'statusesAfter':prev['statusesAfter'],
157 'originalRegistrationsAdded':0,'XTMReaderAnyTypeIntegration':True,'C14NAlgorithmChanged':False,'newExternalDependencies':0,
158 'preImplementationContractPassed':True,'acceptancePassed':True})
159 print(json.dumps({'acceptancePassed':True,'XTMChecks':len(labels),'newChecks':len(added),
160 'positiveReaderProbes':len(x['positiveProbes']),'negativeReaderProbes':len(x['negativeProbes']),
161 'sourceReplays':len(x['formerSourceReplays']),'faultControls':113,'statuses':prev['statusesAfter']}),flush=True)
162
163 def main():
164 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True);ap.add_argument('--audit-only',action='store_true');a=ap.parse_args()
165 out=a.output.resolve();assert not out.is_relative_to(ROOT)
166 if not a.audit_only:
167 spec=importlib.util.spec_from_file_location('slice22s_collector',D/'collect.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);m.collect(out)
168 audit(out)
169 if __name__=='__main__':main()