]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22s/reproduce.py
2 """Reproduce/audit 22S through the existing offline harness; no downloads.
3 Use --audit-only for SAVED runs. Output must be outside the repository.
5 import argparse
, datetime
, hashlib
, importlib
.util
, json
, re
, sys
6 from pathlib
import Path
7 sys
.dont_write_bytecode
= True
8 ROOT
= Path(__file__
).resolve().parents
[2]
9 D
= ROOT
/'reports/slice-22s'
10 PRIOR
= ROOT
/'reports/slice-22r/after'
11 BASE
= 'c1fa847c85017d4edf5c2439c772d903261e5a12'
13 def read(p
): return json
.loads(p
.read_text())
14 def sha(p
): return hashlib
.sha256(p
.read_bytes()).hexdigest()
15 def save(p
, v
): p
.write_text(json
.dumps(v
, indent
=2, ensure_ascii
=False)+'\n')
16 def differences(a
,b
,path
=''):
18 if isinstance(a
,dict) and isinstance(b
,dict):
19 return [d
for k
in sorted(set(a
)|
set(b
)) for d
in differences(a
.get(k
),b
.get(k
),path
+'/'+k
)]
20 if isinstance(a
,list) and isinstance(b
,list) and len(a
)==len(b
):
21 return [d
for i
,(x
,y
) in enumerate(zip(a
,b
)) for d
in differences(x
,y
,path
+'/'+str(i
))]
22 return [{'path':path
,'before':a
,'after':b
}]
24 def check_control_row(a
,b
,where
):
26 ca
,cb
=a
.get('condition'),b
.get('condition')
27 assert ca
and cb
and ca
['class']==cb
['class']=='XTM-UNSUPPORTED',where
28 assert ca
['code']=='resource-data-datatype' and cb
['code'] in {'resource-resolver-required','cyclic-mergemap'},where
29 assert all(a
.get(k
)==b
.get(k
) for k
in set(a
)|
set(b
) if k
not in {'condition','sourceText','expectedCode'}),where
30 assert all(ca
.get(k
)==cb
.get(k
) for k
in set(ca
)|
set(cb
) if k
not in {'code','detail','message','line','column'}),where
31 return {'where':where
,'purpose':'retain real Unsupported non-PASS and source diagnostic checks',
32 'before':a
,'after':b
,'differences':differences(a
,b
)}
34 def compare_existing(a
,b
,where
):
38 assert [c
for c
in b
[k
] if c
in set(v
)]==v
,(where
,k
)
40 assert len(v
)==len(b
[k
])
41 for old
,new
in zip(v
,b
[k
]):
42 assert {q
:z
for q
,z
in old
.items() if q
!='diagnostics'}=={q
:z
for q
,z
in new
.items() if q
!='diagnostics'},(where
,k
)
44 # Fault Unsupported payload changes code/text/position only.
45 ds
=differences(old
,new
)
46 assert all('/diagnostics/' in d
['path'] for d
in ds
),(where
,ds
)
47 changed
.append({'where':where
+'/cases/'+old
['id'],'differences':ds
})
48 elif k
.endswith('Controls'):
49 assert len(v
)==len(b
[k
]),(where
,k
)
50 for i
,(old
,new
) in enumerate(zip(v
,b
[k
])):
51 row
=check_control_row(old
,new
,where
+'/'+k
+'/'+str(i
))
52 if row
:changed
.append(row
)
53 elif v
!=b
[k
] and isinstance(v
,list):
54 assert len(v
)==len(b
[k
]),(where
,k
)
55 for i
,(old
,new
) in enumerate(zip(v
,b
[k
])):
56 row
=check_control_row(old
,new
,where
+'/'+k
+'/'+str(i
))
57 if row
:changed
.append(row
)
58 else: assert v
==b
[k
],(where
,k
)
62 initial
=read(D
/'initial-state.json');scope
=read(D
/'change-scope.json')
63 assert initial
['baseCommit']==BASE
and initial
['initialIndexAndWorktreeClean']
64 allowed
=set(scope
['allowedTrackedChanges'])
65 assert all(p
in {'src/xtm/reader.lisp','topic-maps.asd','tests/ctm-xtm-mergemap.lisp'} or p
.startswith('tests/xtm-') for p
in allowed
)
66 protected
={p
:h
for p
,h
in initial
['trackedFileSHA256'].items() if p
not in allowed
}
67 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
68 pre
=read(D
/'integration-preflight-run.json');proof
=read(D
/'integration-preflight.json')
69 assert pre
['exitCode']==0 and pre
['readerMatchesBase'] and proof
['qualificationPassed']
70 assert pre
['readerSHA256']==initial
['trackedFileSHA256']['src/xtm/reader.lisp']==scope
['preflightReaderSHA256']
71 assert scope
['preflightToolSHA256']==sha(D
/'integration-preflight.lisp')
72 run
=read(out
/'runs.json');assert len(run
['runs'])==119
73 assert all(r
['exitCode']==r
['expectedExitCode'] and r
['network']==r
['repositoryWrites']==r
['hyperdocWrites']=='denied'
74 and sha(out
/(r
['label']+'.log'))==r
['logSHA256'] for r
in run
['runs'])
75 assert run
['executionSourcesSHA256']=={p
:sha(ROOT
/p
) for p
in run
['executionSourcesSHA256']}
76 assert run
['rawReportSHA256']=={p
:sha(out
/p
) for p
in run
['rawReportSHA256']}
77 old
,new
=read(PRIOR
/'xtm.json'),read(out
/'xtm.json')
78 retargets
=compare_existing(old
,new
,'normal')
79 assert new
['cases']==old
['cases']
80 labels
=re
.findall(r
'^XTM-CHECK-PASS (.+)$',(out
/'xtm.log').read_text(),re
.M
)
81 assert labels
==new
['checks'] and len(labels
)==len(set(labels
))
82 added
=[c
for c
in labels
if c
not in set(old
['checks'])]
83 assert added
and all(c
.startswith(('anytype:','anytype-error:','anytype-replay:')) for c
in added
)
84 x
=new
['anyTypeReader'];assert x
['executed'] and x
['XTMReaderAnyTypeIntegration'] and x
['originalRegistrationsAdded']==0
85 assert len(x
['retargetings'])==len(x
['formerSourceReplays']) and all(r
['condition'] is None for r
in x
['formerSourceReplays'])
86 for row
in x
['positiveProbes']:
87 assert sorted(row
['expectedValues'])==sorted(row
['actualValues']),row
['id']
88 assert bytes
.fromhex(row
['actualCXTMHex'])==row
['actualCXTM'].encode('utf-8')
89 assert row
['canonicalReferencesEQ'] and row
['repeatedCXTMPure'] and row
['inputHex']
90 assert all(row
['condition'] for row
in x
['negativeProbes'])
91 assert new
['resourceDataC14N']==old
['resourceDataC14N'] and new
['c14nPreflight']==old
['c14nPreflight']
92 assert len(new
['resourceDataC14N']['probes'])==44 and len(new
['resourceDataC14N']['nativeValues'])==41
93 before
,after
=read(PRIOR
/'after-diagnostics.json'),read(out
/'after-diagnostics.json')
94 assert before
==after
and len(after
)==124
97 if '-control-' not in r
['label']:continue
98 name
=r
['label']+'.json';a
,b
=read(PRIOR
/name
),read(out
/name
)
99 changes
=compare_existing(a
,b
,r
['label'])
100 assert a
['checks']==b
['checks'] and a
['statuses']==b
['statuses']
101 assert r
['exitCode']==1 and not b
['anyTypeReader']['executed'] and not b
['anyTypeReader']['positiveProbes']
102 fault
.append({'label':r
['label'],'expectedExitCode':1,'caseStatusesAndOutcomesUnchanged':True,
103 'previousCheckLabelsUnchanged':True,'faultCountedAsOriginalPASS':False,'newReaderProbesNotClaimedExecuted':True,
104 'retargetedDiagnostics':changes
})
105 assert len(fault
)==113
106 retained
=[];retained_deltas
=[]
107 for p
in sorted((out
/'retained').rglob('*')):
108 if not p
.is_file():continue
109 rel
=p
.relative_to(out
);prior
=PRIOR
/rel
110 if p
.read_bytes()==prior
.read_bytes():retained
.append({'report':str(rel
),'SHA256':sha(p
)})
112 assert str(rel
)=='retained/xtm-mergemap.integration.json',str(rel
)
113 ds
=differences(read(prior
),read(p
))
114 assert ds
==[{'path':'/rejections/13/code','before':'RESOURCE-DATA-DATATYPE','after':'RESOURCE-RESOLVER-REQUIRED'},
115 {'path':'/rejections/14/code','before':'RESOURCE-DATA-DATATYPE','after':'RESOURCE-RESOLVER-REQUIRED'}],ds
116 retained_deltas
.append({'report':str(rel
),'differences':ds
,
117 'purpose':'two CTM/XTM child-resource Unsupported controls now require explicit XTM resolver; CTM production unchanged'})
118 for rel
in ['ctm.validation.json','ctm.cxtm.json','ctm.encoding.json']:
119 assert (out
/rel
).read_bytes()==(PRIOR
/rel
).read_bytes(),rel
120 prev
=read(PRIOR
/'execution.json');reg
=(out
/'regression.log').read_text()
121 core
=read(ROOT
/'reports/slice-22d/baseline/execution.json')['retainedChecks']
122 assert sum(core
.values())==prev
['retainedLispChecks']
123 for marker
,count
in dict(core
,**prev
['stageChecks']).items():
124 assert int(re
.search(r
'(?m)^'+marker
+r
'-PASS: (\d+)',reg
).group(1))==count
,marker
125 assert 'CTM-XTM-EQUIVALENCE-PASS: 529 checks; 30 actual reader pairs; 3 contrasts; 0 capability gaps.' in reg
126 assert 'Ran 2 tests' in reg
and 'NFC-ORACLE-PASS: 16294' in reg
127 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in (out
/'xml.log').read_text()
128 assert new
['statuses']==old
['statuses']=={'pass':121,'fail':0,'unsupported':0,'not-run':0}
129 assert new
['otherCorpusInputs']=={'positiveUnexecuted':3,'negativeUnregisteredUnexecuted':0}
130 assert read(out
/'ctm.validation.json')['statuses']=={'pass':262,'fail':4,'unsupported':0,'not-run':7}
131 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
132 save(out
/'retargetings.json',{'controlRows':retargets
,'runtimeSourceRetargetings':x
['retargetings'],
133 'formerSourceReplays':x
['formerSourceReplays'],'retainedReportChanges':retained_deltas
,
134 'executorSourceBefore':'reports/slice-22d/failure-controls.lisp','executorSourceAfter':'reports/slice-22s/failure-controls.lisp',
135 'executorChange':'Unsupported fixture is valid mergeMap with missing explicit resolver; 113 existing target/mode controls retained',
136 'legacyLabelsAreHistoricalNotCurrentFeatureClaims':True})
137 save(out
/'diagnostic-changes.json',{'observations':124,'changed':0,'unchanged':124,'changes':[],'registrationsAdded':0})
138 save(out
/'failure-controls-audit.json',fault
);save(out
/'retained-reports.json',retained
)
139 save(out
/'integrity.json',{'protectedBaseFilesSHA256':protected
,'protectedBaseFileCount':len(protected
),
140 'pinnedCorpusFilesVerified':sum(p
.startswith('vendor/cxtm/') for p
in protected
),
141 'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),
142 'allXMLC14NTMDMCTMCXTMSourcesUnchanged':True,'allowedTrackedChanges':sorted(allowed
),
143 'newProductionFeature':'only XTM Reader anyType integration'})
144 save(out
/'execution.json',{'kind':'slice-22s-productive-XTM-anyType-integration','baseCommit':BASE
,
145 'runtime':'SBCL 2.5.4','startedAtUTC':run
['startedAtUTC'],'finishedAtUTC':run
['finishedAtUTC'],
146 'auditedAtUTC':datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),'runs':run
['runs'],
147 'collector':'scripts/validate-slice-22d.py::collect with explicit 22S fault-driver path',
148 'collectorConfigurationSHA256':sha(D
/'collect.py'),'auditToolSHA256':sha(Path(__file__
)),
149 'standaloneXTMChecks':len(labels
),'retainedStandaloneXTMChecks':len(old
['checks']),'newReaderChecks':len(added
),
150 'positiveReaderProbes':len(x
['positiveProbes']),'negativeReaderProbes':len(x
['negativeProbes']),
151 'retargetedSourceReplays':len(x
['formerSourceReplays']),'retargetedRegularControlRows':len(retargets
),
152 'retainedC14NByteRows':44,'retainedNativeTMDMValueRows':41,'retainedLispChecks':prev
['retainedLispChecks'],
153 'stageChecks':prev
['stageChecks'],'equivalenceChecks':529,'equivalenceReaderPairs':30,'equivalenceContrasts':3,
154 'xmlQualificationChecks':146,'pythonUnitTests':2,'nfcOracleComparisons':16294,'faultControls':113,
155 'retainedReportsByteIdentical':len(retained
),'retargetedRetainedReports':len(retained_deltas
),
156 'all124OriginalDiagnosesUnchanged':True,'statusesBefore':prev
['statusesAfter'],'statusesAfter':prev
['statusesAfter'],
157 'originalRegistrationsAdded':0,'XTMReaderAnyTypeIntegration':True,'C14NAlgorithmChanged':False,'newExternalDependencies':0,
158 'preImplementationContractPassed':True,'acceptancePassed':True})
159 print(json
.dumps({'acceptancePassed':True,'XTMChecks':len(labels
),'newChecks':len(added
),
160 'positiveReaderProbes':len(x
['positiveProbes']),'negativeReaderProbes':len(x
['negativeProbes']),
161 'sourceReplays':len(x
['formerSourceReplays']),'faultControls':113,'statuses':prev
['statusesAfter']}),flush
=True)
164 ap
=argparse
.ArgumentParser(description
=__doc__
);ap
.add_argument('--output',type=Path
,required
=True);ap
.add_argument('--audit-only',action
='store_true');a
=ap
.parse_args()
165 out
=a
.output
.resolve();assert not out
.is_relative_to(ROOT
)
167 spec
=importlib
.util
.spec_from_file_location('slice22s_collector',D
/'collect.py');m
=importlib
.util
.module_from_spec(spec
);spec
.loader
.exec_module(m
);m
.collect(out
)
169 if __name__
=='__main__':main()