2 """Reproduce Slice 22D using the existing ASDF, XML and corpus harnesses.
3 Writes only to --output outside the repository. --audit-only rechecks saved runs;
4 it never represents them as new executions. Dependencies come from local pins.
12 from pathlib
import Path
18 sys
.dont_write_bytecode
= True
19 ROOT
= Path(__file__
).resolve().parents
[1]
20 BASE
= 'b5c5ea5cc84dd31104045d032c25880b8bd62839'
21 TOOLS
= ROOT
/ 'reports/slice-22d'
22 TARGET
= 'xtm2/in/variant-datatype-unknown.xtm'
23 RUNTIME
= '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
24 POLICY
= f
'(version 1)(allow default)(deny network*)(deny file-write* (subpath "{ROOT}"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
25 CTM
= {'pass':262,'fail':4,'unsupported':0,'not-run':7}
26 ALLOWED
= {'src/xtm/reader.lisp','topic-maps.asd'} |
{r
['file'] for r
in json
.loads((TOOLS
/'retargetings.json').read_text())}
27 EXECUTION_FILES
= ALLOWED |
{'tests/xtm-resource-datatypes.lisp','reports/slice-22d/failure-controls.lisp'}
28 MODES
= {'mismatch':('cxtm-mismatch',None,None), 'xml':('execution-error','XTM-XML-ERROR','xml-not-well-formed'),
29 'xtm':('execution-error','XTM-REJECTION','invalid-id'), 'unsupported':('unsupported','XTM-UNSUPPORTED','resource-data-datatype'),
30 'tmdm':('execution-error','REIFICATION-ERROR','reifier-conflict')}
32 def read(p
): return json
.loads(p
.read_text())
33 def save(p
,v
): p
.write_text(json
.dumps(v
,indent
=2,ensure_ascii
=False)+'\n')
34 def sha(p
): return hashlib
.sha256(p
.read_bytes()).hexdigest()
35 def git(*args
): return subprocess
.check_output(['git',*args
],cwd
=ROOT
,text
=True).strip()
36 def now(): return datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
37 def prepare_pinned(target
):
38 spec
=importlib
.util
.spec_from_file_location('slice22b',ROOT
/'scripts/validate-slice-22b.py')
39 module
=importlib
.util
.module_from_spec(spec
);spec
.loader
.exec_module(module
)
40 return module
.prepare_pinned(target
)
42 return s
.replace('resourceData-anyType','resourceData-integer').replace('anyType-text-still-unsupported','integer-still-unsupported')
43 def changes(a
,b
,path
=''):
45 if isinstance(a
,dict) and isinstance(b
,dict):
46 return [d
for k
in sorted(a
.keys()|b
.keys()) for d
in changes(a
.get(k
),b
.get(k
),path
+'/'+k
)]
47 if isinstance(a
,list) and isinstance(b
,list) and len(a
)==len(b
):
48 return [d
for i
,(x
,y
) in enumerate(zip(a
,b
)) for d
in changes(x
,y
,path
+'/'+str(i
))]
49 return [{'path':path
,'before':a
,'after':b
}]
52 def condition_nodes(value
,path
=''):
53 if isinstance(value
,dict):
54 for k
,v
in value
.items():
55 if k
=='condition' and isinstance(v
,dict):yield path
+'/'+k
,v
56 else:yield from condition_nodes(v
,path
+'/'+k
)
57 elif isinstance(value
,list):
58 for i
,v
in enumerate(value
):yield from condition_nodes(v
,path
+'/'+str(i
))
60 def verify_baseline(baseline
):
61 ex
=read(baseline
/'execution.json'); assert ex
['acceptancePassed']
62 assert ex
['statusesAfter']['XTM']=={'pass':71,'fail':0,'not-run':53}
63 assert ex
['statusesAfter']['CTM']=={'pass':262,'fail':4,'not-run':7}
64 assert ex
['standaloneXTMChecks']==2396 and ex
['retainedLispChecks']==12100
65 assert len(ex
['runs'])==114 and ex
['faultControls']==108
66 assert all(r
['exitCode']==r
['expectedExitCode'] for r
in ex
['runs'])
67 # Preserve the provenance of the reused 22B runner; its older base/statusBefore
68 # fields describe that runner, not the actual 22D pre-change checkout.
69 previous
=ROOT
/'reports/slice-22b/after'
71 for p
in sorted(baseline
.rglob('*.json')):
72 rel
=p
.relative_to(baseline
); old
=previous
/rel
73 if str(rel
) in {'execution.json','integrity.json','source-preparation.json','baseline-comparison.json','retained-fault-reports.json'}: continue
75 assert read(p
)==read(old
),str(rel
)
76 rows
.append({'report':str(rel
),'sha256':sha(p
),'historical22BBehaviorIdentical':True})
77 assert read(baseline
/'xtm.json')['cases']==read(previous
/'xtm.json')['cases']
80 def collect(output
,baseline
):
81 assert not (output
/'runs.json').exists(),'Use an unused output directory, or --audit-only.'
82 assert subprocess
.check_output([RUNTIME
,'--version'],text
=True).strip()=='SBCL 2.5.4'
83 env
={k
:v
for k
,v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
84 env
.update(PYTHONDONTWRITEBYTECODE
='1',GIT_OPTIONAL_LOCKS
='0',SBCL
=RUNTIME
)
85 sources
={p
:sha(ROOT
/p
) for p
in sorted(EXECUTION_FILES
)}
86 started
=now();runs
=[];retained
=output
/'retained';retained
.mkdir()
87 def run(label
,cmd
,env
,expected
=0):
88 command
=['/usr/bin/sandbox-exec','-p',POLICY
]+cmd
;t
=now()
89 r
=subprocess
.run(command
,cwd
=ROOT
,env
=env
,stdout
=subprocess
.PIPE
,stderr
=subprocess
.STDOUT
,timeout
=300)
90 log
=output
/(label
+'.log');log
.write_bytes(r
.stdout
)
91 runs
.append({'label':label
,'command':command
,'startedAtUTC':t
,'finishedAtUTC':now(),
92 'exitCode':r
.returncode
,'expectedExitCode':expected
,'logSHA256':sha(log
),'network':'denied',
93 'repositoryWrites':'denied','hyperdocWrites':'denied','freshProcess':True})
94 save(output
/'runs.json',{'startedAtUTC':started
,'runs':runs
,'executionSourcesSHA256':sources
})
95 assert r
.returncode
==expected
,(label
,r
.stdout
.decode()[-2500:])
96 print(f
'{label}: expected exit {expected}',flush
=True)
97 return r
.stdout
.decode()
98 with tempfile
.TemporaryDirectory(prefix
='topic-maps-22d-sources-') as tmp
:
99 # The source-preparation subprocess is also network/repository-write denied.
100 text
=run('source-preparation',['python3',str(Path(__file__
)),'--prepare',tmp
+'/sources'],env
)
101 provenance
=json
.loads(text
);save(output
/'source-preparation.json',provenance
)
102 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],TOPIC_MAPS_REPORT
=str(retained
/'core.json'),
103 TOPIC_MAPS_XTM_REPORT
=str(output
/'xtm.json'),TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/'xtm-mergemap'),
104 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/'slice20a'),TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/'slice20b'),
105 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/'slice20c'),TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/'slice20e'),
106 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/'slice20f'),TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/'slice20g'),TOPIC_MAPS_ENCODING_REPORT
=str(output
/'ctm'))
107 prefix
=[RUNTIME
,'--noinform','--no-sysinit','--no-userinit','--script']
108 run('xml',prefix
+['tests/cxml-probe.lisp'],env
)
109 run('variant-reifier-api-preflight',prefix
+['reports/slice-21o/api-preflight.lisp'],env
)
110 run('xtm',prefix
+['scripts/fresh-xtm.lisp'],env
)
111 run('regression',[str(ROOT
/'scripts/test.sh')],env
)
112 run('diagnostics',prefix
+['reports/slice-21o/diagnostics.lisp'],dict(env
,TOPIC_MAPS_21O_ROOT
=str(ROOT
),
113 TOPIC_MAPS_21O_DIAGNOSTICS
=str(output
/'after-diagnostics.json'),TOPIC_MAPS_21O_EXTRA
=tmp
+'/reader-only-extra.json'))
114 labels
=[r
['label'] for r
in read(baseline
/'execution.json')['runs'] if '-control-' in r
['label']]
115 assert len(labels
)==108
116 labels
+= ['datatype-control-'+mode
for mode
in MODES
]
118 family
,mode
=label
.split('-control-')
119 run(label
,prefix
+[str(TOOLS
/'failure-controls.lisp')],dict(env
,TOPIC_MAPS_XTM_REPORT
=str(output
/(label
+'.json')),
120 TOPIC_MAPS_21O_FAMILY
=family
,TOPIC_MAPS_21O_FAULT
=mode
),1)
121 assert sources
=={p
:sha(ROOT
/p
) for p
in sources
}
122 r
=read(output
/'runs.json');r
['finishedAtUTC']=now()
123 r
['rawReportSHA256']={str(p
.relative_to(output
)):sha(p
) for p
in sorted(output
.rglob('*.json')) if p
.name
!='runs.json'}
124 save(output
/'runs.json',r
)
126 def audit(output
,baseline
,initial
):
127 start
=read(initial
);assert start
['baseCommit']==BASE
and start
['indexClean'] and start
['worktreeClean']
128 assert set(git('diff','--name-only',BASE
).splitlines())&set(start
['trackedFileSHA256'])<=ALLOWED
129 protected
={p
:h
for p
,h
in start
['trackedFileSHA256'].items() if p
not in ALLOWED
}
130 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
131 save(output
/'baseline-comparison.json',verify_baseline(baseline
))
132 runs
=read(output
/'runs.json');assert len(runs
['runs'])==119
133 assert all(r
['exitCode']==r
['expectedExitCode'] and sha(output
/(r
['label']+'.log'))==r
['logSHA256'] for r
in runs
['runs'])
134 assert runs
['executionSourcesSHA256']=={p
:sha(ROOT
/p
) for p
in runs
['executionSourcesSHA256']}
135 assert runs
['rawReportSHA256']=={p
:sha(output
/p
) for p
in runs
['rawReportSHA256']}
136 previous
=read(baseline
/'xtm.json');xtm
=read(output
/'xtm.json')
137 labels
=re
.findall(r
'^XTM-CHECK-PASS (.+)$',(output
/'xtm.log').read_text(),re
.M
)
138 assert labels
==xtm
['checks'] and len(labels
)==len(set(labels
))
139 normalized
=[old_label(l
) for l
in labels
]
140 assert [l
for l
in normalized
if l
in set(previous
['checks'])]==previous
['checks']
141 assert xtm
['statuses']=={'pass':72,'fail':0,'unsupported':0,'not-run':0}
142 assert xtm
['otherCorpusInputs']=={'positiveUnexecuted':49,'negativeUnregisteredUnexecuted':3}
143 assert xtm
['cases'][:71]==previous
['cases'] and len(xtm
['cases'])==len({c
['id'] for c
in xtm
['cases']})==72
144 assert xtm
['cases'][-1]['id']==TARGET
and xtm
['cases'][-1]['status']=='pass'
145 assert f
'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 64 original CXTM PASS; 8 negative originals PASS' in (output
/'xtm.log').read_text()
146 # Every prior original model/output stays identical. Control retargetings are
147 # recorded separately, including source and exact condition/position changes.
148 meta
={'scope','statuses','otherCorpusInputs','checks','cases'}
150 for k
,v
in previous
.items():
151 if k
in meta
: continue
152 if k
.endswith('Controls'):
153 assert len(v
)==len(xtm
[k
]),k
154 for i
,(a
,b
) in enumerate(zip(v
,xtm
[k
])):
157 assert all(d
['path'] in {'/label','/sourceText','/sourceKey','/condition/code','/condition/column','/condition/detail','/condition/message','/condition/sourceKey'} for d
in ds
),(k
,i
,ds
)
158 if a
.get('condition') and b
.get('condition'):assert a
['condition']['class']==b
['condition']['class']
159 control_diffs
.append({'field':k
,'index':i
,'before':a
,'after':b
,'differences':ds
})
160 else:assert xtm
[k
]==v
,k
161 save(output
/'retargeted-control-observations.json',control_diffs
)
162 before_diags
=read(baseline
/'after-diagnostics.json');after_diags
=read(output
/'after-diagnostics.json')
163 assert len(before_diags
)==len(after_diags
)==124
164 delta
=[{'id':a
['id'],'before':a
,'after':b
} for a
,b
in zip(before_diags
,after_diags
) if a
!=b
]
165 assert len(delta
)==1 and delta
[0]['id']==TARGET
166 assert delta
[0]['after']['byteExact'] and delta
[0]['after']['condition'] is None
167 save(output
/'diagnostic-changes.json',{'readerObservations':124,'changed':1,'unchanged':123,'changes':delta
})
169 for r
in runs
['runs']:
171 if '-control-' not in label
:continue
172 result
=read(output
/(label
+'.json'));family
,mode
=label
.split('-control-')
173 if family
=='datatype':
174 assert result
['cases'][:71]==xtm
['cases'][:71] and len(result
['cases'])==72
175 failed
=result
['cases'][-1];outcome
,cls
,code
=MODES
[mode
]
176 assert failed
['id']==TARGET
and failed
['actualOutcome']==outcome
and failed
['status']!='pass'
177 assert result
['statuses']['pass']==71
178 original
=result
['datatypeOriginals'][0];c
=original
['condition']
179 if cls
: assert c
['class']==cls
and c
['code']==code
,(mode
,c
)
180 else: assert c
is None and original
['firstDifferentByte']==0
181 faults
.append({'label':label
,'retained':False,'target':TARGET
,'condition':c
,'actualOutcome':outcome
,
182 'caseStatus':failed
['status'],'retainedOriginalPASS':71,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
184 old
=read(baseline
/(label
+'.json'))
185 assert len(old
['cases'])==len(result
['cases']),label
186 for a
,b
in zip(old
['cases'],result
['cases']):
187 assert {k
:v
for k
,v
in a
.items() if k
!='diagnostics'}=={k
:v
for k
,v
in b
.items() if k
!='diagnostics'},(label
,a
,b
)
188 if a
['status']=='pass': assert a
==b
189 # Stop-on-error execution retains the same successful prefix. Later
190 # cases are not executed; the normal run verifies all prior 71 PASS.
191 assert result
['statuses']==old
['statuses'],label
192 old_conditions
=dict(condition_nodes(old
));new_conditions
=dict(condition_nodes(result
))
193 assert old_conditions
.keys()==new_conditions
.keys(),label
195 for path
,c
in old_conditions
.items():
196 d
=new_conditions
[path
]
197 assert c
['class']==d
['class'],(label
,path
,c
,d
)
198 if c
.get('code')!=d
.get('code'):
199 # This documented synthetic source was changed from invalid
200 # default-string XML to valid but unimplemented anyType XML.
201 assert path
=='/variantControls/52/condition' and c
.get('code')=='resource-data-markup' and d
.get('code')=='resource-data-datatype'
202 if c
!=d
:condition_diffs
.append({'path':path
,'before':c
,'after':d
})
203 faults
.append({'label':label
,'retained':True,'sameCaseStatusesAndOutcomes':True,
204 'conditionClassesUnchanged':True,'conditionCodesUnchangedExceptDocumentedEmbeddedXMLRetargeting':True,'conditionChanges':condition_diffs
,
205 'priorCorrectPASSRetained':sum(c
['status']=='pass' for c
in result
['cases']),
206 'expectedASDFFailureObserved':True,'faultCountedAsPass':False,
207 'caseDiagnosticChanges':changes(old
['cases'],result
['cases'])})
208 assert len(faults
)==113
209 save(output
/'failure-controls.json',{'retained':108,'new':5,'controls':faults
,'allExpectedFailureExitsObserved':True})
211 for old
in [*sorted((baseline
/'retained').glob('*.json')),*[baseline
/f
'ctm.{s}.json' for s
in ['validation','cxtm','encoding']]]:
212 rel
=old
.relative_to(baseline
)
213 if str(rel
)=='retained/xtm-mergemap.integration.json' and old
.read_bytes()!=(output
/rel
).read_bytes():
214 ds
=changes(read(old
),read(output
/rel
))
215 save(output
/'mergemap-retargeting.json',{'report':str(rel
),'differences':ds
})
216 assert all('Only textual xsd:string and xsd:anyURI' in str(d
['before']) or 'XMLSchema#integer' in str(d
['before']) for d
in ds
),ds
218 assert old
.read_bytes()==(output
/rel
).read_bytes(),str(rel
)
219 unchanged
.append({'report':str(rel
),'sha256':sha(old
),'byteIdentical':True})
220 assert len(unchanged
) in (47,48)
221 save(output
/'retained-reports.json',unchanged
)
222 regression
=(output
/'regression.log').read_text();ex
=read(baseline
/'execution.json')
223 for marker
,count
in dict(ex
['retainedChecks'],**ex
['stageChecks']).items():
224 assert int(re
.search(r
'(?m)^'+marker
+r
'-PASS: (\d+)',regression
).group(1))==count
225 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
226 xml
=(output
/'xml.log').read_text();assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
227 assert len(set(re
.findall(r
'^CXML-CHECK-PASS (.+)$',xml
,re
.M
)))==146
228 assert read(output
/'ctm.validation.json')['statuses']==CTM
229 original
=xtm
['datatypeOriginals'][0];source
=ROOT
/'vendor/cxtm'/TARGET
230 baseline_file
=ROOT
/'vendor/cxtm/xtm2/baseline/variant-datatype-unknown.xtm.cxtm'
231 actual
=original
['actualCXTM'].encode();expected
=baseline_file
.read_bytes()
232 assert actual
==expected
==original
['expectedCXTM'].encode() and len(actual
)==original
['actualBytes']==original
['expectedBytes']==579
233 assert source
.read_bytes()==bytes
.fromhex(original
['sourceHex'])==original
['sourceText'].encode()
234 assert original
['sourceBytes']==source
.stat().st_size
and original
['condition'] is None and original
['firstDifferentByte'] is None
235 assert original
['qualifiedParserCalls']==1 and original
['status']=='pass' and original
['sourceKey']==TARGET
236 save(output
/'original-contract.json',dict(original
,sourceSHA256
=sha(source
),baselineSHA256
=sha(baseline_file
),
237 actualSHA256
=hashlib
.sha256(actual
).hexdigest(),byteExact
=True,executedInASDFTestOp
=True))
238 (output
/'actual').mkdir(exist_ok
=True);(output
/'actual/variant-datatype-unknown.xtm.cxtm').write_bytes(actual
)
239 matrix
=read(ROOT
/'reports/slice-22a/matrix.json');rows
=[]
240 prior_registered
={c
['id']:c
['status'] for c
in previous
['cases']}
241 for c
in matrix
['cases']:
242 before
=prior_registered
.get(c
['id'],c
['registeredStatus']);after
='pass' if c
['id']==TARGET
else before
243 rows
.append({'id':c
['id'],'slice22A':c
['registeredStatus'],'slice22C':before
,'slice22D':after
,'changedIn22D':before
!=after
})
244 assert sum(r
['changedIn22D'] for r
in rows
)==1
245 save(output
/'closure-delta.json',{'sourceMatrix':'reports/slice-22a/matrix.json','sourceMatrixSHA256':sha(ROOT
/'reports/slice-22a/matrix.json'),
246 'changedCase':TARGET
,'newPASS':1,'remainingOpenCases':63,'slice22CDecisionsUnchanged':True,'twelveOpenCDecisionsUnchanged':True,'cases':rows
})
247 lock
=read(ROOT
/'references/corpus-lock.json')
248 for entry
in lock
['files']:
249 p
=ROOT
/'vendor/cxtm'/entry
['path'];assert p
.stat().st_size
==entry
['size'] and sha(p
)==entry
['sha256']
250 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
251 save(output
/'integrity.json',{'baseCommit':BASE
,'protectedBaseFilesSHA256':protected
,'executionSourcesSHA256':runs
['executionSourcesSHA256'],
252 'pinnedCorpusFilesVerified':len(lock
['files']),'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),
253 'productionChanges':['src/xtm/reader.lisp'],'historicalReportsUnchanged':True,'originalSourcesBaselinesAndLockUnchanged':True})
254 after
={'XTM':{'pass':72,'fail':0,'not-run':52},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':334,'fail':4,'not-run':59}}
255 save(output
/'execution.json',{'kind':'slice-22d-textual-resource-datatypes','baseCommit':BASE
,'branch':git('branch','--show-current'),
256 'runtime':'SBCL 2.5.4','auditToolSHA256':sha(Path(__file__
)),'startedAtUTC':runs
['startedAtUTC'],'finishedAtUTC':runs
['finishedAtUTC'],'auditedAtUTC':now(),'runs':runs
['runs'],
257 'standaloneXTMChecks':len(labels
),'retainedXTMChecks':2396,'newXTMChecks':len(labels
)-2396,'retainedLispChecks':12100,
258 'retainedChecks':ex
['retainedChecks'],'stageChecks':ex
['stageChecks'],'xmlQualificationChecks':146,
259 'retainedOriginals':71,'newPositiveOriginals':1,'faultControls':113,'retainedFaultControls':108,'newFaultControls':5,
260 'diagnosticOriginalObservations':124,'diagnosticChanges':1,'statusesBefore':ex
['statusesAfter'],'statusesAfter':after
,
261 'pythonUnitTests':2,'nfcOracleComparisons':16294,'retainedNonXTMReportsByteIdentical':len(unchanged
),
262 'knownCTMFailuresUnchanged':[c
['id'] for c
in read(output
/'ctm.validation.json')['cases'] if c
['status']=='fail'],
263 'pinnedCorpusFilesVerified':len(lock
['files']),'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),
264 'productionChanges':['src/xtm/reader.lisp'],'historicalReportsUnchanged':True,'originalSourcesAndBaselinesUnchanged':True,
265 'newDependencyDownloads':0,'persistentLispImagesTouched':False,'acceptancePassed':True})
266 print(json
.dumps({'acceptancePassed':True,'statuses':after
,'xtmChecks':len(labels
),'faultControls':113,'output':str(output
)}))
269 ap
=argparse
.ArgumentParser(description
=__doc__
)
270 ap
.add_argument('--output',type=Path
);ap
.add_argument('--baseline',type=Path
,default
=TOOLS
/'baseline')
271 ap
.add_argument('--initial',type=Path
,default
=TOOLS
/'initial-state.json');ap
.add_argument('--prepare',type=Path
)
272 ap
.add_argument('--audit-only',action
='store_true');args
=ap
.parse_args()
273 if args
.prepare
:print(json
.dumps(prepare_pinned(args
.prepare
)));return
274 assert args
.output
;output
=args
.output
.resolve();assert not output
.is_relative_to(ROOT
)
275 output
.mkdir(parents
=True,exist_ok
=True)
276 verify_baseline(args
.baseline
)
277 if not args
.audit_only
:collect(output
,args
.baseline
)
278 audit(output
,args
.baseline
,args
.initial
)
280 if __name__
=='__main__':main()