2 """Validate one XTM original registration through the existing offline harness.
3 Outputs are outside the repository. No production edits or new test architecture.
4 The pre-change execution is retained in reports/slice-22b/baseline; --baseline may
5 select another verified fresh baseline. The default reruns the complete current
6 regression and all 108 executor controls against the immutable Slice-22A evidence.
14 from pathlib
import Path
21 sys
.dont_write_bytecode
= True
22 ROOT
= Path(__file__
).resolve().parents
[1]
23 BASE
= 'b12ece02faa039e816fca3750acd9eceb6be1513'
24 PRIOR
= ROOT
/ 'reports/slice-22a/baseline'
25 TOOLS
= ROOT
/ 'reports/slice-22b'
26 TARGET
= 'xtm2/in/variant-duplicate-reifier2.xtm'
27 RUNTIME
= '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
28 ALLOWED
= {'tests/xtm-identity.lisp', 'tests/xtm-variant-reification.lisp', 'topic-maps.asd'}
29 POLICY
= '(version 1)(allow default)(deny network*)(deny file-write* (subpath "'+str(ROOT
)+'"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
30 CTM
= {'pass':262,'fail':4,'unsupported':0,'not-run':7}
33 def read(p
): return json
.loads(p
.read_text())
34 def save(p
,v
): p
.write_text(json
.dumps(v
,indent
=2,ensure_ascii
=False)+'\n')
35 def sha(p
): return hashlib
.sha256(p
.read_bytes()).hexdigest()
36 def git(*args
): return subprocess
.check_output(['git',*args
],cwd
=ROOT
,text
=True).strip()
37 def prepare_pinned(target
):
38 spec
=importlib
.util
.spec_from_file_location('slice21o',ROOT
/'scripts/validate-slice-21o.py')
39 module
=importlib
.util
.module_from_spec(spec
);spec
.loader
.exec_module(module
)
40 return module
.prepare_pinned(target
)
43 def verify_before(path
):
44 ex
=read(path
/'execution.json')
45 assert ex
['acceptancePassed'] and ex
['xtmStatusesAfter']=={'pass':70,'fail':0,'unsupported':0,'not-run':54}
46 assert ex
['ctmStatusesAfter']==CTM
and len(ex
['runs'])==109
47 assert all(r
['exitCode']==r
['expectedExitCode'] for r
in ex
['runs'])
49 for item
in read(ROOT
/'reports/slice-22a/baseline-comparison.json'):
50 name
=item
['freshReport'].removeprefix('baseline/')
51 a
,b
=PRIOR
/name
,path
/name
52 assert a
.read_bytes()==b
.read_bytes(),name
53 rows
.append({'slice22AReport':str(a
.relative_to(ROOT
)),'freshBeforeReport':'baseline/'+name
,
54 'sha256':sha(b
),'byteIdentical':True})
60 ap
=argparse
.ArgumentParser(description
=__doc__
)
61 ap
.add_argument('--sbcl',default
=RUNTIME
);ap
.add_argument('--output',type=Path
)
62 ap
.add_argument('--baseline',type=Path
,default
=TOOLS
/'baseline');ap
.add_argument('--prepare',type=Path
)
64 if args
.prepare
: print(json
.dumps(prepare_pinned(args
.prepare
)));return
65 assert args
.output
,'--output outside the repository is required'
66 output
=args
.output
.resolve();assert not output
.is_relative_to(ROOT
)
67 output
.mkdir(parents
=True,exist_ok
=True);assert not (output
/'execution.json').exists(),'Choose an unused output directory.'
68 runtime
=shutil
.which(args
.sbcl
);assert runtime
and subprocess
.check_output([runtime
,'--version'],text
=True).strip()=='SBCL 2.5.4'
69 subprocess
.run(['git','merge-base','--is-ancestor',BASE
,'HEAD'],cwd
=ROOT
,check
=True)
70 tracked
=git('ls-tree','-r','--name-only',BASE
).splitlines()
71 changed
=set(git('diff','--name-only',BASE
).splitlines()) & set(tracked
);assert changed
<=ALLOWED
,changed
72 before_hashes
={p
:sha(ROOT
/p
) for p
in tracked
};protected
={p
:v
for p
,v
in before_hashes
.items() if p
not in ALLOWED
}
73 sources
={p
:sha(ROOT
/p
) for p
in [*sorted(ALLOWED
),'scripts/validate-slice-22b.py','reports/slice-22b/failure-controls.lisp']}
74 save(output
/'baseline-comparison.json',verify_before(args
.baseline
.resolve()))
75 previous
=read(PRIOR
/'xtm.json');assert len(previous
['cases'])==70 and TARGET
not in {c
['id'] for c
in previous
['cases']}
76 started
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat();runs
=[]
77 env
={k
:v
for k
,v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
78 env
.update(PYTHONDONTWRITEBYTECODE
='1',GIT_OPTIONAL_LOCKS
='0',SBCL
=runtime
)
79 def run(label
,command
,env
,exit_code
=0):
80 command
=['/usr/bin/sandbox-exec','-p',POLICY
]+command
81 t
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
82 result
=subprocess
.run(command
,cwd
=ROOT
,env
=env
,stdout
=subprocess
.PIPE
,stderr
=subprocess
.STDOUT
,timeout
=300)
83 (output
/(label
+'.log')).write_bytes(result
.stdout
)
84 runs
.append({'label':label
,'command':command
,'startedAtUTC':t
,'finishedAtUTC':datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
85 'exitCode':result
.returncode
,'expectedExitCode':exit_code
,'network':'denied','repositoryWrites':'denied','hyperdocWrites':'denied','freshProcess':True})
86 assert result
.returncode
==exit_code
,(label
,result
.stdout
.decode()[-3500:])
87 return result
.stdout
.decode()
88 retained
=output
/'retained';retained
.mkdir()
89 with tempfile
.TemporaryDirectory(prefix
='topic-maps-22b-sources-') as tmp
:
90 provenance
=json
.loads(run('source-preparation',['python3',str(Path(__file__
)), '--prepare',tmp
+'/sources'],env
));save(output
/'source-preparation.json',provenance
)
91 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],TOPIC_MAPS_REPORT
=str(retained
/'core.json'),
92 TOPIC_MAPS_XTM_REPORT
=str(output
/'xtm.json'),TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/'xtm-mergemap'),
93 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/'slice20a'),TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/'slice20b'),
94 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/'slice20c'),TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/'slice20e'),
95 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/'slice20f'),TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/'slice20g'),TOPIC_MAPS_ENCODING_REPORT
=str(output
/'ctm'))
96 prefix
=[runtime
,'--noinform','--no-sysinit','--no-userinit','--script']
97 xml
=run('xml',prefix
+['tests/cxml-probe.lisp'],env
)
98 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
99 assert len(set(re
.findall(r
'^CXML-CHECK-PASS (.+)$',xml
,re
.M
)))==146
100 preflight
=run('variant-reifier-api-preflight',prefix
+['reports/slice-21o/api-preflight.lisp'],env
)
101 assert 'VARIANT-REIFIER-API-PREFLIGHT-PASS:' in preflight
102 xtm_log
=run('xtm',prefix
+['scripts/fresh-xtm.lisp'],env
);xtm
=read(output
/'xtm.json')
103 labels
=re
.findall(r
'^XTM-CHECK-PASS (.+)$',xtm_log
,re
.M
);assert labels
==xtm
['checks'] and len(labels
)==len(set(labels
))
104 assert [l
for l
in labels
if l
in set(previous
['checks'])]==previous
['checks']
105 assert len(set(labels
)-set(previous
['checks']))==12
106 assert f
'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 63 original CXTM PASS; 8 negative originals PASS' in xtm_log
107 assert xtm
['statuses']=={'pass':71,'fail':0,'unsupported':0,'not-run':0}
108 assert xtm
['otherCorpusInputs']=={'positiveUnexecuted':50,'negativeUnregisteredUnexecuted':3}
109 assert xtm
['cases'][:70]==previous
['cases'] and len(xtm
['cases'])==len({c
['id'] for c
in xtm
['cases']})==71
110 assert xtm
['cases'][-1]['id']==TARGET
and xtm
['cases'][-1]['status']=='pass' and xtm
['cases'][-1]['actualOutcome']=='cxtm-match'
111 allowed_fields
={'scope','statuses','otherCorpusInputs','checks','cases','variantReifierOriginals'}
112 for k
,v
in previous
.items():
113 if k
not in allowed_fields
:assert xtm
[k
]==v
,k
114 assert xtm
['scope']==previous
['scope'].replace('62 positive','63 positive')
115 assert len(xtm
['variantReifierOriginals'])==3 and xtm
['variantReifierOriginals'][:2]==previous
['variantReifierOriginals']
116 regression
=run('regression',[str(ROOT
/'scripts/test.sh')],env
)
117 # This unchanged reader-only diagnostic driver performs no registration.
118 # Its historical extra flag is process-local and is not a registry report.
119 run('diagnostics',prefix
+['reports/slice-21o/diagnostics.lisp'],dict(env
,TOPIC_MAPS_21O_ROOT
=str(ROOT
),
120 TOPIC_MAPS_21O_DIAGNOSTICS
=str(output
/'after-diagnostics.json'),TOPIC_MAPS_21O_EXTRA
=tmp
+'/reader-only-extra.json'))
121 assert read(output
/'after-diagnostics.json')==read(PRIOR
/'after-diagnostics.json')
122 assert len(read(output
/'after-diagnostics.json'))==124
123 fault_controls
=[];retained_faults
=[]
124 old_fault_runs
=[r
for r
in read(args
.baseline
/'execution.json')['runs'] if '-control-' in r
['label']]
125 assert len(old_fault_runs
)==103
126 for entry
in old_fault_runs
:
127 label
=entry
['label'];family
,mode
=label
.split('-control-')
128 run(label
,prefix
+['reports/slice-21o/failure-controls.lisp'],dict(env
,TOPIC_MAPS_XTM_REPORT
=str(output
/(label
+'.json')),
129 TOPIC_MAPS_21O_FAMILY
=family
,TOPIC_MAPS_21O_FAULT
=mode
),1)
130 old
,new
=read(args
.baseline
/(label
+'.json')),read(output
/(label
+'.json'))
131 assert new
['scope']==old
['scope'].replace('62 positive','63 positive')
132 assert {k
:v
for k
,v
in old
.items() if k
!='scope'}=={k
:v
for k
,v
in new
.items() if k
!='scope'},label
133 retained_faults
.append({'label':label
,'allBehavioralFieldsUnchanged':True,'onlyMetadataChange':'/scope: 62 -> 63 positive capacity',
134 'beforeSHA256':sha(args
.baseline
/(label
+'.json')),'afterSHA256':sha(output
/(label
+'.json'))})
135 fault_controls
.append({'label':label
,'retained':True,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
136 modes
={'mismatch':('cxtm-mismatch',None,None),'xml':('execution-error','XTM-XML-ERROR','xml-not-well-formed'),
137 'xtm':('execution-error','XTM-REJECTION','invalid-id'),'unsupported':('unsupported','XTM-UNSUPPORTED','resource-data-datatype'),
138 'tmdm':('execution-error','REIFICATION-ERROR','reifier-conflict')}
139 for mode
,(outcome
,cls
,code
) in modes
.items():
140 label
='variant-reifier-followup-control-'+mode
141 run(label
,prefix
+[str(TOOLS
/'failure-controls.lisp')],dict(env
,TOPIC_MAPS_XTM_REPORT
=str(output
/(label
+'.json')),TOPIC_MAPS_22B_FAULT
=mode
),1)
142 result
=read(output
/(label
+'.json'));assert result
['cases'][:70]==xtm
['cases'][:70] and len(result
['cases'])==71
143 failed
=result
['cases'][-1];assert failed
['id']==TARGET
and failed
['actualOutcome']==outcome
and failed
['status']!='pass'
144 assert result
['statuses']['pass']==70
145 actual
=next(c
for c
in result
['variantReifierOriginals'] if c
['id']==TARGET
);c
=actual
['condition']
146 if cls
:assert c
['class']==cls
and c
['code']==code
,(mode
,c
)
147 else:assert c
is None and actual
['firstDifferentByte']==0
148 fault_controls
.append({'label':label
,'retained':False,'target':TARGET
,'mode':mode
,'actualOutcome':outcome
,'condition':c
,
149 'caseStatus':failed
['status'],'retainedOriginalPASS':70,'expectedASDFFailureObserved':True,'faultCountedAsPass':False})
151 for old
in sorted((PRIOR
/'retained').glob('*.json')):
152 assert old
.read_bytes()==(retained
/old
.name
).read_bytes(),old
.name
153 unchanged
.append({'previousReport':str(old
.relative_to(ROOT
)),'freshReport':'retained/'+old
.name
,'sha256':sha(old
),'byteIdentical':True})
154 for suffix
in ['validation','cxtm','encoding']:
155 name
=f
'ctm.{suffix}.json';assert (PRIOR
/name
).read_bytes()==(output
/name
).read_bytes()
156 unchanged
.append({'previousReport':str((PRIOR
/name
).relative_to(ROOT
)),'freshReport':name
,'sha256':sha(output
/name
),'byteIdentical':True})
157 totals
=read(PRIOR
/'execution.json')['retainedChecks'];stages
=read(PRIOR
/'execution.json')['stageChecks']
158 for marker
,count
in dict(totals
,**stages
).items():assert int(re
.search(r
'(?m)^'+marker
+r
'-PASS: (\d+)',regression
).group(1))==count
159 assert sum(totals
.values())==12100 and 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
160 assert read(output
/'ctm.validation.json')['statuses']==CTM
161 new
=xtm
['variantReifierOriginals'][-1];source
=ROOT
/'vendor/cxtm'/TARGET
;baseline
=ROOT
/'vendor/cxtm/xtm2/baseline/variant-duplicate-reifier2.xtm.cxtm'
162 actual
=new
['actualCXTM'].encode('utf-8');raw
=source
.read_bytes();expected
=baseline
.read_bytes()
163 assert raw
==bytes
.fromhex(new
['sourceHex'])==new
['sourceText'].encode('utf-8') and len(raw
)==new
['sourceBytes']==543
164 assert sha(source
)=='87b740c6ec7d11b485b223cb5c28807bce9259d04c2d586ea557e6cd10c3faf4'
165 assert actual
==expected
==new
['expectedCXTM'].encode('utf-8') and len(actual
)==new
['actualBytes']==new
['expectedBytes']==702
166 assert sha(baseline
)==hashlib
.sha256(actual
).hexdigest()=='014773fca95a2b7bb70038f8e118d402d311a2804f54f105ff05ee53c7eb0a60'
167 assert new
['firstDifferentByte'] is None and new
['condition'] is None and new
['status']=='pass'
168 assert new
['documentIRI']==new
['model']['identity']['identity']['nameModel']['identity']['topics'][0]['itemIdentifiers'][0].removesuffix('#topic')
169 assert new
['sourceKey']==TARGET
and new
['primaryQualifiedParserCalls']==1
170 assert new
['model']==read(PRIOR
/'further-variant-reification.json')['model']
171 assert new
['nativeHandleObservation']=={'sourceNameCalls':2,'sameRawNameHandleEQ':True,'preNormalizationPairs':1,'distinctRetainedVariantHandles':2,
172 'retainedNameCanonicalEQ':True,'retainedVariantCanonicalEQ':True,'variantReplacementObserved':True,'canonicalParentScopeReifierInverseEQ':True}
173 new
=dict(new
,sourceSHA256
=sha(source
),baselineSHA256
=sha(baseline
),actualSHA256
=hashlib
.sha256(actual
).hexdigest(),byteExact
=True,executedInASDFTestOp
=True)
174 (output
/'actual').mkdir();(output
/'actual/variant-duplicate-reifier2.xtm.cxtm').write_bytes(actual
)
175 save(output
/'original-contract.json',new
)
176 save(output
/'diagnostic-changes.json',{'readerObservations':124,'changed':0,'byteIdenticalToSlice22A':(output
/'after-diagnostics.json').read_bytes()==(PRIOR
/'after-diagnostics.json').read_bytes(),
177 'beforeSHA256':sha(PRIOR
/'after-diagnostics.json'),'afterSHA256':sha(output
/'after-diagnostics.json'),'changes':[]})
178 save(output
/'retained-reports.json',unchanged
);save(output
/'retained-fault-reports.json',retained_faults
)
179 save(output
/'failure-controls.json',{'kind':'executor-fault-controls-not-corpus-results','controls':fault_controls
,'retained':103,'new':5,'allExpectedFailureExitsObserved':True})
180 matrix
=read(ROOT
/'reports/slice-22a/matrix.json');rows
=[]
181 for c
in matrix
['cases']:
182 after
='pass' if c
['id']==TARGET
else c
['registeredStatus']
183 rows
.append({'id':c
['id'],'before':c
['registeredStatus'],'after':after
,'changed':after
!=c
['registeredStatus'],
184 'readerConditionUnchanged':True,'newCapability':False,'newNormativeDecision':False})
185 assert sum(c
['changed'] for c
in rows
)==1
186 save(output
/'closure-delta.json',{'kind':'slice-22b-delta-to-immutable-slice-22a-matrix','sourceMatrix':'reports/slice-22a/matrix.json',
187 'sourceMatrixSHA256':sha(ROOT
/'reports/slice-22a/matrix.json'),'changedCase':TARGET
,'newPASS':1,'otherOpenCasesUnchanged':64,'remainingOpenCases':64,
188 'newCapability':False,'newNormativeDecision':False,'cases':rows
})
189 lock
=read(ROOT
/'references/corpus-lock.json')
190 for entry
in lock
['files']:
191 p
=ROOT
/'vendor/cxtm'/entry
['path'];assert p
.stat().st_size
==entry
['size'] and sha(p
)==entry
['sha256']
192 for group
in [protected
,sources
]:assert group
=={p
:sha(ROOT
/p
) for p
in group
}
193 assert set(git('diff','--name-only',BASE
).splitlines()) & set(tracked
)<=ALLOWED
194 save(output
/'integrity.json',{'baseCommit':BASE
,'protectedBaseFilesSHA256':protected
,'executionSourcesSHA256':sources
,
195 'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),'pinnedCorpusFilesVerified':len(lock
['files']),
196 'productionSourcesTestsExceptThreeRegistrationFilesAndHistoricalEvidenceUnchanged':True})
197 before
={'XTM':{'pass':70,'fail':0,'not-run':54},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':332,'fail':4,'not-run':61}}
198 after
={'XTM':{'pass':71,'fail':0,'not-run':53},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':333,'fail':4,'not-run':60}}
199 save(output
/'execution.json',{'kind':'slice-22b-one-positive-original-registration','baseCommit':BASE
,'branch':git('branch','--show-current'),
200 'runtime':'SBCL 2.5.4','startedAtUTC':started
,'finishedAtUTC':datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),'runs':runs
,
201 'baselineBehavioralReportsReproduced':154,'retainedLispChecks':12100,'retainedChecks':totals
,'stageChecks':stages
,'xmlQualificationChecks':146,
202 'standaloneXTMChecks':len(labels
),'retainedXTMChecks':len(previous
['checks']),'newXTMChecks':12,'retainedOriginals':70,'retainedNegativeOriginals':8,
203 'newPositiveOriginals':1,'faultControls':108,'newFaultControls':5,'diagnosticOriginalObservations':124,'diagnosticChanges':0,
204 'statusesBefore':before
,'statusesAfter':after
,'pythonUnitTests':2,'nfcOracleComparisons':16294,'retainedNonXTMReportsByteIdentical':len(unchanged
),
205 'knownCTMFailuresUnchanged':[c
['id'] for c
in read(output
/'ctm.validation.json')['cases'] if c
['status']=='fail'],
206 'pinnedCorpusFilesVerified':len(lock
['files']),'historicalReportFilesVerified':sum(p
.startswith('reports/') for p
in protected
),
207 'productionChanges':[],'historicalReportsUnchanged':True,'originalSourcesAndBaselinesUnchanged':True,'hyperdocLoadedOrChanged':False,
208 'newDependencyDownloads':0,'persistentLispImagesTouched':False,'acceptancePassed':True})
209 print(json
.dumps({'acceptancePassed':True,'statuses':after
,'xtmChecks':len(labels
),'faultControls':108,'output':str(output
)}))
211 if __name__
=='__main__':main()