2 """Validate cause-qualified negative originals through the existing XTM harness.
4 Use a fresh temporary output directory. No alternate parser/model/classifier is
5 introduced; all results come from ASDF and the existing foundation Case Records.
13 from pathlib
import Path
20 sys
.dont_write_bytecode
= True
21 ROOT
= Path(__file__
).resolve().parents
[1]
22 EVIDENCE
= ROOT
/ 'reports/slice-21c'
23 BASE
= 'd498716b1e30ca1ef2b5c124038b372851ccb731'
24 POLICY
= '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT
) + '"))'
25 ALLOWED_TRACKED
= {'tests/xtm-identity.lisp', 'topic-maps.asd'}
26 EXCLUDED
= ['reifier-collision.xtm', 'role-duplicate-reified.xtm', 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
30 return json
.loads(path
.read_text())
33 def save(path
, value
):
34 path
.write_text(json
.dumps(value
, indent
=2, ensure_ascii
=False) + '\n')
38 return hashlib
.sha256(path
.read_bytes()).hexdigest()
42 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True).strip()
45 def prepare_pinned(target
):
46 # Reuse the existing local Store verifier and patcher, without a Nix daemon
47 # evaluation or dependency download. The historical driver stays unchanged.
48 spec
= importlib
.util
.spec_from_file_location('slice21b', ROOT
/ 'scripts/validate-slice-21b.py')
49 module
= importlib
.util
.module_from_spec(spec
)
50 spec
.loader
.exec_module(module
)
51 return module
.prepare_pinned(target
)
55 parser
= argparse
.ArgumentParser(description
=__doc__
)
56 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
57 parser
.add_argument('--output', type=Path
)
58 parser
.add_argument('--prepare', type=Path
)
59 args
= parser
.parse_args()
61 print(json
.dumps(prepare_pinned(args
.prepare
)))
63 runtime
= shutil
.which(args
.sbcl
)
64 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
65 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
66 output
= args
.output
.resolve() if args
.output
else Path(tempfile
.mkdtemp(prefix
='topic-maps-21c-'))
67 assert not output
.is_relative_to(ROOT
), 'Choose a temporary output directory outside the repository.'
68 output
.mkdir(parents
=True, exist_ok
=True)
69 assert not (output
/ 'execution.json').exists(), 'Choose an unused output directory.'
70 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
71 assert set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
) <= ALLOWED_TRACKED
72 protected
= {p
: sha(ROOT
/ p
) for p
in tracked
if p
not in ALLOWED_TRACKED
}
73 historical
= {p
: h
for p
, h
in protected
.items() if p
.startswith('reports/')}
74 sources
= {p
: sha(ROOT
/ p
) for p
in [*sorted(ALLOWED_TRACKED
), 'tests/xtm-negative-originals.lisp',
75 'scripts/validate-slice-21c.py', 'reports/slice-21c/candidate-inventory.json', 'reports/slice-21c/failure-controls.lisp']}
76 candidates
= read(EVIDENCE
/ 'candidate-inventory.json')['cases']
77 ids
= [c
['id'] for c
in candidates
]
78 assert len(ids
) == len(set(ids
)) == 7
79 assert ids
== ['xtm2/invalid/' + name
+ '.xtm' for name
in ['id-invalid', 'itemid-collision', 'no-version',
80 'reifier-elem-in-2.0', 'subjid-ref-in-2.0', 'subjloc-ref-in-2.0', 'topic-no-id']]
81 prior
= ROOT
/ 'reports/slice-21b'
82 previous_xtm
= read(prior
/ 'xtm.json')
83 assert previous_xtm
['statuses'] == {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 0}
84 assert previous_xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 11}
85 ctm_statuses
= {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
86 assert read(prior
/ 'ctm.validation.json')['statuses'] == ctm_statuses
87 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
90 def run(label
, invocation
, env
, expected_exit
=0):
91 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
92 entry
= {'label': label
, 'command': invocation
, 'network': 'denied', 'repositoryWrites': 'denied',
93 'freshProcess': True, 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
94 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
, timeout
=300)
95 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
96 entry
.update(exitCode
=result
.returncode
, expectedExitCode
=expected_exit
,
97 finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
99 assert result
.returncode
== expected_exit
, f
'{label}: see {output / (label + ".log")}'
100 return result
.stdout
.decode()
102 retained
= output
/ 'retained'
104 env
= {k
: v
for k
, v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
105 env
.update(PYTHONDONTWRITEBYTECODE
='1', GIT_OPTIONAL_LOCKS
='0', SBCL
=runtime
)
106 with tempfile
.TemporaryDirectory(prefix
='topic-maps-21c-sources-') as temp_name
:
107 provenance
= json
.loads(run('source-preparation', ['python3', str(Path(__file__
)), '--prepare', temp_name
+ '/sources'], env
))
108 save(output
/ 'source-preparation.json', provenance
)
109 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'], TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'),
110 TOPIC_MAPS_XTM_REPORT
=str(output
/ 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
111 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
112 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
113 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/ 'slice20g'),
114 TOPIC_MAPS_ENCODING_REPORT
=str(output
/ 'ctm'))
115 prefix
= [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script']
116 xml
= run('xml', prefix
+ ['tests/cxml-probe.lisp'], env
)
117 xml_labels
= re
.findall(r
'^CXML-CHECK-PASS (.+)$', xml
, re
.M
)
118 assert len(xml_labels
) == len(set(xml_labels
)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
119 xtm_log
= run('xtm', prefix
+ ['scripts/fresh-xtm.lisp'], env
)
120 xtm
= read(output
/ 'xtm.json')
121 labels
= re
.findall(r
'^XTM-CHECK-PASS (.+)$', xtm_log
, re
.M
)
122 assert labels
== xtm
['checks'] and len(labels
) == len(set(labels
)) and set(previous_xtm
['checks']) <= set(labels
)
123 assert f
'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 26 original CXTM PASS; 7 negative originals PASS' in xtm_log
124 assert xtm
['statuses'] == {'pass': 33, 'fail': 0, 'unsupported': 0, 'not-run': 0}
125 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 4}
126 assert xtm
['cases'][:26] == previous_xtm
['cases']
127 assert xtm
['actualAndExpected'] == previous_xtm
['actualAndExpected']
128 assert xtm
['helperSource'] == previous_xtm
['helperSource']
129 assert len(xtm
['cases']) == len({c
['id'] for c
in xtm
['cases']}) == 33
130 assert [c
['id'] for c
in xtm
['cases'][26:]] == ids
131 assert all(c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection' for c
in xtm
['cases'][26:])
132 assert len(xtm
['negativeOriginals']) == 7 and len(xtm
['negativeControls']) == 13
133 regression
= run('regression', [str(ROOT
/ 'scripts/test.sh')], env
)
135 # Preserve all five previously established positive-executor guards.
136 for mode
, outcome
in [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'),
137 ('tmdm', 'execution-error'), ('unsupported', 'unsupported')]:
138 control_env
= dict(env
, TOPIC_MAPS_XTM_REPORT
=str(output
/ ('retained-control-' + mode
+ '.json')), TOPIC_MAPS_21B_FAULT
=mode
)
139 run('retained-control-' + mode
, prefix
+ [str(prior
/ 'failure-controls.lisp')], control_env
, expected_exit
=1)
140 result
= read(output
/ ('retained-control-' + mode
+ '.json'))
141 assert result
['statuses']['pass'] == 0 and result
['cases'][0]['actualOutcome'] == outcome
142 for mode
, outcome
, condition_class
in [('accepted', 'ctm-acceptance', None), ('xml', 'execution-error', 'XTM-XML-ERROR'),
143 ('unsupported', 'unsupported', 'XTM-UNSUPPORTED'), ('wrong-cause', 'execution-error', 'XTM-REJECTION')]:
144 control_env
= dict(env
, TOPIC_MAPS_XTM_REPORT
=str(output
/ ('negative-control-' + mode
+ '.json')), TOPIC_MAPS_21C_FAULT
=mode
)
145 run('negative-control-' + mode
, prefix
+ [str(EVIDENCE
/ 'failure-controls.lisp')], control_env
, expected_exit
=1)
146 result
= read(output
/ ('negative-control-' + mode
+ '.json'))
147 assert result
['statuses']['pass'] == 26 and result
['cases'][:26] == previous_xtm
['cases']
148 assert len(result
['cases']) == 27 and result
['cases'][-1]['actualOutcome'] == outcome
149 negative
= result
['negativeOriginals'][0]
150 assert not negative
['intendedCauseReached'] and negative
['status'] != 'pass'
151 condition
= negative
['condition']
152 assert (condition
['class'] if condition
else None) == condition_class
153 fault_controls
.append({'mode': mode
, 'actualOutcome': outcome
, 'condition': condition
,
154 'expectedASDFFailureObserved': True, 'positiveOriginalPassesPreserved': 26, 'negativeOriginalCountedAsPass': False})
157 for old
in sorted((prior
/ 'retained').glob('*.json')):
158 assert read(old
) == read(retained
/ old
.name
), str(old
)
159 verified
.append(str(old
.relative_to(ROOT
)))
160 for suffix
in ['validation', 'cxtm', 'encoding']:
161 old
= prior
/ f
'ctm.{suffix}.json'
162 assert read(old
) == read(output
/ f
'ctm.{suffix}.json'), str(old
)
163 verified
.append(str(old
.relative_to(ROOT
)))
164 totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
165 assert sum(totals
.values()) == 12100
166 stages
= read(prior
/ 'execution.json')['stageChecks']
167 for marker
, count
in dict(totals
, **stages
).items():
168 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
169 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
170 assert read(output
/ 'ctm.validation.json')['statuses'] == ctm_statuses
171 for old
, result
in zip(candidates
, xtm
['negativeOriginals']):
172 assert old
['id'] == result
['id'] and old
['condition'] == result
['condition']
173 source
= ROOT
/ 'vendor/cxtm' / old
['id']
174 raw
= source
.read_bytes()
175 assert raw
== bytes
.fromhex(result
['sourceHex']) == result
['sourceText'].encode('utf-8')
176 assert sha(source
) == old
['sourceSHA256'] and len(raw
) == result
['sourceBytes'] == old
['sourceBytes']
177 assert result
['xmlWellFormed'] and result
['intendedCauseReached'] and result
['status'] == 'pass'
178 assert not result
['unsupported'] and result
['condition']['cause'] is None
179 result
['sourceSHA256'] = sha(source
)
180 for positive
in xtm
['actualAndExpected']:
181 p
= ROOT
/ 'vendor/cxtm/xtm2/baseline' / (Path(positive
['id']).name
+ '.cxtm')
182 assert positive
['actual'].encode('utf-8') == positive
['expected'].encode('utf-8') == p
.read_bytes()
183 pending_negative
= sorted('xtm2/invalid/' + p
.name
for p
in (ROOT
/ 'vendor/cxtm/xtm2/invalid').glob('*.xtm') if 'xtm2/invalid/' + p
.name
not in ids
)
184 assert pending_negative
== ['xtm2/invalid/' + name
for name
in EXCLUDED
]
185 assert not set(pending_negative
) & {c
['id'] for c
in xtm
['cases']}
186 lock
= read(ROOT
/ 'references/corpus-lock.json')
187 for entry
in lock
['files']:
188 p
= ROOT
/ 'vendor/cxtm' / entry
['path']
189 assert p
.stat().st_size
== entry
['size'] and sha(p
) == entry
['sha256']
190 for snapshot
in [protected
, historical
, sources
]:
191 assert snapshot
== {p
: sha(ROOT
/ p
) for p
in snapshot
}
192 assert set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
) <= ALLOWED_TRACKED
193 save(output
/ 'negative-originals.json', {'kind': 'executed-cause-qualified-negative-originals', 'cases': xtm
['negativeOriginals'],
194 'excludedOriginals': [{'id': p
, 'registeredStatus': 'not-run', 'executedInThisSlice': False} for p
in pending_negative
],
195 'outcomeVocabulary': 'ctm-rejection is the existing foundation negative-outcome token; actual deserialization is XTM/TMDM.'})
196 save(output
/ 'failure-controls.json', {'kind': 'negative-executor-fault-controls-not-corpus-results', 'controls': fault_controls
})
197 save(output
/ 'integrity.json', {'protectedBaseFilesSHA256': protected
, 'historicalReportsSHA256': historical
, 'executionSourcesSHA256': sources
})
198 statuses
= dict(xtm
['statuses'], **{'not-run': sum(xtm
['otherCorpusInputs'].values())})
199 save(output
/ 'execution.json', {'kind': 'xtm-negative-original-registration-validation', 'baseCommit': BASE
,
200 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started
,
201 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(), 'runs': runs
, 'sourcePreparation': 'source-preparation.json',
202 'retainedLispChecks': 12100, 'retainedChecks': totals
, 'stageChecks': stages
, 'xmlQualificationChecks': 146,
203 'standaloneXTMChecks': len(labels
), 'retainedXTMCheckNamesPreserved': len(previous_xtm
['checks']),
204 'positiveOriginalComparisons': 26, 'positiveOriginalResultsUnchanged': True, 'newNegativeOriginals': 7,
205 'validNegativeCounterexamples': 7, 'negativeClassificationControls': 6, 'positiveFaultControlsRetained': 5, 'negativeFaultControls': 4,
206 'xtmStatusesBefore': {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 98}, 'xtmStatusesAfter': statuses
,
207 'xtmStatusChanges': [{'id': p
, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'ctm-rejection'} for p
in ids
],
208 'ctmStatusesBefore': ctm_statuses
, 'ctmStatusesAfter': ctm_statuses
, 'ctmStatusChanges': [],
209 'knownFailuresUnchanged': [c
['id'] for c
in read(output
/ 'ctm.validation.json')['cases'] if c
['status'] == 'fail'],
210 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'allRetainedNonXTMReportsEqualToSlice21B': verified
,
211 'historicalReportFilesVerified': len(historical
), 'protectedBaseFilesVerified': len(protected
),
212 'pinnedCorpusFilesVerified': len(lock
['files']), 'historicalReportsUnchanged': True, 'originalSourcesAndBaselinesUnchanged': True,
213 'productionCodeUnchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
214 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
215 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.', 'acceptancePassed': True})
216 print(json
.dumps({'acceptancePassed': True, 'xtm': statuses
, 'ctm': ctm_statuses
, 'checks': len(labels
), 'output': str(output
)}))
219 if __name__
== '__main__':