2 """Run Name Item Identity acceptance through the existing ASDF harness, offline.
4 All model construction, conformance classification and original execution use
5 existing Lisp functions. This driver verifies their reports and retains evidence.
13 from pathlib
import Path
20 sys
.dont_write_bytecode
= True
21 ROOT
= Path(__file__
).resolve().parents
[1]
22 EVIDENCE
= ROOT
/ 'reports/slice-21f'
23 PRIOR
= ROOT
/ 'reports/slice-21e'
24 BASE
= '5d3ed0455de32b9d19ac791b9e4c4882a03c79b8'
25 POLICY
= '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT
) + '"))'
26 ALLOWED
= {'src/xtm/reader.lisp', 'tests/xtm-identity.lisp', 'tests/xtm-names.lisp',
27 'tests/xtm-negative-originals.lisp', 'tests/ctm-xtm-mergemap.lisp', 'tests/xtm-name-scopes.lisp', 'topic-maps.asd'}
28 CTM
= {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
29 EXCLUDED
= ['reifier-collision.xtm', 'role-duplicate-reified.xtm',
30 'topicref-no-fragment-id.xtm', 'variant-missing-scope-duplicate.xtm']
34 return json
.loads(p
.read_text())
38 p
.write_text(json
.dumps(v
, indent
=2, ensure_ascii
=False) + '\n')
42 return hashlib
.sha256(p
.read_bytes()).hexdigest()
46 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True).strip()
49 def prepare_pinned(target
):
50 spec
= importlib
.util
.spec_from_file_location('slice21e', ROOT
/ 'scripts/validate-slice-21e.py')
51 module
= importlib
.util
.module_from_spec(spec
)
52 spec
.loader
.exec_module(module
)
53 return module
.prepare_pinned(target
)
57 if isinstance(v
, dict):
58 return {k
: qualify_guard(x
) for k
, x
in v
.items()}
59 if isinstance(v
, list):
60 return [qualify_guard(x
) for x
in v
]
61 if isinstance(v
, str) and (v
== 'unsupported-name-item-identity' or v
.startswith('unsupported-name-item-identity:')):
62 return v
.replace('unsupported-name-item-identity', 'unsupported-name-variant', 1)
67 parser
= argparse
.ArgumentParser(description
=__doc__
)
68 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
69 parser
.add_argument('--output', type=Path
)
70 parser
.add_argument('--prepare', type=Path
)
71 args
= parser
.parse_args()
73 print(json
.dumps(prepare_pinned(args
.prepare
)))
75 runtime
= shutil
.which(args
.sbcl
)
76 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
77 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
78 output
= args
.output
.resolve() if args
.output
else Path(tempfile
.mkdtemp(prefix
='topic-maps-21f-'))
79 assert not output
.is_relative_to(ROOT
)
80 output
.mkdir(parents
=True, exist_ok
=True)
81 assert not (output
/ 'execution.json').exists(), 'Choose an unused output directory.'
82 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
83 assert set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
) <= ALLOWED
84 protected
= {p
: sha(ROOT
/ p
) for p
in tracked
if p
not in ALLOWED
}
85 historical
= {p
: h
for p
, h
in protected
.items() if p
.startswith('reports/')}
86 source_paths
= [*sorted(ALLOWED
), 'tests/xtm-name-item-identities.lisp', 'scripts/validate-slice-21f.py',
87 *['reports/slice-21f/' + p
for p
in ['candidate-inventory.json', 'failure-controls.lisp',
88 'diagnostics.lisp', 'before-diagnostics.json', 'baseline-xtm.json', 'model-preflight.lisp', 'model-preflight.json']]]
89 sources
= {p
: sha(ROOT
/ p
) for p
in source_paths
}
90 previous
= read(PRIOR
/ 'xtm.json')
91 assert read(EVIDENCE
/ 'baseline-xtm.json') == previous
92 assert previous
['statuses'] == {'pass': 48, 'fail': 0, 'unsupported': 0, 'not-run': 0}
93 assert previous
['otherCorpusInputs'] == {'positiveUnexecuted': 72, 'negativeUnregisteredUnexecuted': 4}
94 assert read(PRIOR
/ 'ctm.validation.json')['statuses'] == CTM
95 candidates
= read(EVIDENCE
/ 'candidate-inventory.json')['candidates']
96 ids
= [c
['id'] for c
in candidates
]
97 assert ids
== ['xtm2/in/itemid-name.xtm', 'xtm2/in/name-duplicate-iid.xtm']
98 assert len(set(ids
)) == 2 and not set(ids
) & {c
['id'] for c
in previous
['cases']}
99 assert read(EVIDENCE
/ 'model-preflight.json')['allSixMergesPassed']
100 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
103 def run(label
, invocation
, env
, expected_exit
=0):
104 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
105 entry
= {'label': label
, 'command': invocation
, 'network': 'denied', 'repositoryWrites': 'denied',
106 'freshProcess': True, 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
107 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
108 stderr
=subprocess
.STDOUT
, timeout
=300)
109 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
110 entry
.update(exitCode
=result
.returncode
, expectedExitCode
=expected_exit
,
111 finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
113 assert result
.returncode
== expected_exit
, f
'{label}: see {output / (label + ".log")}'
114 return result
.stdout
.decode()
116 retained
= output
/ 'retained'
118 env
= {k
: v
for k
, v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
119 env
.update(PYTHONDONTWRITEBYTECODE
='1', GIT_OPTIONAL_LOCKS
='0', SBCL
=runtime
)
120 with tempfile
.TemporaryDirectory(prefix
='topic-maps-21f-sources-') as temp_name
:
121 provenance
= json
.loads(run('source-preparation', ['python3', str(Path(__file__
)), '--prepare', temp_name
+ '/sources'], env
))
122 save(output
/ 'source-preparation.json', provenance
)
123 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'], TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'),
124 TOPIC_MAPS_XTM_REPORT
=str(output
/ 'xtm.json'), TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
125 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'), TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
126 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
127 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/ 'slice20g'),
128 TOPIC_MAPS_ENCODING_REPORT
=str(output
/ 'ctm'))
129 prefix
= [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script']
130 preflight_env
= dict(env
, TOPIC_MAPS_21F_ROOT
=str(ROOT
), TOPIC_MAPS_21F_PREFLIGHT
=str(output
/ 'model-preflight.json'))
131 run('model-preflight', prefix
+ [str(EVIDENCE
/ 'model-preflight.lisp')], preflight_env
)
132 assert read(output
/ 'model-preflight.json') == read(EVIDENCE
/ 'model-preflight.json')
133 xml
= run('xml', prefix
+ ['tests/cxml-probe.lisp'], env
)
134 xml_labels
= re
.findall(r
'^CXML-CHECK-PASS (.+)$', xml
, re
.M
)
135 assert len(xml_labels
) == len(set(xml_labels
)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
136 xtm_log
= run('xtm', prefix
+ ['scripts/fresh-xtm.lisp'], env
)
137 xtm
= read(output
/ 'xtm.json')
138 labels
= re
.findall(r
'^XTM-CHECK-PASS (.+)$', xtm_log
, re
.M
)
139 assert labels
== xtm
['checks'] and len(labels
) == len(set(labels
))
140 removed
= ['name-syntax:unsupported-itemIdentity:class-code', 'name-syntax:unsupported-itemIdentity:context-position',
141 'name-syntax:scope:name-II-stays-unsupported:class-code', 'name-syntax:scope:name-II-stays-unsupported:context-position']
142 assert {qualify_guard(x
) for x
in previous
['checks'] if x
not in removed
} <= set(labels
)
143 assert f
'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 43 original CXTM PASS; 7 negative originals PASS' in xtm_log
144 assert xtm
['statuses'] == {'pass': 50, 'fail': 0, 'unsupported': 0, 'not-run': 0}
145 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 70, 'negativeUnregisteredUnexecuted': 4}
146 assert xtm
['cases'][:48] == previous
['cases']
147 for field
in ['actualAndExpected', 'negativeOriginals', 'nameOriginals', 'scopeOriginals', 'helperSource']:
148 assert xtm
[field
] == previous
[field
], field
149 assert xtm
['nameControls'] == [c
for c
in previous
['nameControls'] if c
['label'] != 'unsupported-itemIdentity']
150 assert xtm
['scopeControls'] == [c
for c
in previous
['scopeControls'] if c
['label'] != 'scope:name-II-stays-unsupported']
151 assert len(xtm
['cases']) == len({c
['id'] for c
in xtm
['cases']}) == 50
152 assert [c
['id'] for c
in xtm
['cases'][48:]] == ids
and len(xtm
['nameItemIdentityOriginals']) == 2
153 assert all(c
['status'] == 'pass' and c
['actualOutcome'] == 'cxtm-match' for c
in xtm
['cases'][48:])
155 assert len(previous
['negativeControls']) == len(xtm
['negativeControls'])
156 for old
, current
in zip(previous
['negativeControls'], xtm
['negativeControls']):
157 if old
['label'] != 'unsupported-is-not-pass':
158 assert old
== current
160 assert old
['label'] == current
['label'] and old
['status'] == current
['status'] == 'unsupported'
161 a
, b
= old
['condition'], current
['condition']
162 assert a
['class'] == b
['class'] == 'XTM-UNSUPPORTED' and a
['code'] == b
['code'] == 'unsupported-construct'
163 assert a
['detail'] == 'XTM itemIdentity is not implemented in this subset.'
164 assert b
['detail'] == 'XTM variant is not implemented in this subset.'
165 assert {k
: v
for k
, v
in a
.items() if k
not in ['detail', 'message']} == {k
: v
for k
, v
in b
.items() if k
not in ['detail', 'message']}
166 guard_changes
.append({'before': old
, 'after': current
, 'reason': 'Valid Name Variant replaces the now supported Name Item Identity as Unsupported guard.'})
167 assert len(xtm
['nameItemIdentityControls']) == 46
168 regression
= run('regression', [str(ROOT
/ 'scripts/test.sh')], env
)
169 diag_env
= dict(env
, TOPIC_MAPS_21F_ROOT
=str(ROOT
), TOPIC_MAPS_21F_DIAGNOSTICS
=str(output
/ 'after-diagnostics.json'))
170 run('diagnostics', prefix
+ [str(EVIDENCE
/ 'diagnostics.lisp')], diag_env
)
172 families
= {'positive': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('xtm', 'execution-error'), ('tmdm', 'execution-error'), ('unsupported', 'unsupported')],
173 'negative': [('accepted', 'ctm-acceptance'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('wrong-cause', 'execution-error')],
174 'name': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')],
175 'scope': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')],
176 'iid': [('mismatch', 'cxtm-mismatch'), ('xml', 'execution-error'), ('unsupported', 'unsupported'), ('tmdm', 'execution-error')]}
177 for family
, modes
in families
.items():
178 for mode
, outcome
in modes
:
179 label
= family
+ '-control-' + mode
180 control_env
= dict(env
, TOPIC_MAPS_XTM_REPORT
=str(output
/ (label
+ '.json')),
181 TOPIC_MAPS_21F_FAMILY
=family
, TOPIC_MAPS_21F_FAULT
=mode
)
182 run(label
, prefix
+ [str(EVIDENCE
/ 'failure-controls.lisp')], control_env
, expected_exit
=1)
183 result
= read(output
/ (label
+ '.json'))
184 retained_count
= {'positive': 0, 'negative': 26, 'name': 33, 'scope': 43, 'iid': 48}[family
]
185 assert result
['statuses']['pass'] == retained_count
186 assert len(result
['cases']) == retained_count
+ 1 and result
['cases'][-1]['actualOutcome'] == outcome
187 assert result
['cases'][:retained_count
] == previous
['cases'][:retained_count
]
188 fault_controls
.append({'family': family
, 'mode': mode
, 'actualOutcome': outcome
,
189 'expectedASDFFailureObserved': True, 'retainedOriginalPasses': retained_count
, 'faultCountedAsPass': False})
192 for old
in sorted((PRIOR
/ 'retained').glob('*.json')):
193 assert qualify_guard(read(old
)) == read(retained
/ old
.name
), str(old
)
194 verified
.append(str(old
.relative_to(ROOT
)))
195 for suffix
in ['validation', 'cxtm', 'encoding']:
196 old
= PRIOR
/ f
'ctm.{suffix}.json'
197 assert qualify_guard(read(old
)) == read(output
/ f
'ctm.{suffix}.json'), str(old
)
198 verified
.append(str(old
.relative_to(ROOT
)))
199 totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
200 assert sum(totals
.values()) == 12100
201 stages
= read(PRIOR
/ 'execution.json')['stageChecks']
202 for marker
, count
in dict(totals
, **stages
).items():
203 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
204 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
205 assert read(output
/ 'ctm.validation.json')['statuses'] == CTM
207 (output
/ 'actual').mkdir()
208 for old
, result
in zip(candidates
, xtm
['nameItemIdentityOriginals']):
209 assert old
['id'] == result
['id'] and result
['condition'] is None and result
['status'] == 'pass'
210 source
= ROOT
/ 'vendor/cxtm' / old
['id']
211 baseline
= ROOT
/ 'vendor/cxtm/xtm2/baseline' / (source
.name
+ '.cxtm')
212 raw
= source
.read_bytes()
213 assert raw
== bytes
.fromhex(result
['sourceHex']) == result
['sourceText'].encode('utf-8')
214 assert sha(source
) == old
['sourceSHA256'] and len(raw
) == result
['sourceBytes'] == old
['sourceBytes']
215 actual
= result
['actualCXTM'].encode('utf-8')
216 assert actual
== result
['expectedCXTM'].encode('utf-8') == baseline
.read_bytes()
217 assert len(actual
) == result
['actualBytes'] == result
['expectedBytes'] == old
['baselineBytes']
218 assert sha(baseline
) == old
['baselineSHA256'] and result
['firstDifferentByte'] is None
219 assert result
['model']['canonicalNameParentTypeReferences'] and result
['documentIRI'] == old
['documentIRI']
220 assert result
['sourceKey'] == result
['id']
221 (output
/ 'actual' / baseline
.name
).write_bytes(actual
)
222 comparisons
.append(dict(result
, sourceSHA256
=sha(source
), baselineSHA256
=sha(baseline
),
223 actualSHA256
=hashlib
.sha256(actual
).hexdigest(), executedInASDFTestOp
=True))
224 before
= read(EVIDENCE
/ 'before-diagnostics.json')
225 after
= read(output
/ 'after-diagnostics.json')
226 assert len(before
) == len(after
) == 124
228 for a
, b
in zip(before
, after
):
229 assert a
['id'] == b
['id'] and a
['documentIRI'] == b
['documentIRI']
232 assert a
['condition']['class'] == 'XTM-UNSUPPORTED'
233 assert a
['condition']['detail'] == 'XTM itemIdentity is not implemented in this subset.'
234 if b
['status'] == 'deserialized':
235 assert b
['id'] in ids
and b
['byteExact']
237 assert b
['condition']['class'] == 'XTM-UNSUPPORTED' and b
['condition']['code'] == 'unsupported-construct'
238 changes
.append({'id': a
['id'], 'before': a
, 'after': b
,
239 'warrant': 'A formerly Unsupported Name Item Identity is processed; excluded features remain Unsupported and native model conflicts remain distinct.'})
240 assert {b
['id'] for a
, b
in zip(before
, after
) if a
['status'] != 'deserialized' and b
['status'] == 'deserialized'} == set(ids
)
241 pending_negative
= sorted('xtm2/invalid/' + p
.name
for p
in (ROOT
/ 'vendor/cxtm/xtm2/invalid').glob('*.xtm')
242 if 'xtm2/invalid/' + p
.name
not in {c
['id'] for c
in xtm
['negativeOriginals']})
243 assert pending_negative
== ['xtm2/invalid/' + name
for name
in EXCLUDED
]
244 assert not set(pending_negative
) & {c
['id'] for c
in xtm
['cases']}
245 lock
= read(ROOT
/ 'references/corpus-lock.json')
246 for entry
in lock
['files']:
247 p
= ROOT
/ 'vendor/cxtm' / entry
['path']
248 assert p
.stat().st_size
== entry
['size'] and sha(p
) == entry
['sha256']
249 for snapshot
in [protected
, historical
, sources
]:
250 assert snapshot
== {p
: sha(ROOT
/ p
) for p
in snapshot
}
251 assert set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
) <= ALLOWED
252 save(output
/ 'name-item-identity-originals.json', {'kind': 'executed-name-item-identity-originals', 'cases': comparisons
,
253 'excludedNegativeOriginals': pending_negative
})
254 save(output
/ 'diagnostic-changes.json', {'kind': 'observations-no-additional-registration',
255 'changes': changes
, 'unchangedOriginalObservations': 124 - len(changes
),
256 'newByteExactOriginals': ids
, 'counterprobeFixtureChanges': guard_changes
})
257 save(output
/ 'failure-controls.json', {'kind': 'executor-fault-controls-not-corpus-results', 'controls': fault_controls
})
258 save(output
/ 'integrity.json', {'protectedBaseFilesSHA256': protected
, 'historicalReportsSHA256': historical
,
259 'executionSourcesSHA256': sources
})
260 statuses
= dict(xtm
['statuses'], **{'not-run': sum(xtm
['otherCorpusInputs'].values())})
261 save(output
/ 'execution.json', {'kind': 'xtm-name-item-identity-registration-validation', 'baseCommit': BASE
,
262 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started
,
263 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(), 'runs': runs
,
264 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals
,
265 'stageChecks': stages
, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels
),
266 'retainedXTMCheckNamesPreserved': len(set(previous
['checks']) & set(labels
)),
267 'retainedXTMChecksQualified': len(previous
['checks']) - len(removed
),
268 'removedItemIdentityUnsupportedChecksReplacedByPositiveCoverage': removed
,
269 'renamedXTMChecks': [{'before': x
, 'after': qualify_guard(x
)} for x
in previous
['checks'] if x
!= qualify_guard(x
)],
270 'retainedOriginals': 48, 'retainedOriginalResultsUnchanged': True, 'newNameItemIdentityOriginals': 2,
271 'nameItemIdentityCounterprobes': len(xtm
['nameItemIdentityControls']),
272 'nameItemIdentityCounterprobeKinds': {k
: sum(c
['kind'] == k
for c
in xtm
['nameItemIdentityControls'])
273 for k
in sorted({c
['kind'] for c
in xtm
['nameItemIdentityControls']})},
274 'faultControls': len(fault_controls
), 'diagnosticOriginalObservations': 124,
275 'itemIdentityRelatedDiagnosticChanges': len(changes
),
276 'xtmStatusesBefore': {'pass': 48, 'fail': 0, 'unsupported': 0, 'not-run': 76}, 'xtmStatusesAfter': statuses
,
277 'xtmStatusChanges': [{'id': p
, 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for p
in ids
],
278 'ctmStatusesBefore': CTM
, 'ctmStatusesAfter': CTM
, 'ctmStatusChanges': [],
279 'knownFailuresUnchanged': [c
['id'] for c
in read(output
/ 'ctm.validation.json')['cases'] if c
['status'] == 'fail'],
280 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
281 'allRetainedNonXTMReportsQualifiedAgainstSlice21E': verified
,
282 'retainedReportQualification': 'Only unsupported-name-item-identity -> unsupported-name-variant fixture/check labels change.',
283 'historicalReportFilesVerified': len(historical
), 'protectedBaseFilesVerified': len(protected
),
284 'pinnedCorpusFilesVerified': len(lock
['files']), 'historicalReportsUnchanged': True,
285 'originalSourcesAndBaselinesUnchanged': True, 'productionChangesLimitedToXTMReader': True,
286 'TMDM_CTM_CXTM_XML_Unchanged': True, 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
287 'executionAuthorization': 'Regular require_escalated approval; all tests additionally deny network and repository writes; no symlink/configuration changes.',
288 'fullOfflineNixRebuild': 'NOT-RUN; fresh SBCL loads with verified pinned local sources and existing XML patches.',
289 'acceptancePassed': True})
290 print(json
.dumps({'acceptancePassed': True, 'xtm': statuses
, 'ctm': CTM
, 'checks': len(labels
), 'output': str(output
)}))
293 if __name__
== '__main__':