2 """Execute the existing ASDF harness and offline regressions; retain byte evidence.
4 No alternate deserializer, model builder, or conformance classifier lives here.
5 Reports are produced by the existing Lisp harness. Use a new temporary output
6 folder; copy accepted evidence into version control only after this exits zero.
15 from pathlib
import Path
22 sys
.dont_write_bytecode
= True
23 ROOT
= Path(__file__
).resolve().parents
[1]
24 EVIDENCE
= ROOT
/ 'reports/slice-21b'
25 BASE
= '1a2808b2d123184b2494f93b19738a31fc27fd52'
26 POLICY
= '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str(ROOT
) + '"))'
27 ALLOWED_TRACKED
= {'tests/xtm-identity.lisp', 'topic-maps.asd'}
31 return json
.loads(path
.read_text())
34 def save(path
, value
):
35 path
.write_text(json
.dumps(value
, indent
=2, ensure_ascii
=False) + '\n')
39 return hashlib
.sha256(path
.read_bytes()).hexdigest()
43 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True).strip()
46 def prepare_pinned(target
):
47 prior
= read(ROOT
/ 'reports/slice-20h/source-preparation.json')
48 spec
= importlib
.util
.spec_from_file_location('prepare_cxml', ROOT
/ 'scripts/prepare-cxml.py')
49 module
= importlib
.util
.module_from_spec(spec
)
50 spec
.loader
.exec_module(module
)
51 return module
.prepare({'manifest': prior
['sources'],
52 'localPatches': [{'package': p
['package'], 'sha256': p
['sha256']} for p
in prior
['patches']]}, target
)
56 parser
= argparse
.ArgumentParser(description
=__doc__
)
57 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
58 parser
.add_argument('--output', type=Path
)
59 parser
.add_argument('--prepare', type=Path
)
60 args
= parser
.parse_args()
62 print(json
.dumps(prepare_pinned(args
.prepare
)))
64 runtime
= shutil
.which(args
.sbcl
)
65 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
66 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
67 output
= args
.output
.resolve() if args
.output
else Path(tempfile
.mkdtemp(prefix
='topic-maps-21b-'))
68 assert not output
.is_relative_to(ROOT
), 'Run tests with repository writes denied; choose a temporary output directory.'
69 output
.mkdir(parents
=True, exist_ok
=True)
70 assert not (output
/ 'execution.json').exists(), 'Choose an unused output directory.'
71 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
72 changed
= set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
)
73 assert changed
<= ALLOWED_TRACKED
74 protected
= {p
: sha(ROOT
/ p
) for p
in tracked
if p
not in ALLOWED_TRACKED
}
75 historical
= {p
: h
for p
, h
in protected
.items() if p
.startswith('reports/')}
76 sources
= {p
: sha(ROOT
/ p
) for p
in [*sorted(ALLOWED_TRACKED
), 'scripts/validate-slice-21b.py',
77 'reports/slice-21b/candidate-inventory.json', 'reports/slice-21b/failure-controls.lisp']}
78 candidates
= read(EVIDENCE
/ 'candidate-inventory.json')['candidates']
79 assert len(candidates
) == len({c
['id'] for c
in candidates
}) == 20
80 assert collections
.Counter(c
['group'] for c
in candidates
) == {
81 'item-identifier': 4, 'merging': 4, 'subject-identifier': 7, 'subject-locator': 5}
82 prior
= ROOT
/ 'reports/slice-20j'
83 baseline
= read(EVIDENCE
/ 'baseline-xtm.json')
84 assert baseline
== read(prior
/ 'retained/xtm.json')
85 assert baseline
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
86 assert baseline
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
87 expected_ctm
= {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
88 assert read(prior
/ 'current.validation.json')['statuses'] == expected_ctm
89 assert not {c
['id'] for c
in baseline
['cases']} & {c
['id'] for c
in candidates
}
90 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
93 def run(label
, invocation
, env
, expected_exit
=0):
94 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
95 entry
= {'label': label
, 'command': invocation
, 'network': 'denied', 'repositoryWrites': 'denied',
96 'freshProcess': True, 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
97 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
, timeout
=300)
98 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
99 entry
.update(exitCode
=result
.returncode
, expectedExitCode
=expected_exit
,
100 finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
102 assert result
.returncode
== expected_exit
, f
'{label}: see {output / (label + ".log")}'
103 return result
.stdout
.decode()
105 retained
= output
/ 'retained'
107 env
= {k
: v
for k
, v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
108 env
.update(PYTHONDONTWRITEBYTECODE
='1', GIT_OPTIONAL_LOCKS
='0', SBCL
=runtime
)
109 with tempfile
.TemporaryDirectory(prefix
='topic-maps-21b-sources-') as temp_name
:
110 provenance
= json
.loads(run('source-preparation', ['python3', str(Path(__file__
)), '--prepare', temp_name
+ '/sources'], env
))
111 save(output
/ 'source-preparation.json', provenance
)
112 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],
113 TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'), TOPIC_MAPS_XTM_REPORT
=str(output
/ 'xtm.json'),
114 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
115 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'),
116 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
117 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
118 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/ 'slice20g'),
119 TOPIC_MAPS_ENCODING_REPORT
=str(output
/ 'ctm'))
120 prefix
= [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script']
121 xml
= run('xml', prefix
+ ['tests/cxml-probe.lisp'], env
)
122 labels
= re
.findall(r
'^CXML-CHECK-PASS (.+)$', xml
, re
.M
)
123 assert len(labels
) == len(set(labels
)) == 146 and 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
124 xtm_log
= run('xtm', prefix
+ ['scripts/fresh-xtm.lisp'], env
)
125 xtm
= read(output
/ 'xtm.json')
126 labels
= re
.findall(r
'^XTM-CHECK-PASS (.+)$', xtm_log
, re
.M
)
127 assert labels
== xtm
['checks'] and len(labels
) == len(set(labels
))
128 assert set(baseline
['checks']) <= set(labels
), 'A retained XTM check disappeared'
129 assert f
'XTM-SUMMARY {len(labels)} PASS 0 FAIL; 26 original CXTM PASS' in xtm_log
130 assert xtm
['statuses'] == {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 0}
131 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 87, 'negativeUnregisteredUnexecuted': 11}
132 assert xtm
['cases'][:6] == baseline
['cases']
133 assert len({c
['id'] for c
in xtm
['cases']}) == len(xtm
['cases']) == 26
134 assert [c
['id'] for c
in xtm
['cases'][6:]] == [c
['id'] for c
in candidates
]
135 assert all(c
['status'] == 'pass' and c
['actualOutcome'] == 'cxtm-match' for c
in xtm
['cases'])
136 assert xtm
['helperSource'] == baseline
['helperSource']
137 regression
= run('regression', [str(ROOT
/ 'scripts/test.sh')], env
)
139 for mode
, outcome
, condition_class
in [('mismatch', 'cxtm-mismatch', None),
140 ('xml', 'execution-error', 'XTM-XML-ERROR'), ('xtm', 'execution-error', 'XTM-REJECTION'),
141 ('tmdm', 'execution-error', 'OCCURRENCE-IDENTIFIER-ERROR'), ('unsupported', 'unsupported', 'XTM-UNSUPPORTED')]:
142 control_env
= dict(env
, TOPIC_MAPS_XTM_REPORT
=str(output
/ ('control-' + mode
+ '.json')), TOPIC_MAPS_21B_FAULT
=mode
)
143 run('control-' + mode
, prefix
+ [str(EVIDENCE
/ 'failure-controls.lisp')], control_env
, expected_exit
=1)
144 report
= read(output
/ ('control-' + mode
+ '.json'))
145 assert report
['statuses']['pass'] == 0 and len(report
['cases']) == 1
146 assert report
['cases'][0]['actualOutcome'] == outcome
147 condition
= report
['actualAndExpected'][0]['condition']
148 assert (condition
['class'] if condition
else None) == condition_class
149 controls
.append({'mode': mode
, 'expectedASDFFailureObserved': True, 'actualOutcome': outcome
,
150 'condition': condition
, 'countedAsOriginalPass': False})
153 for old
in sorted((prior
/ 'retained').glob('*.json')):
154 if old
.name
== 'xtm.json':
156 assert read(old
) == read(retained
/ old
.name
), str(old
)
157 verified
.append(str(old
.relative_to(ROOT
)))
158 for suffix
in ['validation', 'cxtm', 'encoding']:
159 old
= prior
/ f
'current.{suffix}.json'
160 assert read(old
) == read(output
/ f
'ctm.{suffix}.json'), str(old
)
161 verified
.append(str(old
.relative_to(ROOT
)))
162 assert read(output
/ 'ctm.validation.json')['statuses'] == expected_ctm
163 totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
164 assert sum(totals
.values()) == 12100
165 for marker
, count
in totals
.items():
166 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
167 stages
= {'XTM-MERGEMAP': 217, 'REGISTERED-ORIGINALS': 286, 'REGISTERED-NEGATIVES': 25,
168 'PROLOG': 171, 'EMPTY2': 13, 'CTM-INTEGER': 112, 'TRIPLE': 226, 'ENCODING': 722}
169 for marker
, count
in stages
.items():
170 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', regression
).group(1)) == count
171 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
173 (output
/ 'actual').mkdir()
174 for record
, result
in zip(xtm
['cases'], xtm
['actualAndExpected']):
175 assert record
['id'] == result
['id'] and result
['condition'] is None
176 source
= ROOT
/ 'vendor/cxtm' / record
['id']
177 baseline_path
= ROOT
/ 'vendor/cxtm/xtm2/baseline' / (source
.name
+ '.cxtm')
178 actual
= result
['actual'].encode('utf-8')
179 assert actual
== result
['expected'].encode('utf-8') == baseline_path
.read_bytes()
180 assert result
['byteLength'] == result
['expectedByteLength'] == len(actual
)
181 assert result
['sourceByteLength'] == source
.stat().st_size
and result
['firstDifferentByte'] is None
182 assert result
['model']['canonicalParentsAndInverseReferences']
183 (output
/ 'actual' / baseline_path
.name
).write_bytes(actual
)
184 comparison
= {'id': record
['id'], 'documentIRI': record
['documentIRI'], 'sourceKey': record
['id'],
185 'sourceBytes': source
.stat().st_size
, 'sourceSHA256': sha(source
),
186 'baseline': str(baseline_path
.relative_to(ROOT
/ 'vendor/cxtm')), 'baselineSHA256': sha(baseline_path
),
187 'expectedBytes': len(actual
), 'actualBytes': len(actual
), 'actualSHA256': hashlib
.sha256(actual
).hexdigest(),
188 'firstDifferentByte': None, 'byteExact': True, 'condition': None, 'status': record
['status'],
189 'model': result
['model'], 'executedInASDFTestOp': True,
190 'checks': [c
for c
in labels
if c
.startswith(source
.stem
+ ':')]}
191 if record
['id'] in {c
['id'] for c
in candidates
}:
192 candidate
= next(c
for c
in candidates
if c
['id'] == record
['id'])
193 assert candidate
['sourceSHA256'] == comparison
['sourceSHA256'] and candidate
['sourceBytes'] == comparison
['sourceBytes']
194 assert candidate
['baselineSHA256'] == comparison
['baselineSHA256'] and candidate
['baselineBytes'] == len(actual
)
195 assert candidate
['documentIRI'] == comparison
['documentIRI']
196 comparisons
.append(comparison
)
197 lock
= read(ROOT
/ 'references/corpus-lock.json')
198 for entry
in lock
['files']:
199 p
= ROOT
/ 'vendor/cxtm' / entry
['path']
200 assert p
.stat().st_size
== entry
['size'] and sha(p
) == entry
['sha256']
201 for snapshot
in [protected
, historical
, sources
]:
202 assert snapshot
== {p
: sha(ROOT
/ p
) for p
in snapshot
}
203 assert set(command(['git', 'diff', '--name-only', BASE
]).splitlines()) & set(tracked
) <= ALLOWED_TRACKED
204 save(output
/ 'comparisons.json', {'kind': 'actual-original-cxtm-comparisons', 'cases': comparisons
})
205 save(output
/ 'failure-controls.json', {'kind': 'executor-fault-injection-controls-not-corpus-results', 'controls': controls
})
206 save(output
/ 'integrity.json', {'protectedBaseFilesSHA256': protected
, 'historicalReportsSHA256': historical
,
207 'executionSourcesSHA256': sources
})
208 save(output
/ 'execution.json', {'kind': 'xtm-original-registration-validation', 'baseCommit': BASE
,
209 'branch': command(['git', 'branch', '--show-current']), 'runtime': 'SBCL 2.5.4', 'startedAtUTC': started
,
210 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(), 'runs': runs
,
211 'sourcePreparation': 'source-preparation.json', 'retainedLispChecks': 12100, 'retainedChecks': totals
,
212 'stageChecks': stages
, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': len(labels
),
213 'retainedXTMCheckNamesPreserved': len(baseline
['checks']), 'originalComparisons': 26, 'newOriginalComparisons': 20,
214 'xtmStatusesBefore': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
215 'xtmStatusesAfter': {'pass': 26, 'fail': 0, 'unsupported': 0, 'not-run': 98},
216 'xtmStatusChanges': [{'id': c
['id'], 'before': 'not-run', 'after': 'pass', 'actualOutcome': 'cxtm-match'} for c
in candidates
],
217 'ctmStatusesBefore': expected_ctm
, 'ctmStatusesAfter': expected_ctm
, 'ctmStatusChanges': [],
218 'knownFailuresUnchanged': [c
['id'] for c
in read(output
/ 'ctm.validation.json')['cases'] if c
['status'] == 'fail'],
219 'negativeXTMOriginalsRegistered': 0, 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
220 'faultInjectionControls': controls
, 'allRetainedNonXTMReportsEqualToSlice20J': verified
,
221 'historicalReportFilesVerified': len(historical
), 'protectedBaseFilesVerified': len(protected
),
222 'pinnedCorpusFilesVerified': len(lock
['files']), 'historicalReportsUnchanged': True,
223 'originalSourcesAndBaselinesUnchanged': True, 'languageImplementationUnchanged': True,
224 'newDependencyDownloads': 0, 'persistentLispImagesTouched': False,
225 'fullOfflineNixRebuild': 'NOT-RUN; verified local pinned Store sources and existing qualified XML patches.',
226 'executionAuthorization': 'Regular require_escalated approval after managed sandbox symlink launch error; no symlink or configuration changes.',
227 'acceptancePassed': True})
228 print(json
.dumps({'acceptancePassed': True, 'xtm': {'pass': 26, 'fail': 0, 'not-run': 98},
229 'ctm': expected_ctm
, 'checks': len(labels
), 'output': str(output
)}))
232 if __name__
== '__main__':