2 """Validate ISO-8859-5 and the complete retained offline suite in fresh SBCL."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= '7f0e40d1558255773357c5ed4fa44bb5184d85e0'
16 POLICY
= '(version 1)(allow default)(deny network*)'
17 ORIGINAL
= 'ctm/in/encoding-8859-5.ctm'
18 ISO_NAMES
= ['ISO-8859-5', 'ISO_8859-5:1988', 'ISO_8859-5', 'iso-ir-144',
19 'cyrillic', 'csISOLatinCyrillic']
23 spec
= importlib
.util
.spec_from_file_location('slice20a', ROOT
/ 'scripts/validate-slice-20a.py')
24 common
= importlib
.util
.module_from_spec(spec
)
25 spec
.loader
.exec_module(common
)
26 read
, save
, sha
, command
, snapshot
= common
.read
, common
.save
, common
.sha
, common
.command
, common
.snapshot
27 parser
= argparse
.ArgumentParser(description
=__doc__
)
28 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
29 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-20h')
30 parser
.add_argument('--preimplementation', type=Path
, default
=ROOT
/ 'reports/slice-20h/preimplementation.json')
31 parser
.add_argument('--runtime-qualification', type=Path
, default
=ROOT
/ 'reports/slice-20h/runtime-qualification.json')
32 args
= parser
.parse_args()
33 pre
, qualification
= read(args
.preimplementation
), read(args
.runtime_qualification
)
34 oracle
= [ord(c
) for c
in bytes(range(256)).decode('iso8859_5')]
35 assert pre
['baseCommit'] == qualification
['baseCommit'] == BASE
36 assert pre
['runtime'] == qualification
['runtime'] == 'SBCL 2.5.4'
37 assert pre
['network'] == qualification
['network'] == 'denied'
38 assert pre
['original']['id'] == ORIGINAL
and pre
['original']['code'] == 'unsupported-encoding'
39 assert pre
['original']['condition'] == 'CTM-UNSUPPORTED' and pre
['original']['position'] == 11
40 assert pre
['original']['positionUnit'] == 'byte'
41 assert qualification
['all256DecodedWithoutReplacement'] and qualification
['all256EncodeRoundTrip']
42 assert qualification
['pythonCodecOracleEqual'] and qualification
['codepoints'] == oracle
43 runtime
= shutil
.which(args
.sbcl
)
44 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
45 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
46 mutable
= {'src/ctm/unicode.lisp', 'src/ctm/reader.lisp', 'tests/ctm-prolog.lisp',
47 'tests/ctm-integer-literal.lisp', 'tests/ctm-registered-originals.lisp',
48 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
49 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
50 protected_paths
= [p
for p
in tracked
if p
not in mutable
]
51 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
52 old_unicode
= subprocess
.check_output(['git', 'show', BASE
+ ':src/ctm/unicode.lisp'], cwd
=ROOT
).decode()
53 new_unicode
= (ROOT
/ 'src/ctm/unicode.lisp').read_text()
54 # Includes both strict UTF-8 and IRI unescaping, byte for byte.
55 assert old_unicode
[:old_unicode
.index(';; Slice 20C:')] == new_unicode
[:new_unicode
.index(';; Inspect')]
56 output
= args
.output
.resolve()
57 output
.mkdir(parents
=True, exist_ok
=False)
58 retained
= output
/ 'retained'
60 save(output
/ 'preimplementation.json', pre
)
61 save(output
/ 'runtime-qualification.json', qualification
)
62 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
63 protected
= snapshot(ROOT
/ p
for p
in protected_paths
)
64 historical
= snapshot(p
for p
in (ROOT
/ 'reports').rglob('*') if output
not in p
.parents
)
65 sources
= snapshot(p
for directory
in ['src', 'tests', 'scripts', 'docs']
66 for p
in (ROOT
/ directory
).rglob('*') if p
.suffix
in ['.lisp', '.py', '.sh', '.md'])
67 sources
.update(snapshot(ROOT
/ p
for p
in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
69 with tempfile
.TemporaryDirectory(prefix
='topic-maps-20h-') as temp_name
:
70 preparation
= ['python3', 'scripts/prepare-cxml.py', '--directory', str(Path(temp_name
) / 'sources')]
71 provenance
= json
.loads(command(preparation
)) # Pinned local Store sources, nix --offline.
72 save(output
/ 'source-preparation.json', provenance
)
73 env
= dict(os
.environ
, SBCL
=runtime
, GIT_OPTIONAL_LOCKS
='0', PYTHONDONTWRITEBYTECODE
='1',
74 CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],
75 TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'),
76 TOPIC_MAPS_XTM_REPORT
=str(retained
/ 'xtm.json'),
77 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
78 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'),
79 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
80 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'),
81 TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
82 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'),
83 TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/ 'slice20g'),
84 TOPIC_MAPS_ENCODING_REPORT
=str(output
/ 'current'))
85 for label
, invocation
in [
86 ('xml', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
87 ('xtm', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
88 ('regression', [str(ROOT
/ 'scripts/test.sh')]),
90 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
91 run
= {'label': label
, 'command': invocation
, 'freshProcess': True, 'network': 'denied by sandbox-exec',
92 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
93 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
94 stderr
=subprocess
.STDOUT
, timeout
=300)
95 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
96 run
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
98 assert result
.returncode
== 0, f
'{label} failed; see {label}.log'
99 if label
in ['xml', 'xtm']:
100 marker
= 'CXML' if label
== 'xml' else 'XTM'
101 labels
= re
.findall(r
'^' + marker
+ r
'-CHECK-PASS (.+)$', result
.stdout
.decode(), re
.M
)
102 assert len(labels
) == len(set(labels
)) == (146 if label
== 'xml' else 327)
103 assert f
'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result
.stdout
.decode()
104 run
['passedChecks'] = len(labels
)
106 # Only the seven obsolete Unsupported fixtures and their check label may
107 # change. Their complete prior rows are matched before this transformation.
108 old_prolog
= read(ROOT
/ 'reports/slice-20g/retained/slice20c.prolog.json')
109 old_integer
= read(ROOT
/ 'reports/slice-20g/retained/slice20f.literal.json')
110 obsolete
= {r
['label']: r
for r
in old_prolog
['negativeControls'] + old_integer
['negativeControls']
111 if r
['condition']['code'] == 'unsupported-encoding'}
112 assert len(obsolete
) == 7
113 def qualified_retained(value
):
114 if isinstance(value
, dict):
115 if value
.get('label') in obsolete
:
116 assert value
== obsolete
[value
['label']]
117 current
= dict(value
)
118 condition
= dict(value
['condition'])
119 condition
['message'] = condition
['message'].replace(
120 'Only UTF-8 CTM input is implemented.',
121 'Only UTF-8 and ISO-8859-5 CTM input are implemented.').replace(
122 'Charset "iso-8859-5" is recognized but only UTF-8 decoding is implemented.',
123 'Charset "iso-8859-1" is not implemented; supported CTM encodings are UTF-8 and ISO-8859-5.')
124 if value
['label'] == 'iso88595-still-unsupported':
125 current
.update(label
='iso88591-still-unsupported',
126 sourceText
=value
['sourceText'].replace('iso-8859-5', 'iso-8859-1'))
127 condition
['source'] = current
['label']
128 condition
['message'] = condition
['message'].replace(value
['label'], current
['label'])
129 current
['condition'] = condition
131 return {k
: qualified_retained(v
) for k
, v
in value
.items()}
132 if isinstance(value
, list):
133 return [qualified_retained(v
) for v
in value
]
134 if value
== 'negative:iso88595-still-unsupported:class-code-exact-position':
135 return 'negative:iso88591-still-unsupported:class-code-exact-position'
138 for old
in sorted((ROOT
/ 'reports/slice-20g/retained').glob('*.json')):
139 assert qualified_retained(read(old
)) == read(retained
/ old
.name
), str(old
)
140 verified
.append(str(old
.relative_to(ROOT
)))
141 for suffix
in ['validation', 'cxtm', 'triple']:
142 old
= ROOT
/ f
'reports/slice-20g/current.{suffix}.json'
143 assert qualified_retained(read(old
)) == read(retained
/ f
'slice20g.{suffix}.json'), str(old
)
144 verified
.append(str(old
.relative_to(ROOT
)))
145 before
= read(retained
/ 'slice20g.validation.json')
146 final
, encoding
, cxtm
= [read(output
/ f
'current.{name}.json') for name
in ['validation', 'encoding', 'cxtm']]
147 assert before
['statuses'] == encoding
['entryStatuses'] == {'pass': 261, 'fail': 4, 'unsupported': 0, 'not-run': 8}
148 assert final
['statuses'] == {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
149 assert encoding
['passed'] == len(encoding
['checks']) == len(set(encoding
['checks'])) and encoding
['failed'] == 0
150 assert encoding
['decoderCodepoints'] == oracle
== qualification
['codepoints']
151 assert encoding
['ISO88595Names'] == ISO_NAMES
and encoding
['UTF8Names'] == ['UTF-8', 'csUTF8']
152 assert encoding
['selectionEvidence'] == {'conflictAndUnsupportedUTF8Calls': 0, 'conflictAndUnsupportedISOCalls': 0}
153 positives
= {p
['label']: p
for p
in encoding
['positives']}
154 negatives
= {r
['label']: r
for r
in encoding
['negativeControls']}
155 assert len(positives
) == len(encoding
['positives']) == 74
156 assert len(negatives
) == len(encoding
['negativeControls']) == 33
157 for charset
in ['ISO-8859-5', 'csUTF8']:
158 row
= negatives
['source-position:' + charset
]
159 condition
= row
['condition']
160 assert condition
['code'] == 'unexpected-character' and condition
['positionUnit'] == 'character'
161 assert condition
['line'] == 3 and condition
['column'] == 1
162 assert (row
['positionInOriginalBytes'] == condition
['position']) == (charset
== 'ISO-8859-5')
163 for purpose
in ['INCLUDE', 'MERGEMAP']:
164 for generic
in ['NIL', 'T']:
165 row
= positives
[f
'resource:{purpose}:generic={generic}']
166 assert row
['finalCounter'] == (20 if purpose
== 'INCLUDE' else 18)
167 assert 'C0 DE E1 E1 D8 EF' in row
['inputHex']
168 row
= encoding
['original']
169 assert row
['id'] == ORIGINAL
and row
['status'] == 'pass' and row
['actualOutcome'] == 'cxtm-match'
170 assert row
['diagnostics'] == [] and row
['sourceBytes'] == 42
171 source
= ROOT
/ 'vendor/cxtm' / ORIGINAL
172 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline/encoding-8859-5.ctm.cxtm'
173 assert source
.read_bytes() == b
'%encoding "iso-8859-5"\nrussia - "\xc0\xde\xe1\xe1\xd8\xef" .'
174 expected
= baseline
.read_bytes() # Read the actual pinned byte count; never prescribe it.
175 comparison
= row
['comparison']
176 assert comparison
['firstDifferentByte'] is None
177 assert comparison
['actualBytes'] == comparison
['expectedBytes'] == len(expected
)
178 model
= comparison
['model']
179 assert model
['topicCount'] == 2 and model
['associationCount'] == 0
180 names
= [n
for topic
in model
['topics'] for n
in topic
['names']]
181 assert len(names
) == 1 and names
[0]['value'] == 'Россия'
182 assert len(before
['cases']) == len(final
['cases']) == 273
184 for a
, b
in zip(before
['cases'], final
['cases']):
185 assert a
['id'] == b
['id']
186 if a
['id'] != ORIGINAL
:
187 assert a
== b
, a
['id']
189 assert a
['status'] == 'not-run' and a
['actualOutcome'] is None and a
['category'] == 'valid'
190 assert a
['documentIRI'] == row
['documentIRI'] == pre
['original']['documentIRI']
191 assert a
['documentIRI'] == 'https://ctm-tests.example.invalid/' + before
['commit'] + '/' + ORIGINAL
192 assert b
== dict(a
, status
='pass', actualOutcome
='cxtm-match')
193 changes
.append({'id': ORIGINAL
, 'before': a
['status'], 'after': b
['status']})
194 assert len(changes
) == 1
195 assert {k
: v
for k
, v
in before
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
196 k
: v
for k
, v
in final
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
197 old_cxtm
= read(retained
/ 'slice20g.cxtm.json')
198 assert [c
for c
in cxtm
['cases'] if c
['id'] != ORIGINAL
] == old_cxtm
['cases']
199 assert cxtm
['contractCounts'] == {'pass': 224, 'fail': 4}
200 for case
in cxtm
['cases']:
201 pinned
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (Path(case
['id']).name
+ '.cxtm')
202 actual
= case
['actualCXTM'].encode('utf-8')
203 assert (actual
== pinned
.read_bytes()) == (case
['status'] == 'pass')
204 assert case
['expectedCXTM'].encode('utf-8') == pinned
.read_bytes()
205 original_case
= next(c
for c
in cxtm
['cases'] if c
['id'] == ORIGINAL
)
206 actual
= original_case
['actualCXTM'].encode('utf-8')
207 assert actual
== original_case
['expectedCXTM'].encode('utf-8') == expected
208 (output
/ 'actual').mkdir()
209 (output
/ 'actual/encoding-8859-5.ctm.cxtm').write_bytes(actual
)
210 log
= (output
/ 'regression.log').read_text()
211 prior_totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
212 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for marker
in prior_totals
}
213 assert totals
== prior_totals
and sum(totals
.values()) == 12100
214 for marker
, count
in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25),
215 ('PROLOG', 171), ('EMPTY2', 13), ('CTM-INTEGER', 112), ('TRIPLE', 226),
216 ('ENCODING', encoding
['passed'])]:
217 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == count
218 assert re
.findall(r
'^ENCODING-ORIGINAL (ctm/in/[^:]+): PASS;', log
, re
.M
) == [ORIGINAL
]
219 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
220 xtm
= read(retained
/ 'xtm.json')
221 assert xtm
== read(ROOT
/ 'reports/slice-19d2/xtm.json')
222 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
223 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
224 lock
= read(ROOT
/ 'references/corpus-lock.json')
225 for entry
in lock
['files']:
226 path
= ROOT
/ 'vendor/cxtm' / entry
['path']
227 assert path
.stat().st_size
== entry
['size'] and sha(path
) == entry
['sha256'], str(path
)
228 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
229 assert historical
== {p
: sha(ROOT
/ p
) for p
in historical
}
230 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
231 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
232 save(output
/ 'remaining.json', [c
for c
in final
['cases'] if c
['status'] == 'not-run'])
233 save(output
/ 'integrity.json', {'protectedFilesSHA256': protected
, 'historicalReportsSHA256': historical
})
235 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
236 'initialRepositoryState': {'head': BASE
, 'branch': 'codex/ctm-slice-20g', 'indexClean': True, 'worktreeClean': True},
237 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
238 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources
, 'newDependencyDownloads': 0,
239 'sourcePreparationCommand': preparation
, 'qualifiedSourceProvenance': provenance
, 'runs': runs
,
240 'retainedLispChecks': sum(totals
.values()), 'retainedChecks': totals
,
241 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
242 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': 13,
243 'ctmIntegerChecks': 112, 'tripleChecks': 226, 'encodingChecks': encoding
['passed'],
244 'positiveControls': len(positives
), 'negativeControls': len(negatives
),
245 'all256ByteMappingsMatchIndependentOracle': True, 'runtimeQualificationBeforeImplementation': qualification
,
246 'supportedISO88595Names': ISO_NAMES
, 'supportedUTF8Names': ['UTF-8', 'csUTF8'],
247 'strictUTF8AndIRIUnescapingByteIdenticalToBase': True,
248 'originalComparison': {'id': ORIGINAL
, 'documentIRI': row
['documentIRI'], 'sourceBytes': source
.stat().st_size
,
249 'sourceSHA256': sha(source
), 'expectedBytes': len(expected
), 'expectedSHA256': sha(baseline
),
250 'actualBytes': len(actual
), 'actualSHA256': common
.digest(actual
), 'firstDifferentByte': None,
251 'originalNameBytesHex': 'C0 DE E1 E1 D8 EF', 'value': 'Россия',
252 'canonicalizationPreservesMap': True, 'executedInFullRegression': True},
253 'updatedUnsupportedAssertions': sorted(obsolete
),
254 'updatedAssertionReason': 'ISO-8859-5 is now supported; ISO-8859-1 remains unsupported. Only messages and one named fixture/check label change in regenerated reports.',
255 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
256 'ctmStatusesBefore': before
['statuses'], 'ctmStatusesAfter': final
['statuses'], 'ctmStatusChanges': changes
,
257 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
258 'knownFailuresUnchanged': [c
['id'] for c
in final
['cases'] if c
['status'] == 'fail'],
259 'retainedReportsVerified': verified
, 'allHistoricalReportBytesUnchanged': True,
260 'historicalReportFilesVerified': len(historical
), 'allProtectedFileBytesUnchanged': True,
261 'protectedFilesVerified': len(protected
), 'pinnedCorpusFilesVerified': len(lock
['files']),
262 'persistentLispImagesTouched': False,
263 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
264 'sandboxStartupError': 'writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.',
265 'executionAuthorization': 'Regular require_escalated approval, canonical SBCL/Nix Store paths; no symlink or sandbox configuration changes. All regression subprocesses deny network.',
266 'acceptancePassed': True,
268 save(output
/ 'execution.json', evidence
)
269 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
270 'standaloneXTMChecks', 'xtmMergemapChecks', 'encodingChecks', 'positiveControls', 'negativeControls',
271 'ctmStatusesAfter', 'xtmCorpusStatuses', 'originalComparison', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent
=2))
274 if __name__
== '__main__':