2 """Validate the minimal CTM prolog implementation and every retained suite offline."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= 'd024dc96ce091db06b9d09ea7c1ccb5f90b2886d'
16 POLICY
= '(version 1)(allow default)(deny network*)'
18 'BOM-contradicts-encoding': ('encoding-conflict', 14, 'byte', None, None),
19 'unknown-directive': ('unknown-directive', 0, 'character', 1, 1),
20 'wrong-version': ('invalid-version', 9, 'character', 1, 10),
22 # Five prior 19D2 labels plus the one obsolete version-unimplemented assertion.
24 'diagnostic:known-XTM-unimplemented:specific-condition': 'diagnostic:XTM-explicit-resolver-required:specific-condition',
25 'diagnostic:known-XTM-unimplemented:source-position': 'diagnostic:XTM-explicit-resolver-required:source-position',
26 'policy:XTM-recognized-without-retrieval:specific-condition': 'policy:XTM-explicit-resolver-before-retrieval:specific-condition',
27 'policy:XTM-recognized-without-retrieval:source-position': 'policy:XTM-explicit-resolver-before-retrieval:source-position',
28 'policy:XTM-does-not-call-resolver': 'policy:XTM-without-resolver-does-not-fetch',
29 'remaining-construct-unsupported:%version 1.0': 'prefix:version-1.0-prolog-supported',
34 spec
= importlib
.util
.spec_from_file_location('slice20a', ROOT
/ 'scripts/validate-slice-20a.py')
35 common
= importlib
.util
.module_from_spec(spec
)
36 spec
.loader
.exec_module(common
)
37 read
, save
, sha
, command
, snapshot
= common
.read
, common
.save
, common
.sha
, common
.command
, common
.snapshot
38 parser
= argparse
.ArgumentParser(description
=__doc__
)
39 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
40 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-20c')
41 parser
.add_argument('--preimplementation-log', type=Path
)
42 args
= parser
.parse_args()
43 runtime
= shutil
.which(args
.sbcl
)
44 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
45 output
= args
.output
.resolve()
46 output
.mkdir(parents
=True, exist_ok
=False)
47 retained_dir
= output
/ 'retained'
49 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
50 protected_paths
= ['src/tmdm', 'src/cxtm', 'src/xml', 'src/xtm', 'patches', 'vendor', 'references',
51 'flake.nix', 'flake.lock', 'scripts/test.sh', 'scripts/prepare-cxml.py',
52 'scripts/validate-slice-19d2.py', 'scripts/validate-slice-20a.py', 'scripts/validate-slice-20b.py']
53 tracked_tests
= command(['git', 'ls-tree', '-r', '--name-only', BASE
, '--', 'tests']).splitlines()
54 protected_paths
+= [p
for p
in tracked_tests
if p
!= 'tests/ctm-prefixes.lisp']
55 protected_paths
+= ['src/ctm/' + p
.name
for p
in (ROOT
/ 'src/ctm').glob('*.lisp')
56 if p
.name
not in ['unicode.lisp', 'lexer.lisp', 'reader.lisp', 'resources.lisp']]
57 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
58 protected
= snapshot(p
for directory
in ['src/tmdm', 'src/cxtm', 'src/xml', 'src/xtm', 'vendor', 'references', 'patches']
59 for p
in (ROOT
/ directory
).rglob('*'))
60 historical
= snapshot(p
for p
in (ROOT
/ 'reports').rglob('*') if output
not in p
.parents
)
61 sources
= snapshot(p
for directory
in ['src', 'tests', 'scripts', 'docs']
62 for p
in (ROOT
/ directory
).rglob('*') if p
.suffix
in ['.lisp', '.py', '.sh', '.md'])
63 sources
.update(snapshot(ROOT
/ p
for p
in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
64 if args
.preimplementation_log
:
65 preimplementation
= [json
.loads(line
) for line
in args
.preimplementation_log
.read_text().splitlines() if line
.startswith('{')]
66 assert [Path(r
['id']).stem
for r
in preimplementation
] == list(EXPECTED
)
67 assert all(r
['condition']['condition'] == 'CTM-UNSUPPORTED' and r
['condition']['code'] == 'unsupported-directive'
68 for r
in preimplementation
)
69 save(output
/ 'preimplementation.json', {'baseCommit': BASE
, 'runtime': 'SBCL 2.5.4',
70 'freshProcess': True, 'network': 'denied by sandbox-exec', 'cases': preimplementation
})
72 with tempfile
.TemporaryDirectory(prefix
='topic-maps-20c-') as tmp_name
:
74 preparation
= ['python3', 'scripts/prepare-cxml.py', '--directory', str(tmp
/ 'sources')]
75 provenance
= json
.loads(command(preparation
))
76 save(output
/ 'source-preparation.json', provenance
)
77 env
= dict(os
.environ
, SBCL
=runtime
, PYTHONDONTWRITEBYTECODE
='1', CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'])
78 env
.update(TOPIC_MAPS_REPORT
=str(retained_dir
/ 'core.json'),
79 TOPIC_MAPS_XTM_REPORT
=str(retained_dir
/ 'xtm.json'),
80 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained_dir
/ 'xtm-mergemap'),
81 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained_dir
/ 'slice20a'),
82 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained_dir
/ 'slice20b'),
83 TOPIC_MAPS_PROLOG_REPORT
=str(output
/ 'current'))
85 ('xml', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
86 ('xtm', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
87 ('regression', [str(ROOT
/ 'scripts/test.sh')]),
89 for label
, invocation
in invocations
:
90 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
91 context
= {'label': label
, 'command': invocation
, 'freshProcess': True, 'network': 'denied by sandbox-exec',
92 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
93 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
, timeout
=300)
94 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
95 context
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
97 assert result
.returncode
== 0, f
'{label} failed; see {label}.log'
98 if label
in ['xml', 'xtm']:
99 marker
= 'CXML' if label
== 'xml' else 'XTM'
100 labels
= re
.findall(r
'^' + marker
+ r
'-CHECK-PASS (.+)$', result
.stdout
.decode(), re
.M
)
101 assert len(labels
) == len(set(labels
)) == (146 if label
== 'xml' else 327)
102 assert f
'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result
.stdout
.decode()
103 context
['passedChecks'] = len(labels
)
104 # Keep the two retained documented inspection examples executable too.
105 for label
, document
in [('xtm-inspection', 'docs/slice-19d2-xtm-mergemap.md'),
106 ('ctm-inspection', 'docs/slice-20a-registration.md')]:
107 forms
= (ROOT
/ document
).read_text().split('```lisp\n')[-1].split('\n```', 1)[0]
108 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', '/dev/stdin']
109 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, input=forms
.encode(), stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
, timeout
=120)
110 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
111 assert result
.returncode
== 0, label
112 def retained_equal(prior
, current
):
115 assert 'checks' in prior
and 'checks' in current
116 assert current
['checks'] == [RENAMED
.get(c
, c
) for c
in prior
['checks']]
117 assert {k
: v
for k
, v
in prior
.items() if k
!= 'checks'} == {k
: v
for k
, v
in current
.items() if k
!= 'checks'}
118 baseline
= ROOT
/ 'reports/slice-19c'
119 verified_reports
= []
120 for old
in sorted(baseline
.glob('*.json')):
121 if old
.name
== 'execution.json':
123 current
= retained_dir
/ ('core.json' if old
.name
== 'validation.json' else 'core.json.' + old
.name
)
124 retained_equal(read(old
), read(current
))
125 verified_reports
.append(str(old
.relative_to(ROOT
)))
126 for suffix
in ['validation', 'cxtm', 'integration']:
127 retained_equal(read(ROOT
/ f
'reports/slice-19d2/{suffix}.json'), read(retained_dir
/ f
'xtm-mergemap.{suffix}.json'))
128 for stage
, directory
, suffixes
in [('slice20a', 'slice-20a', ['validation', 'cxtm', 'registration']),
129 ('slice20b', 'slice-20b', ['validation', 'cxtm', 'negative'])]:
130 for suffix
in suffixes
:
131 retained_equal(read(ROOT
/ f
'reports/{directory}/current.{suffix}.json'), read(retained_dir
/ f
'{stage}.{suffix}.json'))
132 before
= read(retained_dir
/ 'slice20b.validation.json')
133 final
= read(output
/ 'current.validation.json')
134 cxtm
= read(output
/ 'current.cxtm.json')
135 prolog
= read(output
/ 'current.prolog.json')
136 assert prolog
['entryStatuses'] == before
['statuses'] == {'pass': 252, 'fail': 4, 'unsupported': 0, 'not-run': 17}
137 assert final
['statuses'] == {'pass': 255, 'fail': 4, 'unsupported': 0, 'not-run': 14}
138 assert prolog
['failed'] == 0 and prolog
['passed'] == len(prolog
['checks']) == len(set(prolog
['checks']))
139 rows
= {r
['id']: r
for r
in prolog
['originals']}
140 assert list(rows
) == ['ctm/invalid/' + name
+ '.ctm' for name
in EXPECTED
]
141 assert len(before
['cases']) == len(final
['cases']) == 273
142 changes
, candidates
= [], []
143 for a
, b
in zip(before
['cases'], final
['cases']):
144 assert a
['id'] == b
['id']
145 if a
['id'] not in rows
:
146 assert a
== b
, a
['id']
149 assert row
['qualified'] and row
['status'] == 'pass' and row
['control']['accepted'] and row
['control']['condition'] is None
150 assert row
['documentIRI'] == a
['documentIRI']
151 assert a
['documentIRI'] == 'https://ctm-tests.example.invalid/' + before
['commit'] + '/' + a
['id']
152 assert row
['condition'] == row
['repeatCondition']
154 expected
= EXPECTED
[Path(a
['id']).stem
]
155 assert c
['condition'] == 'CTM-REJECTION' and (c
['code'], c
['position'], c
['positionUnit'], c
['line'], c
['column']) == expected
156 assert c
['source'] == a
['id'] and c
['cause'] is c
['context'] is None
157 data
= (ROOT
/ 'vendor/cxtm' / a
['sourcePathname']).read_bytes()
158 assert row
['sourceHex'] == data
.hex(' ').upper()
159 assert row
['sourceTextAfterBOM'] == data
.decode('utf-8-sig')
160 if c
['positionUnit'] == 'byte':
161 assert data
[:3] == bytes
.fromhex('ef bb bf') and data
[c
['position']:].startswith(b
'iso-8859-1')
164 assert text
[c
['position']:].startswith('10.0' if Path(a
['id']).stem
== 'wrong-version' else '%ctm')
165 assert text
[:c
['position']].count('\n') + 1 == c
['line']
166 assert len(text
[:c
['position']].rsplit('\n', 1)[-1]) + 1 == c
['column']
167 assert a
['status'] == 'not-run' and a
['actualOutcome'] is None
168 assert b
== dict(a
, status
='pass', actualOutcome
='ctm-rejection', diagnostics
=[c
['message']])
169 changes
.append({'id': a
['id'], 'before': a
['status'], 'after': b
['status']})
170 candidates
.append({'id': a
['id'], 'status': b
['status'], 'sourceSHA256': sha(ROOT
/ 'vendor/cxtm' / a
['id']),
171 'documentIRI': a
['documentIRI'], 'clauses': row
['clauses'], 'condition': c
})
172 assert {k
: v
for k
, v
in before
.items() if k
not in ['cases', 'statuses', 'implementationScope']} == {
173 k
: v
for k
, v
in final
.items() if k
not in ['cases', 'statuses', 'implementationScope']}
174 assert cxtm
['cases'] == read(retained_dir
/ 'slice20b.cxtm.json')['cases']
175 assert cxtm
['contractCounts'] == {'pass': 217, 'fail': 4}
176 # Independently recheck every actual positive CXTM stream against its pin.
177 for case
in cxtm
['cases']:
178 name
= Path(case
['id']).name
179 baseline_bytes
= (ROOT
/ 'vendor/cxtm/ctm/baseline' / (name
+ '.cxtm')).read_bytes()
180 actual
= case
['actualCXTM'].encode('utf-8')
181 assert (actual
== baseline_bytes
) == (case
['status'] == 'pass')
182 assert case
['expectedCXTM'].encode('utf-8') == baseline_bytes
183 positives
= prolog
['positives']
184 assert len([p
for p
in positives
if p
['label'].startswith('utf8:provided=')]) == 18
185 assert all(p
['accepted'] for p
in positives
)
186 assert 'encoding-selection-precedes-decoder:zero-calls' in prolog
['checks']
187 negative_labels
= {r
['label']: r
['condition'] for r
in prolog
['negativeControls']}
188 for label
, c
in negative_labels
.items():
189 assert c
['condition'] in ['CTM-REJECTION', 'CTM-UNSUPPORTED']
190 if c
['positionUnit'] == 'byte':
191 assert c
['line'] is c
['column'] is None
192 assert negative_labels
['conflict-before-invalid-body-decoding']['code'] == 'encoding-conflict'
193 assert negative_labels
['unsupported-before-invalid-body-decoding']['condition'] == 'CTM-UNSUPPORTED'
194 assert negative_labels
['known-literal-unsupported']['code'] == 'occurrence-literal-datatype'
195 log
= (output
/ 'regression.log').read_text()
196 previous_totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
197 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for marker
in previous_totals
}
198 assert totals
== previous_totals
and sum(totals
.values()) == 12100
199 for marker
, expected
in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25), ('PROLOG', prolog
['passed'])]:
200 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == expected
201 assert len(re
.findall(r
'^PROLOG-ORIGINAL ctm/invalid/[^:]+: PASS;', log
, re
.M
)) == 3
202 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
203 xtm
= read(retained_dir
/ 'xtm.json')
204 assert xtm
== read(ROOT
/ 'reports/slice-19d2/xtm.json')
205 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
206 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
207 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
208 assert historical
== {p
: sha(ROOT
/ p
) for p
in historical
}
209 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
210 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
211 save(output
/ 'candidates.json', candidates
)
212 save(output
/ 'remaining.json', [c
for c
in final
['cases'] if c
['status'] == 'not-run'])
213 save(output
/ 'integrity.json', {'protectedFilesSHA256': protected
, 'historicalReportsSHA256': historical
})
215 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
216 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
217 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources
, 'newDependencyDownloads': 0,
218 'sourcePreparationCommand': preparation
, 'qualifiedSourceProvenance': provenance
, 'runs': runs
,
219 'retainedLispChecks': sum(totals
.values()), 'retainedChecks': totals
,
220 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
221 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': prolog
['passed'],
222 'positivePrologAndResourceControls': len(positives
), 'negativeControls': len(prolog
['negativeControls']),
223 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
224 'ctmStatusesBefore': before
['statuses'], 'ctmStatusesAfter': final
['statuses'], 'ctmStatusChanges': changes
,
225 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
226 'knownFailuresUnchanged': [c
['id'] for c
in final
['cases'] if c
['status'] == 'fail'],
227 'retainedReportsVerified': verified_reports
, 'retainedReportLabelChanges': RENAMED
,
228 'retainedPositiveCXTMComparisons': len(cxtm
['cases']),
229 'allHistoricalReportBytesUnchanged': True, 'allProtectedFileBytesUnchanged': True,
230 'encodingConflictBeforeDecoderCalls': 0, 'optionalEncodingAPI': 'deserialize-file :encoding',
231 'encodingScope': 'UTF-8 only; existing strict decoder unchanged; no aliases or new charset decoder; resolver protocol unchanged.',
232 'persistentLispImagesTouched': False, 'documentedRetainedInspectionsPassed': True,
233 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
234 'acceptancePassed': True,
236 save(output
/ 'execution.json', evidence
)
237 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks', 'standaloneXTMChecks',
238 'xtmMergemapChecks', 'slice20aChecks', 'slice20bChecks', 'prologChecks', 'ctmStatusesAfter', 'xtmCorpusStatuses', 'acceptancePassed']}, indent
=2))
241 if __name__
== '__main__':