2 """Validate four Triple String originals and the complete retained offline suite."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= '1ef5768ce6913f0c04da111d5f525b6d50a27e0b'
16 POLICY
= '(version 1)(allow default)(deny network*)'
18 'occurrence-string-triple': (41, 367, 'Occurrence " Value'),
19 'occurrence-string-triple2': (48, 370, 'Occurrence"" " Value"'),
20 'unicode-escape-u4-triple-string': (33, 331, 'ZZ \nTop'),
21 'unicode-escape-u6-triple-string': (37, 331, 'ZZ \nTop'),
26 spec
= importlib
.util
.spec_from_file_location('slice20a', ROOT
/ 'scripts/validate-slice-20a.py')
27 common
= importlib
.util
.module_from_spec(spec
)
28 spec
.loader
.exec_module(common
)
29 read
, save
, sha
, command
, snapshot
= common
.read
, common
.save
, common
.sha
, common
.command
, common
.snapshot
30 parser
= argparse
.ArgumentParser(description
=__doc__
)
31 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
32 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-20g')
33 parser
.add_argument('--preimplementation', type=Path
, default
=ROOT
/ 'reports/slice-20g/preimplementation.json')
34 args
= parser
.parse_args()
35 pre
= read(args
.preimplementation
)
36 assert pre
['baseCommit'] == BASE
and len(pre
['cases']) == 12 and len(pre
['originals']) == 4
37 runtime
= shutil
.which(args
.sbcl
)
38 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
39 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
40 mutable
= {'src/ctm/lexer.lisp', 'tests/ctm-identity.lisp', 'tests/ctm-templates.lisp',
41 'tests/ctm-prolog.lisp', 'tests/ctm-integer-literal.lisp',
42 'tests/ctm-registered-originals.lisp', 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
43 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
44 protected_paths
= [p
for p
in tracked
if p
not in mutable
]
45 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
46 output
= args
.output
.resolve()
47 output
.mkdir(parents
=True, exist_ok
=False)
48 retained
= output
/ 'retained'
50 save(output
/ 'preimplementation.json', pre
)
51 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
52 protected
= snapshot(ROOT
/ p
for p
in protected_paths
)
53 historical
= snapshot(p
for p
in (ROOT
/ 'reports').rglob('*') if output
not in p
.parents
)
54 sources
= snapshot(p
for directory
in ['src', 'tests', 'scripts', 'docs']
55 for p
in (ROOT
/ directory
).rglob('*') if p
.suffix
in ['.lisp', '.py', '.sh', '.md'])
56 sources
.update(snapshot(ROOT
/ p
for p
in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
58 with tempfile
.TemporaryDirectory(prefix
='topic-maps-20g-') as temp_name
:
59 preparation
= ['python3', 'scripts/prepare-cxml.py', '--directory', str(Path(temp_name
) / 'sources')]
60 provenance
= json
.loads(command(preparation
)) # Existing pinned local sources, nix --offline.
61 save(output
/ 'source-preparation.json', provenance
)
62 env
= dict(os
.environ
, SBCL
=runtime
, GIT_OPTIONAL_LOCKS
='0', PYTHONDONTWRITEBYTECODE
='1',
63 CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],
64 TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'),
65 TOPIC_MAPS_XTM_REPORT
=str(retained
/ 'xtm.json'),
66 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
67 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'),
68 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
69 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'),
70 TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
71 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'),
72 TOPIC_MAPS_TRIPLE_REPORT
=str(output
/ 'current'))
73 for label
, invocation
in [
74 ('xml', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
75 ('xtm', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
76 ('regression', [str(ROOT
/ 'scripts/test.sh')]),
78 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
79 run
= {'label': label
, 'command': invocation
, 'freshProcess': True, 'network': 'denied by sandbox-exec',
80 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
81 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
82 stderr
=subprocess
.STDOUT
, timeout
=300)
83 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
84 run
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
86 assert result
.returncode
== 0, f
'{label} failed; see {label}.log'
87 if label
in ['xml', 'xtm']:
88 marker
= 'CXML' if label
== 'xml' else 'XTM'
89 labels
= re
.findall(r
'^' + marker
+ r
'-CHECK-PASS (.+)$', result
.stdout
.decode(), re
.M
)
90 assert len(labels
) == len(set(labels
)) == (146 if label
== 'xml' else 327)
91 assert f
'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result
.stdout
.decode()
92 run
['passedChecks'] = len(labels
)
93 # Permit only named obsolete Unsupported assertions in regenerated reports.
94 # Archived report bytes are independently hashed and remain unchanged.
96 'diagnostic:unchanged-excluded-feature:def f() a - """N""". end': 'diagnostic:triple-string-body-supported',
97 'known-triple-string-unsupported:specific-condition-code-position-source': 'known-string-unclosed:specific-condition-code-position-source',
98 'known-triple-string-unsupported:position-units': 'known-string-unclosed:position-units',
99 'negative:triple-still-unsupported:class-code-exact-position': 'negative:triple-unclosed:class-code-exact-position',
101 old_prolog
= read(ROOT
/ 'reports/slice-20f/retained/slice20c.prolog.json')
102 old_prolog_row
= next(r
for r
in old_prolog
['negativeControls'] if r
['label'] == 'known-triple-string-unsupported')
103 old_integer
= read(ROOT
/ 'reports/slice-20f/current.literal.json')
104 old_integer_row
= next(r
for r
in old_integer
['negativeControls'] if r
['label'] == 'triple-still-unsupported')
105 def updated_condition(prior
, source
):
106 return dict(prior
, condition
='CTM-REJECTION', code
='unterminated-string', source
=source
,
107 message
=f
'UNTERMINATED-STRING in {source} at CHARACTER {prior["position"]} (line {prior["line"]}, column {prior["column"]}): Unterminated triple-quoted string.')
108 def qualified_retained(value
):
109 if isinstance(value
, dict):
110 if value
.get('label') == 'known-triple-string-unsupported':
111 assert value
== old_prolog_row
112 return dict(value
, label
='known-string-unclosed',
113 condition
=updated_condition(value
['condition'], 'known-string-unclosed'))
114 if value
.get('label') == 'triple-still-unsupported':
115 assert value
== old_integer_row
116 return dict(value
, label
='triple-unclosed', sourceText
='topic t: """x.',
117 condition
=updated_condition(value
['condition'], 'triple-unclosed'))
118 return {k
: qualified_retained(v
) for k
, v
in value
.items()}
119 if isinstance(value
, list):
120 return [qualified_retained(v
) for v
in value
]
121 if isinstance(value
, str):
122 return labels
.get(value
, value
)
125 for old
in sorted((ROOT
/ 'reports/slice-20f/retained').glob('*.json')):
126 assert qualified_retained(read(old
)) == read(retained
/ old
.name
), str(old
)
127 verified
.append(str(old
.relative_to(ROOT
)))
128 for suffix
in ['validation', 'cxtm', 'literal']:
129 old
= ROOT
/ f
'reports/slice-20f/current.{suffix}.json'
130 assert qualified_retained(read(old
)) == read(retained
/ f
'slice20f.{suffix}.json'), str(old
)
131 verified
.append(str(old
.relative_to(ROOT
)))
132 before
= read(retained
/ 'slice20f.validation.json')
133 final
= read(output
/ 'current.validation.json')
134 triple
= read(output
/ 'current.triple.json')
135 cxtm
= read(output
/ 'current.cxtm.json')
136 assert before
['statuses'] == triple
['entryStatuses'] == {'pass': 257, 'fail': 4, 'unsupported': 0, 'not-run': 12}
137 assert final
['statuses'] == {'pass': 261, 'fail': 4, 'unsupported': 0, 'not-run': 8}
138 assert triple
['passed'] == len(triple
['checks']) == len(set(triple
['checks'])) and triple
['failed'] == 0
139 rows
= {r
['id']: r
for r
in triple
['originals']}
140 ids
= ['ctm/in/' + n
+ '.ctm' for n
in CONTRACTS
]
141 assert list(rows
) == ids
142 changes
, comparisons
= [], []
143 positives
= {p
['label']: p
for p
in triple
['positives']}
144 assert len(positives
) == len(triple
['positives']) == 77
145 assert all(p
['initialCounter'] == p
['finalCounter'] == 37 for p
in positives
.values())
146 negatives
= {r
['label']: r
for r
in triple
['negativeControls']}
147 assert len(negatives
) == len(triple
['negativeControls']) == 31
148 for prior
in pre
['cases']:
149 current
= negatives
[prior
['label']]
150 assert prior
['sourceText'] == current
['sourceText']
151 assert prior
['result'] == {k
: current
['condition'][k
] for k
in prior
['result']}, prior
['label']
152 assert len(before
['cases']) == len(final
['cases']) == 273
153 for a
, b
in zip(before
['cases'], final
['cases']):
154 assert a
['id'] == b
['id']
155 if a
['id'] not in rows
:
156 assert a
== b
, a
['id']
159 name
= Path(a
['id']).stem
160 source_size
, expected_size
, expected_value
= CONTRACTS
[name
]
161 source
= ROOT
/ 'vendor/cxtm' / a
['id']
162 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (name
+ '.ctm.cxtm')
163 assert a
['status'] == 'not-run' and a
['actualOutcome'] is None and a
['category'] == 'valid'
164 assert a
['documentIRI'] == row
['documentIRI'] == 'https://ctm-tests.example.invalid/' + before
['commit'] + '/' + a
['id']
165 assert b
== dict(a
, status
='pass', actualOutcome
='cxtm-match')
166 assert row
['status'] == 'pass' and row
['actualOutcome'] == 'cxtm-match' and row
['diagnostics'] == []
167 assert source
.stat().st_size
== row
['sourceBytes'] == source_size
168 assert row
['comparison']['actualBytes'] == row
['comparison']['expectedBytes'] == expected_size
169 assert row
['comparison']['firstDifferentByte'] is None
170 model
= row
['comparison']['model']
171 assert model
['topicCount'] == 2 and model
['associationCount'] == 0
172 constructs
= [item
for t
in model
['topics'] for item
in t
['occurrences' if name
.startswith('occurrence-') else 'names']]
173 assert len(constructs
) == 1 and constructs
[0]['value'] == expected_value
174 if name
.startswith('occurrence-'):
175 assert constructs
[0]['datatype'] == 'http://www.w3.org/2001/XMLSchema#string'
176 actual_case
= next(c
for c
in cxtm
['cases'] if c
['id'] == a
['id'])
177 actual
= actual_case
['actualCXTM'].encode('utf-8')
178 assert actual
== actual_case
['expectedCXTM'].encode('utf-8') == baseline
.read_bytes()
179 control
= positives
[a
['id'] + ':ordinary-control']['actualCXTM'].encode('utf-8')
180 assert control
== actual
181 prior
= next(r
for r
in pre
['originals'] if r
['original'] == a
['id'])
182 assert prior
['documentIRI'] == row
['documentIRI']
183 assert prior
['result']['condition'] == 'CTM-UNSUPPORTED' and prior
['result']['code'] == 'triple-quoted-string'
184 comparisons
.append({'id': a
['id'], 'documentIRI': row
['documentIRI'], 'sourceBytes': source_size
,
185 'sourceSHA256': sha(source
), 'expectedBytes': expected_size
, 'expectedSHA256': sha(baseline
),
186 'actualBytes': len(actual
), 'actualSHA256': common
.digest(actual
), 'firstDifferentByte': None,
187 'ordinaryControlBytes': len(control
), 'ordinaryControlSHA256': common
.digest(control
),
188 'value': expected_value
, 'datatype': 'http://www.w3.org/2001/XMLSchema#string',
189 'canonicalizationPreservesMap': True, 'executedInFullRegression': True})
190 changes
.append({'id': a
['id'], 'before': a
['status'], 'after': b
['status']})
191 assert len(changes
) == 4
192 assert {k
: v
for k
, v
in before
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
193 k
: v
for k
, v
in final
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
194 old_cxtm
= read(retained
/ 'slice20f.cxtm.json')
195 assert [c
for c
in cxtm
['cases'] if c
['id'] not in rows
] == old_cxtm
['cases']
196 assert cxtm
['contractCounts'] == {'pass': 223, 'fail': 4}
197 for case
in cxtm
['cases']:
198 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (Path(case
['id']).name
+ '.cxtm')
199 actual
= case
['actualCXTM'].encode('utf-8')
200 assert (actual
== baseline
.read_bytes()) == (case
['status'] == 'pass')
201 assert case
['expectedCXTM'].encode('utf-8') == baseline
.read_bytes()
202 log
= (output
/ 'regression.log').read_text()
203 prior_totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
204 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for marker
in prior_totals
}
205 assert totals
== prior_totals
and sum(totals
.values()) == 12100
206 for marker
, count
in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286), ('REGISTERED-NEGATIVES', 25),
207 ('PROLOG', 171), ('EMPTY2', 13), ('CTM-INTEGER', 112), ('TRIPLE', triple
['passed'])]:
208 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == count
209 assert re
.findall(r
'^TRIPLE-ORIGINAL (ctm/in/[^:]+): PASS;', log
, re
.M
) == ids
210 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
211 xtm
= read(retained
/ 'xtm.json')
212 assert xtm
== read(ROOT
/ 'reports/slice-19d2/xtm.json')
213 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
214 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
215 lock
= read(ROOT
/ 'references/corpus-lock.json')
216 for entry
in lock
['files']:
217 path
= ROOT
/ 'vendor/cxtm' / entry
['path']
218 assert path
.stat().st_size
== entry
['size'] and sha(path
) == entry
['sha256'], str(path
)
219 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
220 assert historical
== {p
: sha(ROOT
/ p
) for p
in historical
}
221 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
222 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
223 save(output
/ 'remaining.json', [c
for c
in final
['cases'] if c
['status'] == 'not-run'])
224 save(output
/ 'integrity.json', {'protectedFilesSHA256': protected
, 'historicalReportsSHA256': historical
})
226 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
227 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
228 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources
, 'newDependencyDownloads': 0,
229 'sourcePreparationCommand': preparation
, 'qualifiedSourceProvenance': provenance
, 'runs': runs
,
230 'retainedLispChecks': sum(totals
.values()), 'retainedChecks': totals
,
231 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
232 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': 13, 'ctmIntegerChecks': 112,
233 'tripleChecks': triple
['passed'], 'positiveControls': len(positives
), 'negativeControls': len(negatives
),
234 'ordinaryStringDiagnosticSignaturesUnchanged': len(pre
['cases']), 'originalComparisons': comparisons
,
235 'retainedReportLabelChanges': labels
, 'updatedAssertions': [
236 'Identity Triple String boundary now checks its actual Name value.',
237 'Existing Template body String path checks Name N after invocation.',
238 'Prolog and CTM-integer old valid-triple Unsupported probes now reject unclosed Triple Strings.'],
239 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
240 'ctmStatusesBefore': before
['statuses'], 'ctmStatusesAfter': final
['statuses'], 'ctmStatusChanges': changes
,
241 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
242 'knownFailuresUnchanged': [c
['id'] for c
in final
['cases'] if c
['status'] == 'fail'],
243 'retainedReportsVerified': verified
, 'allHistoricalReportBytesUnchanged': True,
244 'allProtectedFileBytesUnchanged': True, 'pinnedCorpusFilesVerified': len(lock
['files']),
245 'persistentLispImagesTouched': False,
246 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
247 'executionAuthorization': 'Regular require_escalated approval; no symlink or sandbox configuration changes.',
248 'acceptancePassed': True,
250 save(output
/ 'execution.json', evidence
)
251 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
252 'standaloneXTMChecks', 'xtmMergemapChecks', 'tripleChecks', 'positiveControls', 'negativeControls',
253 'ctmStatusesAfter', 'xtmCorpusStatuses', 'originalComparisons', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent
=2))
256 if __name__
== '__main__':