2 """Validate the empty2 original registration and all retained suites offline."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= '9866a0e94a2a43c6aa73025f1c8d06b0f093a34b'
16 ID
= 'ctm/in/empty2.ctm'
17 POLICY
= '(version 1)(allow default)(deny network*)'
18 SOURCE
= b
'%version 1.0\n'
19 EXPECTED
= b
'<topicMap>\n</topicMap>\n'
23 # Reuse existing report/hash utilities and historical label qualification.
25 spec
= importlib
.util
.spec_from_file_location(name
, ROOT
/ ('scripts/validate-' + name
+ '.py'))
26 module
= importlib
.util
.module_from_spec(spec
)
27 spec
.loader
.exec_module(module
)
29 common
, prolog_driver
= previous('slice-20a'), previous('slice-20c')
30 read
, save
, sha
, command
, snapshot
= common
.read
, common
.save
, common
.sha
, common
.command
, common
.snapshot
31 parser
= argparse
.ArgumentParser(description
=__doc__
)
32 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
33 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-20e')
34 args
= parser
.parse_args()
35 runtime
= shutil
.which(args
.sbcl
)
36 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
37 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
38 mutable
= {'tests/ctm-registered-originals.lisp', 'topic-maps.asd', 'scripts/fresh-sbcl.lisp'}
39 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
40 protected_paths
= [p
for p
in tracked
if p
not in mutable
]
41 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
42 output
= args
.output
.resolve()
43 output
.mkdir(parents
=True, exist_ok
=False)
44 retained
= output
/ 'retained'
46 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
47 protected
= snapshot(ROOT
/ p
for p
in protected_paths
)
48 historical
= snapshot(p
for p
in (ROOT
/ 'reports').rglob('*') if output
not in p
.parents
)
49 sources
= snapshot(p
for directory
in ['src', 'tests', 'scripts', 'docs']
50 for p
in (ROOT
/ directory
).rglob('*') if p
.suffix
in ['.lisp', '.py', '.sh', '.md'])
51 sources
.update(snapshot(ROOT
/ p
for p
in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
52 source
= ROOT
/ 'vendor/cxtm' / ID
53 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline/empty2.ctm.cxtm'
54 assert source
.read_bytes() == SOURCE
and baseline
.read_bytes() == EXPECTED
56 with tempfile
.TemporaryDirectory(prefix
='topic-maps-20e-') as temp_name
:
57 temp
= Path(temp_name
)
58 preparation
= ['python3', 'scripts/prepare-cxml.py', '--directory', str(temp
/ 'sources')]
59 provenance
= json
.loads(command(preparation
)) # nix eval --offline; verifies local Store source hashes.
60 save(output
/ 'source-preparation.json', provenance
)
61 env
= dict(os
.environ
, SBCL
=runtime
, GIT_OPTIONAL_LOCKS
='0', PYTHONDONTWRITEBYTECODE
='1',
62 CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],
63 TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'),
64 TOPIC_MAPS_XTM_REPORT
=str(retained
/ 'xtm.json'),
65 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
66 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'),
67 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
68 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'),
69 TOPIC_MAPS_EMPTY2_REPORT
=str(output
/ 'current'))
71 ('xml', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'tests/cxml-probe.lisp']),
72 ('xtm', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', 'scripts/fresh-xtm.lisp']),
73 ('regression', [str(ROOT
/ 'scripts/test.sh')]),
75 for label
, invocation
in invocations
:
76 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
] + invocation
77 run
= {'label': label
, 'command': invocation
, 'freshProcess': True,
78 'network': 'denied by sandbox-exec',
79 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
80 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
81 stderr
=subprocess
.STDOUT
, timeout
=300)
82 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
83 run
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
85 assert result
.returncode
== 0, f
'{label} failed; see {label}.log'
86 if label
in ['xml', 'xtm']:
87 marker
= 'CXML' if label
== 'xml' else 'XTM'
88 labels
= re
.findall(r
'^' + marker
+ r
'-CHECK-PASS (.+)$', result
.stdout
.decode(), re
.M
)
89 assert len(labels
) == len(set(labels
)) == (146 if label
== 'xml' else 327)
90 assert f
'{marker}-SUMMARY {len(labels)} PASS 0 FAIL' in result
.stdout
.decode()
91 run
['passedChecks'] = len(labels
)
92 # Compare regenerated retained stages with their historical provenance.
93 def retained_equal(prior
, current
):
96 assert current
['checks'] == [prolog_driver
.RENAMED
.get(c
, c
) for c
in prior
['checks']]
97 assert {k
: v
for k
, v
in prior
.items() if k
!= 'checks'} == {
98 k
: v
for k
, v
in current
.items() if k
!= 'checks'}
100 for old
in sorted((ROOT
/ 'reports/slice-19c').glob('*.json')):
101 if old
.name
== 'execution.json':
103 current
= retained
/ ('core.json' if old
.name
== 'validation.json' else 'core.json.' + old
.name
)
104 retained_equal(read(old
), read(current
))
105 verified
.append(str(old
.relative_to(ROOT
)))
106 for suffix
in ['validation', 'cxtm', 'integration']:
107 old
= ROOT
/ f
'reports/slice-19d2/{suffix}.json'
108 retained_equal(read(old
), read(retained
/ f
'xtm-mergemap.{suffix}.json'))
109 verified
.append(str(old
.relative_to(ROOT
)))
110 for stage
, directory
, suffixes
in [
111 ('slice20a', 'slice-20a', ['validation', 'cxtm', 'registration']),
112 ('slice20b', 'slice-20b', ['validation', 'cxtm', 'negative']),
113 ('slice20c', 'slice-20c', ['validation', 'cxtm', 'prolog']),
115 for suffix
in suffixes
:
116 old
= ROOT
/ f
'reports/{directory}/current.{suffix}.json'
117 assert read(old
) == read(retained
/ f
'{stage}.{suffix}.json'), str(old
)
118 verified
.append(str(old
.relative_to(ROOT
)))
119 before
= read(retained
/ 'slice20c.validation.json')
120 final
= read(output
/ 'current.validation.json')
121 registration
= read(output
/ 'current.registration.json')
122 cxtm
= read(output
/ 'current.cxtm.json')
123 assert before
['statuses'] == registration
['entryStatuses'] == {
124 'pass': 255, 'fail': 4, 'unsupported': 0, 'not-run': 14}
125 assert final
['statuses'] == {'pass': 256, 'fail': 4, 'unsupported': 0, 'not-run': 13}
126 assert registration
['passed'] == len(registration
['checks']) == len(set(registration
['checks']))
127 assert registration
['failed'] == 0 and len(registration
['cases']) == 1
128 row
= registration
['cases'][0]
129 assert row
['id'] == ID
and row
['sourceBytes'] == 13
130 assert row
['status'] == 'pass' and row
['actualOutcome'] == 'cxtm-match' and row
['diagnostics'] == []
131 assert row
['comparison']['actualBytes'] == row
['comparison']['expectedBytes'] == 23
132 assert row
['comparison']['firstDifferentByte'] is None
133 assert row
['comparison']['model'] == {'topicCount': 0, 'associationCount': 0, 'topics': []}
135 assert len(before
['cases']) == len(final
['cases']) == 273
136 for a
, b
in zip(before
['cases'], final
['cases']):
137 assert a
['id'] == b
['id']
139 assert a
== b
, a
['id']
141 assert a
['category'] == 'valid' and a
['expectedOutcome'] == 'cxtm-equal'
142 assert a
['status'] == 'not-run' and a
['actualOutcome'] is None
143 assert a
['documentIRI'] == row
['documentIRI'] == (
144 'https://ctm-tests.example.invalid/' + before
['commit'] + '/' + ID
)
145 assert b
== dict(a
, status
='pass', actualOutcome
='cxtm-match')
146 changes
.append({'id': ID
, 'before': a
['status'], 'after': b
['status']})
147 assert len(changes
) == 1
148 assert {k
: v
for k
, v
in before
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']} == {
149 k
: v
for k
, v
in final
.items() if k
not in ['cases', 'statuses', 'implementationScope', 'infrastructureChecks']}
150 old_cxtm
= read(retained
/ 'slice20c.cxtm.json')
151 assert [c
for c
in cxtm
['cases'] if c
['id'] != ID
] == old_cxtm
['cases']
152 assert cxtm
['contractCounts'] == {'pass': 218, 'fail': 4}
153 for case
in cxtm
['cases']:
154 pinned
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (Path(case
['id']).name
+ '.cxtm')
155 actual
= case
['actualCXTM'].encode('utf-8')
156 assert (actual
== pinned
.read_bytes()) == (case
['status'] == 'pass')
157 assert case
['expectedCXTM'].encode('utf-8') == pinned
.read_bytes()
158 comparison
= next(c
for c
in cxtm
['cases'] if c
['id'] == ID
)
159 assert comparison
['actualCXTM'].encode() == comparison
['expectedCXTM'].encode() == EXPECTED
160 log
= (output
/ 'regression.log').read_text()
161 prior_totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
162 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for marker
in prior_totals
}
163 assert totals
== prior_totals
and sum(totals
.values()) == 12100
164 for marker
, count
in [('XTM-MERGEMAP', 217), ('REGISTERED-ORIGINALS', 286),
165 ('REGISTERED-NEGATIVES', 25), ('PROLOG', 171), ('EMPTY2', registration
['passed'])]:
166 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == count
167 assert log
.count('EMPTY2-ORIGINAL ctm/in/empty2.ctm: PASS; 23 byte-exact CXTM bytes') == 1
168 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
and 'XTM-ADAPTER-BOUNDARY-PASS:' in log
169 xtm
= read(retained
/ 'xtm.json')
170 assert xtm
== read(ROOT
/ 'reports/slice-19d2/xtm.json')
171 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
172 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
173 lock
= read(ROOT
/ 'references/corpus-lock.json')
174 for entry
in lock
['files']:
175 path
= ROOT
/ 'vendor/cxtm' / entry
['path']
176 assert path
.stat().st_size
== entry
['size'] and sha(path
) == entry
['sha256'], str(path
)
177 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
178 assert historical
== {p
: sha(ROOT
/ p
) for p
in historical
}
179 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
180 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
181 save(output
/ 'remaining.json', [c
for c
in final
['cases'] if c
['status'] == 'not-run'])
182 save(output
/ 'integrity.json', {'protectedFilesSHA256': protected
, 'historicalReportsSHA256': historical
})
184 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
185 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
186 'runtime': 'SBCL 2.5.4', 'sourceSHA256': sources
, 'newDependencyDownloads': 0,
187 'sourcePreparationCommand': preparation
, 'qualifiedSourceProvenance': provenance
, 'runs': runs
,
188 'retainedLispChecks': sum(totals
.values()), 'retainedChecks': totals
,
189 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327, 'xtmMergemapChecks': 217,
190 'slice20aChecks': 286, 'slice20bChecks': 25, 'prologChecks': 171, 'empty2Checks': registration
['passed'],
191 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
192 'ctmStatusesBefore': before
['statuses'], 'ctmStatusesAfter': final
['statuses'], 'ctmStatusChanges': changes
,
193 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
194 'knownFailuresUnchanged': [c
['id'] for c
in final
['cases'] if c
['status'] == 'fail'],
195 'retainedReportsVerified': verified
, 'allHistoricalReportBytesUnchanged': True,
196 'allProtectedFileBytesUnchanged': True, 'pinnedCorpusFilesVerified': len(lock
['files']),
197 'originalComparison': {
198 'id': ID
, 'documentIRI': row
['documentIRI'], 'sourceBytes': len(SOURCE
), 'sourceSHA256': sha(source
),
199 'expectedBytes': len(EXPECTED
), 'expectedSHA256': sha(baseline
),
200 'actualBytes': len(EXPECTED
), 'actualSHA256': common
.digest(comparison
['actualCXTM'].encode()),
201 'firstDifferentByte': None, 'topics': 0, 'associations': 0,
202 'canonicalizationPreservesMap': True, 'executedInFullRegression': True,
204 'persistentLispImagesTouched': False,
205 'fullOfflineNixRebuild': 'NOT-RUN; fresh offline SBCL loads from verified local Store sources and unchanged XML patches.',
206 'sandboxStartError': 'writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.',
207 'executionAuthorization': 'Regular require_escalated auto-review approval; no symlink or sandbox configuration changes.',
208 'acceptancePassed': True,
210 save(output
/ 'execution.json', evidence
)
211 print(json
.dumps({k
: evidence
[k
] for k
in [
212 'runtime', 'retainedLispChecks', 'xmlQualificationChecks', 'standaloneXTMChecks', 'xtmMergemapChecks',
213 'slice20aChecks', 'slice20bChecks', 'prologChecks', 'empty2Checks', 'ctmStatusesAfter',
214 'xtmCorpusStatuses', 'originalComparison', 'pinnedCorpusFilesVerified', 'acceptancePassed']}, indent
=2))
217 if __name__
== '__main__':