2 """Register forty original CTM contracts after the complete retained offline suite."""
8 from pathlib
import Path
13 ROOT
= Path(__file__
).resolve().parent
.parent
14 BASE
= '9c196ddc16cf5cd727fcde81ad7835edefa0f787'
15 POLICY
= '(version 1)(allow default)(deny network*)'
17 'comments': ['comment-ml', 'comment-ml-nested', 'comment-ml-nested2', 'comment-sl',
18 'comment-sl2', 'comment-sl3', 'comment-sl4', 'comment-sl5', 'comment-sl6', 'empty'],
19 'identity': ['association-qname-iid', 'item-identifier-INF', 'item-identifier-relative',
20 'qname-date', 'qname-digits', 'rootless-uri', 'subject-identifier-dot',
21 'subject-identifier-non-http', 'subject-identifier-relative', 'subject-locator-dot',
22 'subject-locator-relative', 'topic-identifier-INF', 'topic-identifier-unicode'],
23 'occurrences': ['occurrence-decimal-dot-neg2', 'occurrence-decimal-dot-pos',
24 'occurrence-decimal-leading-zeros2', 'occurrence-duplicate-string',
25 'occurrence-duplicate-uri', 'occurrence-qname-value', 'occurrence-string-multiline',
26 'occurrence-string-multiline2', 'occurrence-uri-semicolon'],
27 'prefixes': ['prefix-multiline2', 'prefix-multiline3', 'prefix-multiline4', 'prefix-with-qm'],
28 'escapes': ['string-escape', 'string-escape2', 'unicode-escape-u4', 'unicode-escape-u6'],
31 'topic-maps.asd': '9013f4ace885dc049e98f674e8f59378d108b391b2aeac73fb54ee45119f8842',
32 'tests/ctm-registered-originals.lisp': '13eb134dde1620114e838f7595d1b1a0cfbd06624b1a0d23f5971441648e42e4',
37 return json
.loads(path
.read_text())
41 return hashlib
.sha256(data
).hexdigest()
45 return digest(path
.read_bytes())
49 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True).strip()
52 def save(path
, value
):
53 path
.write_text(json
.dumps(value
, indent
=2) + '\n')
57 return {str(p
.relative_to(ROOT
)): sha(p
) for p
in sorted(paths
) if p
.is_file()}
61 parser
= argparse
.ArgumentParser(description
=__doc__
)
62 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
63 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-20a')
64 args
= parser
.parse_args()
65 runtime
= shutil
.which(args
.sbcl
)
66 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
67 output
= args
.output
.resolve()
68 output
.mkdir(parents
=True, exist_ok
=False)
69 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
70 tracked_tests
= command(['git', 'ls-tree', '-r', '--name-only', BASE
, '--', 'tests']).splitlines()
71 protected_paths
= ['src', 'vendor', 'references', 'patches', 'flake.nix', 'flake.lock',
72 'scripts/test.sh', 'scripts/prepare-cxml.py', 'scripts/validate-slice-19d2.py'] + tracked_tests
73 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
74 protected
= snapshot(p
for directory
in ['src', 'vendor', 'references', 'patches']
75 for p
in (ROOT
/ directory
).rglob('*'))
76 historical
= snapshot(p
for p
in (ROOT
/ 'reports').rglob('*') if output
not in p
.parents
)
77 sources
= snapshot(p
for directory
in ['src', 'tests', 'scripts', 'docs']
78 for p
in (ROOT
/ directory
).rglob('*')
79 if p
.suffix
in ['.lisp', '.py', '.sh', '.md'])
80 sources
.update(snapshot(ROOT
/ p
for p
in ['topic-maps.asd', 'flake.nix', 'flake.lock', 'README.md']))
81 prefix
= output
/ 'current'
82 environment
= dict(os
.environ
, PYTHONDONTWRITEBYTECODE
='1',
83 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(prefix
))
84 # Reuse the unchanged qualification/regression driver, including its byte
85 # checks against historical reports. Its integration snapshot precedes 20A.
86 invocation
= ['python3', 'scripts/validate-slice-19d2.py', '--sbcl', runtime
,
87 '--output', str(output
/ 'retained')]
88 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=environment
, stdout
=subprocess
.PIPE
,
89 stderr
=subprocess
.STDOUT
, timeout
=600)
90 (output
/ 'retained-driver.log').write_bytes(result
.stdout
)
91 assert result
.returncode
== 0, 'Retained regression failed; see retained-driver.log and retained/*.log'
92 retained
= read(output
/ 'retained/execution.json')
93 old
= read(ROOT
/ 'reports/slice-19d2/validation.json')
94 before
= read(output
/ 'retained/validation.json')
95 old_cxtm
= read(ROOT
/ 'reports/slice-19d2/cxtm.json')
96 before_cxtm
= read(output
/ 'retained/cxtm.json')
98 assert before_cxtm
== old_cxtm
99 assert read(output
/ 'retained/integration.json') == read(ROOT
/ 'reports/slice-19d2/integration.json')
100 final
= read(Path(str(prefix
) + '.validation.json'))
101 cxtm
= read(Path(str(prefix
) + '.cxtm.json'))
102 registration
= read(Path(str(prefix
) + '.registration.json'))
103 ids
= ['ctm/in/' + name
+ '.ctm' for names
in GROUPS
.values() for name
in names
]
104 assert len(ids
) == len(set(ids
)) == 40
105 assert registration
['entryStatuses'] == before
['statuses'] == {
106 'pass': 205, 'fail': 4, 'unsupported': 0, 'not-run': 64}
107 assert final
['statuses'] == {'pass': 245, 'fail': 4, 'unsupported': 0, 'not-run': 24}
108 assert len(before
['cases']) == len(final
['cases']) == 273
109 assert [c
['id'] for c
in before
['cases']] == [c
['id'] for c
in final
['cases']]
111 for a
, b
in zip(before
['cases'], final
['cases']):
113 assert a
['status'] == 'not-run' and a
['actualOutcome'] is None
114 expected
= dict(a
, status
='pass', actualOutcome
='cxtm-match', diagnostics
=[])
115 assert b
== expected
, a
['id']
116 changes
.append({'id': a
['id'], 'before': a
['status'], 'after': b
['status']})
118 assert a
== b
, a
['id']
119 assert {c
['id'] for c
in changes
} == set(ids
)
120 assert {k
: v
for k
, v
in before
.items() if k
not in ['cases', 'statuses', 'implementationScope']} == {
121 k
: v
for k
, v
in final
.items() if k
not in ['cases', 'statuses', 'implementationScope']}
122 assert cxtm
['cases'][:len(before_cxtm
['cases'])] == before_cxtm
['cases']
123 assert cxtm
['contractCounts'] == {'pass': 217, 'fail': 4}
124 added
= cxtm
['cases'][len(before_cxtm
['cases']):]
125 assert [c
['id'] for c
in added
] == ids
== [c
['id'] for c
in registration
['cases']]
126 assert registration
['failed'] == 0
127 assert registration
['passed'] == len(registration
['checks']) == len(set(registration
['checks']))
128 log
= (output
/ 'retained/regression.log').read_text()
129 assert int(re
.search(r
'(?m)^REGISTERED-ORIGINALS-PASS: (\d+)', log
).group(1)) == registration
['passed']
131 (output
/ 'actual').mkdir()
132 for case
, row
in zip(added
, registration
['cases']):
133 name
= Path(case
['id']).stem
134 group
= next(group
for group
, names
in GROUPS
.items() if name
in names
)
135 source
= ROOT
/ 'vendor/cxtm' / case
['id']
136 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (name
+ '.ctm.cxtm')
137 expected
= baseline
.read_bytes()
138 actual
= case
['actualCXTM'].encode('utf-8')
139 assert case
['status'] == row
['status'] == 'pass'
140 assert row
['group'] == group
and row
['actualOutcome'] == 'cxtm-match' and row
['diagnostics'] == []
141 assert row
['sourcePathname'] == case
['id'] and row
['documentIRI'] == case
['baseLocator']
142 assert row
['baseline'] == str(baseline
.relative_to(ROOT
/ 'vendor/cxtm'))
143 assert actual
== expected
== case
['expectedCXTM'].encode('utf-8'), case
['id']
144 assert case
['byteCount'] == row
['comparison']['actualBytes'] == row
['comparison']['expectedBytes'] == len(expected
)
145 assert row
['comparison']['firstDifferentByte'] is None and row
['sourceBytes'] == len(source
.read_bytes())
146 (output
/ 'actual' / (name
+ '.ctm.cxtm')).write_bytes(actual
)
147 candidates
.append({'id': case
['id'], 'group': group
, 'status': 'pass',
148 'documentIRI': case
['baseLocator'], 'bytes': len(actual
),
149 'sourceSHA256': sha(source
), 'baselineSHA256': sha(baseline
),
150 'actualSHA256': digest(actual
), 'byteExact': True})
151 save(output
/ 'candidates.json', candidates
)
152 (output
/ 'contracts.md').write_text(
153 '# Slice 20A: actual original comparisons\n\n'
154 '| Original CTM | Group | Status | CXTM bytes |\n| --- | --- | --- | ---: |\n' +
155 ''.join(f
"| `{Path(c['id']).name}` | {c['group']} | PASS | {c['bytes']} |\n" for c
in candidates
) +
156 '\nAll forty actual outputs equal the unchanged pinned baselines byte for byte. '
157 'Individual source, baseline and actual SHA-256 digests are in `candidates.json`.\n')
158 xtm
= read(output
/ 'retained/xtm.json')
159 assert xtm
== read(ROOT
/ 'reports/slice-19d2/xtm.json')
160 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
161 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
162 remaining
= [c
for c
in final
['cases'] if c
['status'] == 'not-run']
163 assert len(remaining
) == 24
164 assert sum(c
['category'] == 'valid' for c
in remaining
) == 10
165 save(output
/ 'remaining.json', remaining
)
166 forms
= (ROOT
/ 'docs/slice-20a-registration.md').read_text().split('```lisp\n')[-1].split('\n```', 1)[0]
167 inspection_env
= dict(environment
, CL_SOURCE_REGISTRY
=retained
['qualifiedSourceProvenance']['sourceRegistry'])
168 inspection_command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform',
169 '--no-sysinit', '--no-userinit', '--script', '/dev/stdin']
170 inspection
= subprocess
.run(inspection_command
, cwd
=ROOT
, env
=inspection_env
, input=forms
.encode(),
171 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
, timeout
=120)
172 (output
/ 'inspection.log').write_bytes(inspection
.stdout
)
173 assert inspection
.returncode
== 0, 'Documented inspection failed; see inspection.log'
174 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
175 assert historical
== {p
: sha(ROOT
/ p
) for p
in historical
}
176 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
177 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
178 save(output
/ 'integrity.json', {'protectedFilesSHA256': protected
, 'historicalReportsSHA256': historical
})
180 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
181 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
182 'resumedUncommittedFileSHA256': BEGUN
, 'sourceSHA256': sources
,
183 'runtime': 'SBCL 2.5.4', 'newDependencyDownloads': 0,
184 'retainedValidationCommand': invocation
, 'retainedExecutionEvidence': 'retained/execution.json',
185 'retainedLispChecks': retained
['retainedLispChecks'],
186 'xmlQualificationChecks': next(r
['passedChecks'] for r
in retained
['runs'] if r
['label'] == 'xml'),
187 'standaloneXTMChecks': next(r
['passedChecks'] for r
in retained
['runs'] if r
['label'] == 'xtm'),
188 'xtmMergemapChecks': retained
['newXTMIntegrationChecks'],
189 'registeredOriginalChecks': registration
['passed'], 'newActualByteExactContracts': len(candidates
),
190 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294,
191 'network': 'All Lisp qualification, regression and inspection processes deny network* with sandbox-exec; Nix preparation uses --offline and verified existing Store sources.',
192 'ctmStatusesBefore': before
['statuses'], 'ctmStatusesAfter': final
['statuses'],
193 'ctmStatusChanges': changes
,
194 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
195 'remainingCases': 'remaining.json; ten positive and fourteen negative cases not selected or executed by Slice 20A',
196 'knownFailuresUnchanged': [c
['id'] for c
in final
['cases'] if c
['status'] == 'fail'],
197 'oldOriginalComparisonsUnchanged': len(before_cxtm
['cases']),
198 'allHistoricalReportBytesUnchanged': True, 'allProtectedFileBytesUnchanged': True,
199 'documentedInspectionPassed': True, 'persistentLispImagesTouched': False,
200 'fullOfflineNixRebuild': 'NOT-RUN; reused verified local Store sources, unchanged patches and fresh SBCL loads.',
202 save(output
/ 'execution.json', evidence
)
203 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'retainedLispChecks', 'xmlQualificationChecks',
204 'standaloneXTMChecks', 'xtmMergemapChecks', 'registeredOriginalChecks', 'newActualByteExactContracts',
205 'ctmStatusesAfter', 'xtmCorpusStatuses']}, indent
=2))
208 if __name__
== '__main__':