]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-cxml-19d0b.py
Implement XTM occurrence reification with native conformance evidence
[topic-maps.git] / scripts / validate-cxml-19d0b.py
1 #!/usr/bin/env python3
2 """Reproduce the eight original failures; qualify local patches fully offline."""
3 import argparse
4 import datetime
5 import hashlib
6 import json
7 import os
8 from pathlib import Path
9 import re
10 import shutil
11 import subprocess
12 import tempfile
13
14 ROOT = Path(__file__).resolve().parent.parent
15 BASE = 'ec3f21e45c46849a6bb674194b6f7cc75b67bba1'
16 PIN = '50ab793786d9de88ee30ec4e4c24fb4236fc2674'
17 POLICY = '(version 1)(allow default)(deny network*)'
18 NIX_EVAL = ['nix', 'eval', '--offline', '--option', 'eval-cache', 'false']
19
20
21 def command(args):
22 return subprocess.check_output(args, cwd=ROOT, text=True).strip()
23
24
25 def sha(path):
26 return hashlib.sha256(path.read_bytes()).hexdigest()
27
28
29 def failures():
30 cases = []
31 for name, encoding, bom, label in [
32 ('utf16le-bom', 'utf-16le', b'\xff\xfe', 'UTF-16'),
33 ('utf16be-bom', 'utf-16be', b'\xfe\xff', 'UTF-16'),
34 ('utf16le-explicit-no-bom', 'utf-16le', b'', 'UTF-16LE'),
35 ('utf16be-explicit-no-bom', 'utf-16be', b'', 'UTF-16BE')]:
36 for content, suffix in [('Grüße 水', 'bmp'), ('Grüße 水 🦊', 'astral')]:
37 if bom and suffix == 'bmp':
38 continue
39 source = "<?xml version='1.0' encoding='" + label + "'?><r>" + content + '</r>'
40 cases.append({'name': name + ':' + suffix, 'source': source,
41 'octets': bom + source.encode(encoding), 'expectedText': content,
42 'declaredEncoding': label,
43 'cause': 'closure-common: UTF-16 second-unit check uses x instead of y'
44 if bom else 'closure-common: missing provisional XML signature; CXML: missing LE/BE declaration handling',
45 'functions': ['closure-common/encodings.lisp:decode-sequence'] if bom else
46 ['closure-common/xstream.lisp:figure-encoding', 'cxml/xml/xml-parse.lisp:setup-encoding'],
47 'normative': ['XML 1.0 §2.2, §4.3.3', 'Unicode §3.9.2, §3.10', 'RFC 2781 §2.2, §3.3']})
48 cases.append({'name': 'utf16-bom-without-declaration:astral', 'source': '<r>水🦊</r>',
49 'octets': b'\xff\xfe' + '<r>水🦊</r>'.encode('utf-16le'), 'expectedText': '水🦊',
50 'declaredEncoding': None, 'cause': 'closure-common: UTF-16 second-unit check uses x instead of y',
51 'functions': ['closure-common/encodings.lisp:decode-sequence'],
52 'normative': ['XML 1.0 §2.2, §4.3.3', 'Unicode §3.9.2', 'RFC 2781 §2.2']})
53 cases.append({'name': 'invalid-utf8', 'source': None, 'octets': b'<r>\xc3\x28</r>',
54 'expectedText': None, 'declaredEncoding': None,
55 'cause': 'closure-common: UTF-8 continuation bytes masked without validation',
56 'functions': ['closure-common/encodings.lisp:decode-sequence'],
57 'normative': ['XML 1.0 §4.3.3', 'Unicode §3.9.3 / Table 3-7', 'RFC 3629 §3–4']})
58 return cases
59
60
61 def run_lisp(runtime, registry, output, label, *, source=None, script=None):
62 invocation = ['/usr/bin/sandbox-exec', '-p', POLICY, runtime,
63 '--noinform', '--no-sysinit', '--no-userinit', '--script',
64 str(script) if script else '/dev/stdin']
65 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
66 result = subprocess.run(invocation, cwd=ROOT,
67 env=dict(os.environ, CL_SOURCE_REGISTRY=registry, PYTHONDONTWRITEBYTECODE='1'),
68 input=source.encode() if source else None,
69 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
70 (output / (label + '.log')).write_bytes(result.stdout)
71 return result, {'label': label, 'command': invocation, 'exitCode': result.returncode,
72 'freshProcess': True, 'network': 'denied by sandbox-exec',
73 'startedAtUTC': started,
74 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat()}
75
76
77 def checks(result, context):
78 log = result.stdout.decode()
79 passed = re.findall(r'^CXML-CHECK-PASS (.+)$', log, re.M)
80 failed = re.findall(r'^CXML-CHECK-FAIL (.+)$', log, re.M)
81 assert len(set(passed + failed)) == len(passed + failed)
82 assert 'CXML-SUMMARY ' + str(len(passed)) + ' PASS ' + str(len(failed)) + ' FAIL' in log
83 assert (result.returncode == 0) == (not failed)
84 return dict(context, passedChecks=passed, failedChecks=failed)
85
86
87 def diagnostic_source(cases):
88 source = '''(require :asdf)
89 (setf *compile-verbose* nil *load-verbose* nil)
90 (asdf:load-system "cxml")
91 (defun diagnose (id bytes)
92 (format t "DIAG-ENCODING ~A ~A~%" id
93 (runes::figure-encoding (cxml::make-octet-input-stream bytes)))
94 (handler-case
95 (let* ((node (cxml:parse-octets bytes (cxml-xmls:make-xmls-builder)
96 :disallow-internal-subset t
97 :entity-resolver (lambda (p s) (declare (ignore p s)) (error "External resource forbidden"))))
98 (text (apply #'concatenate 'string (cxml-xmls:node-children node))))
99 (format t "DIAG-ACCEPTED ~A ~{~X~^,~}~%" id (map 'list #'char-code text)))
100 (error (c) (format t "DIAG-REJECTED ~A ~A ~A~%" id (type-of c)
101 (substitute #\\Space #\\Newline (princ-to-string c))))))
102 '''
103 for c in cases:
104 source += '(diagnose ' + json.dumps(c['name']) + ' (make-array ' + str(len(c['octets']))
105 source += " :element-type '(unsigned-byte 8) :initial-contents '(" + ' '.join(map(str, c['octets'])) + ')))\n'
106 return source
107
108
109 def read_diagnostics(result, cases):
110 assert result.returncode == 0
111 lines = result.stdout.decode().splitlines()
112 observations = {}
113 for c in cases:
114 name = c['name']
115 encoding = next(l.split(' ', 2)[2] for l in lines if l.startswith('DIAG-ENCODING ' + name + ' '))
116 accepted = next((l for l in lines if l.startswith('DIAG-ACCEPTED ' + name + ' ')), None)
117 if accepted:
118 codes = accepted.split(' ', 2)[2]
119 observations[name] = {'recognizedEncoding': encoding, 'outcome': 'accepted',
120 'text': ''.join(chr(int(n, 16)) for n in codes.split(','))}
121 else:
122 rejected = next(l.split(' ', 3) for l in lines if l.startswith('DIAG-REJECTED ' + name + ' '))
123 observations[name] = {'recognizedEncoding': encoding, 'outcome': 'rejected',
124 'condition': rejected[2], 'diagnostic': rejected[3]}
125 return observations
126
127
128 def main():
129 parser = argparse.ArgumentParser(description=__doc__)
130 parser.add_argument('--sbcl', default=os.environ.get('SBCL', 'sbcl'))
131 parser.add_argument('--output', type=Path, default=ROOT / 'reports/slice-19d0b')
132 args = parser.parse_args()
133 runtime = shutil.which(args.sbcl)
134 assert runtime and command([runtime, '--version']) == 'SBCL 2.5.4'
135 output = args.output.resolve()
136 output.mkdir(parents=True, exist_ok=True)
137 started = datetime.datetime.now(datetime.timezone.utc).isoformat()
138 system = command(NIX_EVAL + ['--impure', '--raw', '--expr', 'builtins.currentSystem'])
139 environment = json.loads(command(NIX_EVAL + ['--json', '.#lib.cxmlEnvironment',
140 '--apply', 'f: f "' + system + '"']))
141 assert json.loads((ROOT / 'flake.lock').read_text())['nodes']['nixpkgs']['locked']['rev'] == PIN
142 snapshot = subprocess.check_output(['git', 'show', BASE + ':tests/cxml-probe.lisp'], cwd=ROOT).decode()
143 original_evidence = json.loads(command(['git', 'show', BASE + ':reports/slice-19d0/execution.json']))
144 expected_checks = set(original_evidence['xmlRuns'][0]['checks'] + original_evidence['xmlRuns'][0]['failedChecks'])
145 protected = {str(p.relative_to(ROOT)): sha(p)
146 for directory in ['vendor/cxtm', 'references', 'reports/slice-19d0']
147 for p in (ROOT / directory).rglob('*') if p.is_file()}
148 files = ['flake.nix', 'flake.lock', 'topic-maps.asd', 'scripts/prepare-cxml.py',
149 'scripts/validate-cxml-19d0b.py', 'tests/cxml-probe.lisp',
150 'tests/cxml-encoding-regressions.lisp', 'src/xml/package.lisp', 'src/xml/parser.lisp',
151 'patches/cxml-19d0b.patch', 'patches/closure-common-19d0b.patch',
152 'README.md', 'docs/slice-19d0b-cxml-encoding.md']
153 source_hashes = {p: sha(ROOT / p) for p in files}
154 cases = failures()
155 with tempfile.TemporaryDirectory(prefix='topic-maps-19d0b-') as temp:
156 temp = Path(temp)
157 patched = []
158 for number in [1, 2]:
159 patched.append(json.loads(command(['python3', str(ROOT / 'scripts/prepare-cxml.py'),
160 '--directory', str(temp / ('sources-' + str(number)))])))
161 assert patched[0]['patches'] == patched[1]['patches']
162 pristine, context = run_lisp(runtime, environment['sourceRegistry'], output, 'original-58', source=snapshot)
163 baseline = checks(pristine, context)
164 assert len(baseline['passedChecks']) == 50 and len(baseline['failedChecks']) == 8
165 assert set(baseline['failedChecks']) == {c['name'] for c in cases}
166 before_result, before_context = run_lisp(runtime, environment['sourceRegistry'], output,
167 'original-diagnostics', source=diagnostic_source(cases))
168 before = read_diagnostics(before_result, cases)
169 unqualified_source = '''(require :asdf)
170 (asdf:load-asd (truename "topic-maps.asd"))
171 (asdf:load-system "topic-maps/xml")
172 (handler-case
173 (progn (topic-maps.xml:parse-octets
174 (make-array 4 :element-type '(unsigned-byte 8) :initial-contents '(60 114 47 62)) nil)
175 (error "Unpatched dependency unexpectedly accepted"))
176 (topic-maps.xml:unqualified-parser () (format t "UNPATCHED-BOUNDARY-PASS~%")))
177 '''
178 unqualified_result, unqualified_context = run_lisp(
179 runtime, environment['sourceRegistry'], output, 'unqualified-boundary', source=unqualified_source)
180 assert unqualified_result.returncode == 0 and b'UNPATCHED-BOUNDARY-PASS' in unqualified_result.stdout
181 raw_result, context = run_lisp(runtime, patched[0]['sourceRegistry'], output,
182 'patched-original-58', source=snapshot)
183 raw = checks(raw_result, context)
184 assert not raw['failedChecks'] and set(raw['passedChecks']) == expected_checks
185 xml_runs = []
186 for number, provenance in enumerate(patched, 1):
187 result, context = run_lisp(runtime, provenance['sourceRegistry'], output,
188 'qualified-' + str(number), script=ROOT / 'tests/cxml-probe.lisp')
189 run = checks(result, context)
190 assert not run['failedChecks'] and expected_checks <= set(run['passedChecks'])
191 xml_runs.append(run)
192 assert xml_runs[0]['passedChecks'] == xml_runs[1]['passedChecks']
193 after_result, after_context = run_lisp(runtime, patched[0]['sourceRegistry'], output,
194 'patched-diagnostics', source=diagnostic_source(cases))
195 after = read_diagnostics(after_result, cases)
196 failure_report = []
197 for c in cases:
198 observation = after[c['name']]
199 assert (observation['outcome'] == 'accepted') == (c['expectedText'] is not None)
200 if c['expectedText'] is not None:
201 assert observation['text'] == c['expectedText']
202 else:
203 assert observation['condition'] == 'WELL-FORMEDNESS-VIOLATION'
204 failure_report.append({k: v for k, v in c.items() if k != 'octets'} | {
205 'inputOctetsHex': c['octets'].hex(), 'inputLength': len(c['octets']),
206 'inputSHA256': hashlib.sha256(c['octets']).hexdigest(),
207 'before': before[c['name']], 'after': observation})
208 (output / 'original-failures.json').write_text(json.dumps(failure_report, ensure_ascii=False, indent=2) + '\n')
209 invocation = ['python3', str(ROOT / 'scripts/validate-slice-19c.py'), '--sbcl', runtime,
210 '--output', str(temp / 'regression')]
211 result = subprocess.run(invocation, cwd=ROOT,
212 env=dict(os.environ, CL_SOURCE_REGISTRY=patched[0]['sourceRegistry'], PYTHONDONTWRITEBYTECODE='1'),
213 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
214 (output / 'regression-driver.log').write_bytes(result.stdout)
215 assert result.returncode == 0, 'Retained regression failed'
216 regression = json.loads((temp / 'regression/execution.json').read_text())
217 assert regression['importedStatuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
218 assert regression['totalLispChecks'] == 12100
219 identical_reports = []
220 for previous in sorted((ROOT / 'reports/slice-19c').glob('*.json')):
221 if previous.name != 'execution.json':
222 assert (temp / 'regression' / previous.name).read_bytes() == previous.read_bytes(), previous.name
223 identical_reports.append(previous.name)
224 for name in ['execution.json', 'tests.log']:
225 shutil.copyfile(temp / 'regression' / name, output / ('regression-' + name))
226 assert not command(['git', 'diff', BASE, '--', 'src/tmdm', 'src/ctm', 'src/cxtm',
227 'vendor', 'references', 'flake.lock', 'scripts/test.sh'])
228 assert protected == {p: sha(ROOT / p) for p in protected}
229 assert source_hashes == {p: sha(ROOT / p) for p in source_hashes}
230 for package in environment['manifest']:
231 assert command(['nix', 'hash', 'path', '--type', 'sha256', '--base32', package['source']]) == package['sha256']
232 evidence = {'baseCommit': BASE, 'branch': command(['git', 'branch', '--show-current']),
233 'startedAtUTC': started, 'finishedAtUTC': datetime.datetime.now(datetime.timezone.utc).isoformat(),
234 'runtime': 'SBCL 2.5.4', 'nixpkgsCommit': PIN, 'nixSystem': system,
235 'sourceEnvironment': environment, 'sourceSHA256': source_hashes,
236 'sourceHashesVerifiedBeforeAndAfter': True, 'newDependencyNetworkAcquisitions': 0,
237 'patchReproducibleAcrossTwoCopies': True, 'patchProvenance': patched,
238 'temporarySourceContext': 'Effective source copies existed only during these recorded processes; reproduce with prepare-cxml.py.',
239 'originalQualification': baseline, 'patchedUnmodifiedOriginalQualification': raw,
240 'originalProbeSHA256': hashlib.sha256(snapshot.encode()).hexdigest(),
241 'unpatchedBoundaryRefusal': unqualified_context,
242 'original58ExpectationsChanged': False, 'xmlRuns': xml_runs,
243 'originalDiagnosticContext': before_context, 'patchedDiagnosticContext': after_context,
244 'xmlQualification': 'PASS', 'xmlChecksPerRun': len(xml_runs[0]['passedChecks']), 'xmlFailedChecks': 0,
245 'bindingPublicContract': 'topic-maps.xml:parse-octets; verified patched dependencies; raw octets; mandatory namespace processing and DOCTYPE/entity denial; no caller overrides.',
246 'regressionChecks': regression['regressionChecks'], 'totalLispChecks': regression['totalLispChecks'],
247 'importedStatuses': regression['importedStatuses'], 'byteIdenticalRetainedReports': identical_reports,
248 'newXTMContracts': 0, 'pinnedCorpusAndOriginalArchivesUnchanged': True,
249 'fullOfflineNixRebuild': 'NOT-RUN; existing Nix artifacts plus offline source patching/ASDF compilation.'}
250 (output / 'execution.json').write_text(json.dumps(evidence, indent=2) + '\n')
251 print(json.dumps({k: evidence[k] for k in ['runtime', 'xmlQualification', 'xmlChecksPerRun',
252 'xmlFailedChecks', 'original58ExpectationsChanged', 'totalLispChecks', 'importedStatuses']}, indent=2))
253
254
255 if __name__ == '__main__':
256 main()