]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-cxml.py
2 """Qualify pinned CXML sources in fresh, network-denied SBCL 2.5.4 processes."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= 'bc4fcf0f126bab6567b37a21fc8c8eab2944c152'
16 PIN
= '50ab793786d9de88ee30ec4e4c24fb4236fc2674'
17 POLICY
= '(version 1)(allow default)(deny network*)'
18 # Avoid a local Nix evaluator-cache dynamic-derivations error; no cache or
19 # global Nix setting is changed, and no derivation is built here.
20 NIX_EVAL
= ['nix', 'eval', '--offline', '--option', 'eval-cache', 'false']
23 def command(args
, **kwargs
):
24 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True, **kwargs
).strip()
28 return hashlib
.sha256(path
.read_bytes()).hexdigest()
32 parser
= argparse
.ArgumentParser(description
=__doc__
)
33 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
34 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-19d0')
35 args
= parser
.parse_args()
36 runtime
= shutil
.which(args
.sbcl
)
37 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
38 output
= args
.output
.resolve()
39 output
.mkdir(parents
=True, exist_ok
=True)
40 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
41 lock
= json
.loads((ROOT
/ 'flake.lock').read_text())
42 assert lock
['nodes']['nixpkgs']['locked']['rev'] == PIN
43 system
= command(NIX_EVAL
+ ['--impure', '--raw',
44 '--expr', 'builtins.currentSystem'])
45 # Evaluate the same source registry exported by the DevShell. This does
46 # not realize the (separate, not fully cached) default toolchain shell.
47 environment
= json
.loads(command(NIX_EVAL
+ [
48 '--json', '.#lib.cxmlEnvironment',
49 '--apply', 'f: f "' + system
+ '"']))
50 registry
= command(NIX_EVAL
+ ['--raw',
51 '.#devShells.' + system
+ '.default.CL_SOURCE_REGISTRY'])
52 assert registry
== environment
['sourceRegistry']
53 expected
= {'cxml': '20200610-git', 'closure-common': '20181018-git',
54 'babel': '20241012-git', 'alexandria': '20241012-git',
55 'trivial-features': '20230618-git',
56 'trivial-gray-streams': '20241012-git', 'puri': '20201016-git'}
57 assert {p
['name']: p
['version'] for p
in environment
['manifest']} == expected
58 for package
in environment
['manifest']:
59 assert Path(package
['source']).is_dir(), 'Source not provisioned: ' + package
['name']
60 actual
= command(['nix', 'hash', 'path', '--type', 'sha256', '--base32', package
['source']])
61 assert actual
== package
['sha256'], 'SOURCE HASH MISMATCH: ' + package
['name']
62 package
['verifiedNarSHA256'] = actual
63 protected
= {str(p
.relative_to(ROOT
)): sha(p
)
64 for directory
in ['vendor/cxtm', 'references']
65 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
66 files
= ['flake.nix', 'flake.lock', 'tests/cxml-probe.lisp', 'scripts/validate-cxml.py']
67 source_hashes
= {p
: sha(ROOT
/ p
) for p
in files
}
68 env
= dict(os
.environ
, CL_SOURCE_REGISTRY
=registry
, PYTHONDONTWRITEBYTECODE
='1')
71 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
,
72 '--noinform', '--no-sysinit', '--no-userinit', '--script',
73 str(ROOT
/ 'tests/cxml-probe.lisp')]
74 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
,
75 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
76 log_name
= 'cxml-run-' + str(number
) + '.log'
77 (output
/ log_name
).write_bytes(result
.stdout
)
78 log
= result
.stdout
.decode('utf-8')
79 checks
= re
.findall(r
'^CXML-CHECK-PASS (.+)$', log
, re
.M
)
80 failed
= re
.findall(r
'^CXML-CHECK-FAIL (.+)$', log
, re
.M
)
81 assert checks
and len(checks
+ failed
) == len(set(checks
+ failed
))
82 assert 'CXML-SUMMARY ' + str(len(checks
)) + ' PASS ' + str(len(failed
)) + ' FAIL' in log
83 assert (result
.returncode
== 0) == (not failed
), 'Unclassified probe execution failure'
84 runs
.append({'run': number
, 'exitCode': result
.returncode
,
85 'freshProcess': True, 'network': 'denied by sandbox-exec',
86 'checks': checks
, 'failedChecks': failed
,
87 'command': invocation
, 'log': log_name
})
88 assert runs
[0]['checks'] == runs
[1]['checks']
89 assert runs
[0]['failedChecks'] == runs
[1]['failedChecks']
90 # Retain the established full regression command/harness unchanged.
91 with tempfile
.TemporaryDirectory(prefix
='topic-maps-19d0-regression-') as temp
:
92 invocation
= ['python3', str(ROOT
/ 'scripts/validate-slice-19c.py'),
93 '--sbcl', runtime
, '--output', temp
]
94 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
,
95 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
96 (output
/ 'regression-driver.log').write_bytes(result
.stdout
)
97 assert result
.returncode
== 0, 'Retained regression failed'
98 temporary
= Path(temp
)
99 regression
= json
.loads((temporary
/ 'execution.json').read_text())
100 assert regression
['importedStatuses'] == {
101 'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
102 assert regression
['totalLispChecks'] == 12100
103 identical_reports
= []
104 for previous
in sorted((ROOT
/ 'reports/slice-19c').glob('*.json')):
105 if previous
.name
== 'execution.json':
107 assert (temporary
/ previous
.name
).read_bytes() == previous
.read_bytes(), previous
.name
108 identical_reports
.append(previous
.name
)
109 for name
in ['execution.json', 'tests.log']:
110 shutil
.copyfile(temporary
/ name
, output
/ ('regression-' + name
))
111 assert not command(['git', 'diff', BASE
, '--', 'src', 'topic-maps.asd',
112 'vendor', 'references', 'flake.lock', 'scripts/test.sh'])
113 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
114 assert source_hashes
== {p
: sha(ROOT
/ p
) for p
in source_hashes
}
116 'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
117 'startedAtUTC': started
,
118 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
119 'runtime': 'SBCL 2.5.4', 'nixSystem': system
, 'nixpkgsCommit': PIN
,
120 'sourceEnvironment': environment
, 'sourceSHA256': source_hashes
,
121 'sourceHashesVerified': True, 'devShellRegistryEqualsTestRegistry': True,
123 'xmlChecksPerRun': len(runs
[0]['checks']) + len(runs
[0]['failedChecks']),
124 'xmlPassedChecksPerRun': len(runs
[0]['checks']),
125 'xmlFailedChecksPerRun': len(runs
[0]['failedChecks']),
126 'xmlQualification': 'FAIL' if runs
[0]['failedChecks'] else 'PASS',
127 'encodingCounterexamples': {
128 'utf16leAstral': {'character': '\U0001f98a', 'bytesHex': '\U0001f98a'.encode('utf-16le').hex(),
129 'strictPythonRoundTrip': bytes
.fromhex('3ed88add').decode('utf-16le') == '\U0001f98a'},
130 'invalidUtf8': {'bytesHex': 'c328', 'strictPythonRejection': False},
132 'securityPolicy': 'Reject every DOCTYPE; disallow internal subsets; '
133 'resolver signals external-resource-forbidden; no NIL fallback. '
134 'Catalog disabled and DTD cache fresh for each parse.',
135 'securityCounterprobe': 'NIL resolver reaches trapped CXML file-opening fallback; '
136 'rejecting resolver never reaches it.',
137 'sourceProvisioning': 'Only CXML and closure-common downloaded with authorized '
138 'hash-checked nix store prefetch-file; five sources already present.',
139 'defaultDevShellBuild': 'NOT-RUN: existing pinned toolchain not fully provisioned',
140 'fullOfflineRebuild': 'NOT-RUN: existing Nix SBCL and source artifacts used',
141 'regressionChecks': regression
['regressionChecks'],
142 'totalLispChecks': regression
['totalLispChecks'],
143 'importedStatuses': regression
['importedStatuses'],
144 'byteIdenticalRetainedReports': identical_reports
,
145 'newXTMContracts': 0, 'pinnedCorpusAndNormativeReferencesUnchanged': True,
146 'tmdmCtmCxtmAndExistingHarnessUnchanged': True,
149 bytes
.fromhex('c328').decode('utf-8')
150 except UnicodeDecodeError:
151 evidence
['encodingCounterexamples']['invalidUtf8']['strictPythonRejection'] = True
152 assert evidence
['encodingCounterexamples']['utf16leAstral']['strictPythonRoundTrip']
153 assert evidence
['encodingCounterexamples']['invalidUtf8']['strictPythonRejection']
154 (output
/ 'execution.json').write_text(json
.dumps(evidence
, indent
=2) + '\n')
155 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'xmlChecksPerRun', 'xmlPassedChecksPerRun',
156 'xmlFailedChecksPerRun', 'xmlQualification', 'totalLispChecks', 'importedStatuses',
157 'defaultDevShellBuild', 'fullOfflineRebuild']}, indent
=2))
158 # A successful regression/provisioning must never conceal failed parser
159 # qualification. Preserve all evidence first, then report failure to CI.
160 if evidence
['xmlQualification'] != 'PASS':
164 if __name__
== '__main__':