]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-8.py
Implement CTM Include and source-preserving template import
[topic-maps.git] / scripts / validate-slice-8.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 8: two fresh offline runtimes, relocation, report/retained-output checks.
3
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import json
11 import os
12 import pathlib
13 import re
14 import shutil
15 import subprocess
16 import sys
17 import tempfile
18 import unicodedata
19
20 ROOT = pathlib.Path(__file__).resolve().parent.parent
21 BASE = '17dd58775e25ab495e3c83deb4b5a36f5e5a124b'
22 POLICY = '(version 1)(allow default)(deny network*)'
23 SUFFIXES = ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json']
25 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals']
26
27
28 def digest(data):
29 return hashlib.sha256(data).hexdigest()
30
31
32 def main():
33 parser = argparse.ArgumentParser(description=__doc__)
34 parser.add_argument('--sbcl-254', required=True)
35 parser.add_argument('--sbcl-2410', required=True)
36 parser.add_argument('--output', default=str(ROOT / 'reports/slice-8'))
37 args = parser.parse_args()
38 output = pathlib.Path(args.output).resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected = {str(p.relative_to(ROOT)): digest(p.read_bytes())
43 for directory in ['vendor/cxtm', 'references']
44 for p in (ROOT / directory).rglob('*') if p.is_file()}
45 records = []
46 temporary = pathlib.Path(tempfile.mkdtemp(prefix='ctm-slice-8-validation-', dir='/private/tmp'))
47 relocated = temporary / 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil.copytree(ROOT, relocated, ignore=shutil.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-8'))
51 reports = []
52 logs = []
53 for version, runtime, checkout, log_name in [
54 ('2.5.4', args.sbcl_254, ROOT, 'tests-2.5.4.log'),
55 ('2.4.10', args.sbcl_2410, relocated, 'tests-2.4.10-relocated.log'),
56 ]:
57 version_result = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--version'],
58 check=True, capture_output=True, text=True)
59 assert version_result.stdout.strip() == 'SBCL ' + version, version_result.stdout
60 report = temporary / ('validation-' + version + '.json')
61 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(checkout / 'scripts/test.sh')]
63 result = subprocess.run(command, cwd=checkout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64 (output / log_name).write_bytes(result.stdout)
65 records.append(dict(command=command, workdir=str(checkout), exitCode=result.returncode, log=log_name))
66 if result.returncode:
67 raise RuntimeError(f'{version} failed: see {output / log_name}')
68 reports.append(report)
69 logs.append(result.stdout.decode('utf-8'))
70 print(f'OFFLINE-SBCL-{version}-PASS: {log_name}', flush=True)
71 for name, suffix in zip(NAMES, SUFFIXES):
72 a, b = [pathlib.Path(str(p) + suffix).read_bytes() for p in reports]
73 assert a == b, 'Reports differ after runtime change/relocation: ' + name
74 (output / (name + '.json')).write_bytes(a)
75 previous = ROOT / 'reports/slice-7'
76 for name in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers']:
77 old = json.loads((previous / (name + '.json')).read_text())
78 new = json.loads((output / (name + '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old['cases'] == new['cases'], 'Retained object observations changed: ' + name
82 old_cxtm = json.loads((previous / 'cxtm.json').read_text())
83 new_cxtm = json.loads((output / 'cxtm.json').read_text())
84 assert old_cxtm['cases'] == new_cxtm['cases'][:39]
85 assert old_cxtm['checks'][:445] == new_cxtm['checks'][:445]
86 actual_dir = output / 'actual'
87 actual_dir.mkdir(exist_ok=True)
88 for case in new_cxtm['cases']:
89 actual = case['actualCXTM'].encode('utf-8')
90 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
91 assert actual == baseline.read_bytes(), case['id']
92 (actual_dir / baseline.name).write_bytes(actual)
93 # Execute the exact public inspection example, suppressing only environment-
94 # dependent FASL location defaults. It uses no test-only API.
95 document = (ROOT / 'docs/reading-decimal-occurrences.md').read_text()
96 example = document.split('```lisp\n', 1)[1].split('\n```', 1)[0]
97 setup = '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
98 source = setup + example + '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
99 command = ['/usr/bin/sandbox-exec', '-p', POLICY, args.sbcl_254, '--noinform', '--no-sysinit',
100 '--no-userinit', '--script', '/dev/stdin']
101 result = subprocess.run(command, cwd=ROOT, input=source.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
102 (output / 'describe.log').write_bytes(result.stdout)
103 records.append(dict(command=command, workdir=str(ROOT), exitCode=result.returncode, log='describe.log',
104 input='Lisp fence in docs/reading-decimal-occurrences.md plus FASL setup/success marker',
105 inputSHA256=digest(source.encode('utf-8'))))
106 assert result.returncode == 0 and b'DOCUMENTED-INSPECTION-PASS' in result.stdout
107 for name, value in protected.items():
108 assert digest((ROOT / name).read_bytes()) == value, 'Protected file changed: ' + name
109 totals = {}
110 for key, marker in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
111 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
112 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS')]:
113 values = [int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for log in logs]
114 assert values[0] == values[1]
115 totals[key] = values[0]
116 assert [totals[k] for k in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
117 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665]
118 assert all('NFC-ORACLE-PASS: 16294' in log and 'Ran 2 tests' in log
119 and 'INTEGER-INDEPENDENT-PASS:' in log and 'DECIMAL-INDEPENDENT-PASS:' in log for log in logs)
120 totals.update(totalLispChecks=sum(totals.values()), combinedCXTMReportChecks=new_cxtm['passed'],
121 unicodeOracleComparisons=16294, pythonIntegrityTests=2, failures=0, pinnedCanonicalCases=46,
122 newPositiveCases=7, newNegativeCases=0, independentIntegerOracleProbes=143, independentDecimalOracleProbes=157,
123 importedStatuses=json.loads((output / 'validation.json').read_text())['statuses'],
124 allNineReportPairsByteIdentical=True, previous39ObjectObservationsAndCanonicalOutputsUnchanged=True,
125 previous445CanonicalChecksUnchanged=True, originalCorpusAndReferencesUnchanged=True,
126 documentedInspectionPassed=True)
127 artifacts = {str(p.relative_to(output)): digest(p.read_bytes()) for p in sorted(output.rglob('*'))
128 if p.is_file() and p.name != 'execution.json'}
129 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-8', validatedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
130 pythonVersion=sys.version, unicodeOracleVersion=unicodedata.unidata_version, commands=records, results=totals,
131 qualification='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
132 normativeInterpretation='Bare Integer/Decimal literals use canonical strings as documented reader policy; qualified strings retain lexical values. TMDM equality/order stays lexical; CXTM 3.16 formats Decimal output without mutation or floating point. Date/DateTime remain unsupported.',
133 artifactsSHA256=artifacts)
134 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
135 print(json.dumps(totals, sort_keys=True), flush=True)
136 print('SLICE-8-VALIDATION-PASS: offline dual runtime, relocation, 9 identical report pairs and previous 39 outputs unchanged', flush=True)
137
138
139 if __name__ == '__main__':
140 main()