]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-9.py
test(ctm): register forty original Slice 20A contracts
[topic-maps.git] / scripts / validate-slice-9.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 9: two fresh offline runtimes, relocation, report/retained-output checks.
3
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import json
11 import os
12 import pathlib
13 import re
14 import shutil
15 import subprocess
16 import sys
17 import tempfile
18 import unicodedata
19
20 ROOT = pathlib.Path(__file__).resolve().parent.parent
21 BASE = '195d5f5444d776096b765d04c815c23c3db5f84c'
22 POLICY = '(version 1)(allow default)(deny network*)'
23 SUFFIXES = ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json', '.dates.json']
25 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates']
26
27
28 def digest(data):
29 return hashlib.sha256(data).hexdigest()
30
31
32 def main():
33 parser = argparse.ArgumentParser(description=__doc__)
34 parser.add_argument('--sbcl-254', required=True)
35 parser.add_argument('--sbcl-2410', required=True)
36 parser.add_argument('--output', default=str(ROOT / 'reports/slice-9'))
37 args = parser.parse_args()
38 output = pathlib.Path(args.output).resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected = {str(p.relative_to(ROOT)): digest(p.read_bytes())
43 for directory in ['vendor/cxtm', 'references']
44 for p in (ROOT / directory).rglob('*') if p.is_file()}
45 records = []
46 temporary = pathlib.Path(tempfile.mkdtemp(prefix='ctm-slice-9-validation-', dir='/private/tmp'))
47 relocated = temporary / 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil.copytree(ROOT, relocated, ignore=shutil.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-9'))
51 reports = []
52 logs = []
53 for version, runtime, checkout, log_name in [
54 ('2.5.4', args.sbcl_254, ROOT, 'tests-2.5.4.log'),
55 ('2.4.10', args.sbcl_2410, relocated, 'tests-2.4.10-relocated.log'),
56 ]:
57 version_result = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--version'],
58 check=True, capture_output=True, text=True)
59 assert version_result.stdout.strip() == 'SBCL ' + version, version_result.stdout
60 report = temporary / ('validation-' + version + '.json')
61 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(checkout / 'scripts/test.sh')]
63 result = subprocess.run(command, cwd=checkout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64 (output / log_name).write_bytes(result.stdout)
65 records.append(dict(command=command, workdir=str(checkout), exitCode=result.returncode, log=log_name))
66 if result.returncode:
67 raise RuntimeError(f'{version} failed: see {output / log_name}')
68 reports.append(report)
69 logs.append(result.stdout.decode('utf-8'))
70 print(f'OFFLINE-SBCL-{version}-PASS: {log_name}', flush=True)
71 for name, suffix in zip(NAMES, SUFFIXES):
72 a, b = [pathlib.Path(str(p) + suffix).read_bytes() for p in reports]
73 assert a == b, 'Reports differ after runtime change/relocation: ' + name
74 (output / (name + '.json')).write_bytes(a)
75 previous = ROOT / 'reports/slice-8'
76 for name in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals']:
77 old = json.loads((previous / (name + '.json')).read_text())
78 new = json.loads((output / (name + '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old['cases'] == new['cases'], 'Retained object observations changed: ' + name
82 old_cxtm = json.loads((previous / 'cxtm.json').read_text())
83 new_cxtm = json.loads((output / 'cxtm.json').read_text())
84 assert old_cxtm['cases'] == new_cxtm['cases'][:46]
85 assert old_cxtm['checks'][:445] == new_cxtm['checks'][:445]
86 actual_dir = output / 'actual'
87 actual_dir.mkdir(exist_ok=True)
88 for case in new_cxtm['cases']:
89 actual = case['actualCXTM'].encode('utf-8')
90 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
91 assert actual == baseline.read_bytes(), case['id']
92 (actual_dir / baseline.name).write_bytes(actual)
93 # Execute the exact public inspection example, suppressing only environment-
94 # dependent FASL location defaults. It uses no test-only API.
95 document = (ROOT / 'docs/reading-date-occurrences.md').read_text()
96 example = document.split('```lisp\n', 1)[1].split('\n```', 1)[0]
97 setup = '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
98 source = setup + example + '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
99 command = ['/usr/bin/sandbox-exec', '-p', POLICY, args.sbcl_254, '--noinform', '--no-sysinit',
100 '--no-userinit', '--script', '/dev/stdin']
101 result = subprocess.run(command, cwd=ROOT, input=source.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
102 (output / 'describe.log').write_bytes(result.stdout)
103 records.append(dict(command=command, workdir=str(ROOT), exitCode=result.returncode, log='describe.log',
104 input='Lisp fence in docs/reading-date-occurrences.md plus FASL setup/success marker',
105 inputSHA256=digest(source.encode('utf-8'))))
106 assert result.returncode == 0 and b'DOCUMENTED-INSPECTION-PASS' in result.stdout
107 for name, value in protected.items():
108 assert digest((ROOT / name).read_bytes()) == value, 'Protected file changed: ' + name
109 totals = {}
110 for key, marker in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
111 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
112 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS'), ('dateChecks', 'DATES')]:
113 values = [int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for log in logs]
114 assert values[0] == values[1]
115 totals[key] = values[0]
116 assert [totals[k] for k in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
117 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks', 'decimalChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665, 715]
118 assert all('NFC-ORACLE-PASS: 16294' in log and 'Ran 2 tests' in log
119 and 'INTEGER-INDEPENDENT-PASS:' in log and 'DECIMAL-INDEPENDENT-PASS:' in log and 'DATE-INDEPENDENT-PASS:' in log for log in logs)
120 totals.update(totalLispChecks=sum(totals.values()), combinedCXTMReportChecks=new_cxtm['passed'],
121 unicodeOracleComparisons=16294, pythonIntegrityTests=2, failures=0, pinnedCanonicalCases=51,
122 newPositiveCases=5, newNegativeCases=5, independentIntegerOracleProbes=143, independentDecimalOracleProbes=157,
123 independentDateOracleProbes=len(json.loads((output / 'dates.json').read_text())['lexicalProbes']),
124 importedStatuses=json.loads((output / 'validation.json').read_text())['statuses'],
125 allTenReportPairsByteIdentical=True, previous46ObjectObservationsAndCanonicalOutputsUnchanged=True,
126 previous445CanonicalChecksUnchanged=True, originalCorpusAndReferencesUnchanged=True,
127 documentedInspectionPassed=True)
128 artifacts = {str(p.relative_to(output)): digest(p.read_bytes()) for p in sorted(output.rglob('*'))
129 if p.is_file() and p.name != 'execution.json'}
130 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-9', validatedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
131 pythonVersion=sys.version, unicodeOracleVersion=unicodedata.unidata_version, commands=records, results=totals,
132 qualification='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
133 normativeInterpretation='Bare Integer/Decimal/Date literals use canonical strings as documented reader policy; qualified strings retain lexical values. TMDM equality/order stays lexical; CXTM 3.16 formats timezone-free Date output without mutation; Gregorian validation uses no system calendar. DateTime/date-timezone conversion remain unsupported. Published CTM 3.4.1 XSDT validity is used for days 01-09; the HTML omission and unavailable published Annex grammar are documented. BCE leap validation uses the E.1 signed lexical year interpretation; XSD 1.0 ambiguity is qualified.',
134 artifactsSHA256=artifacts)
135 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
136 print(json.dumps(totals, sort_keys=True), flush=True)
137 print('SLICE-9-VALIDATION-PASS: offline dual runtime, relocation, 10 identical report pairs and previous 46 outputs unchanged', flush=True)
138
139
140 if __name__ == '__main__':
141 main()