]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-11.py
Support arbitrary absolute XTM resourceData datatypes (Slice 22D)
[topic-maps.git] / scripts / validate-slice-11.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 11: two fresh offline runtimes, relocation, report/retained-output checks.
3
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import json
11 import os
12 import pathlib
13 import re
14 import shutil
15 import subprocess
16 import sys
17 import tempfile
18 import unicodedata
19
20 ROOT = pathlib.Path(__file__).resolve().parent.parent
21 BASE = '8741d155d948c7317945eee423a977940cad995e'
22 POLICY = '(version 1)(allow default)(deny network*)'
23 SUFFIXES = ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json', '.dates.json', '.datetimes.json', '.timezones.json']
25 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones']
26
27
28 def digest(data):
29 return hashlib.sha256(data).hexdigest()
30
31
32 def main():
33 parser = argparse.ArgumentParser(description=__doc__)
34 parser.add_argument('--sbcl-254', required=True)
35 parser.add_argument('--sbcl-2410', required=True)
36 parser.add_argument('--output', default=str(ROOT / 'reports/slice-11'))
37 args = parser.parse_args()
38 output = pathlib.Path(args.output).resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected = {str(p.relative_to(ROOT)): digest(p.read_bytes())
43 for directory in ['vendor/cxtm', 'references']
44 for p in (ROOT / directory).rglob('*') if p.is_file()}
45 records = []
46 temporary = pathlib.Path(tempfile.mkdtemp(prefix='ctm-slice-11-validation-', dir='/private/tmp'))
47 relocated = temporary / 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil.copytree(ROOT, relocated, ignore=shutil.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-11'))
51 reports = []
52 logs = []
53 for version, runtime, checkout, log_name in [
54 ('2.5.4', args.sbcl_254, ROOT, 'tests-2.5.4.log'),
55 ('2.4.10', args.sbcl_2410, relocated, 'tests-2.4.10-relocated.log'),
56 ]:
57 version_result = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--version'],
58 check=True, capture_output=True, text=True)
59 assert version_result.stdout.strip() == 'SBCL ' + version, version_result.stdout
60 report = temporary / ('validation-' + version + '.json')
61 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(checkout / 'scripts/test.sh')]
63 result = subprocess.run(command, cwd=checkout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64 (output / log_name).write_bytes(result.stdout)
65 records.append(dict(command=command, workdir=str(checkout), exitCode=result.returncode, log=log_name))
66 if result.returncode:
67 raise RuntimeError(f'{version} failed: see {output / log_name}')
68 reports.append(report)
69 logs.append(result.stdout.decode('utf-8'))
70 print(f'OFFLINE-SBCL-{version}-PASS: {log_name}', flush=True)
71 for name, suffix in zip(NAMES, SUFFIXES):
72 a, b = [pathlib.Path(str(p) + suffix).read_bytes() for p in reports]
73 assert a == b, 'Reports differ after runtime change/relocation: ' + name
74 (output / (name + '.json')).write_bytes(a)
75 previous = ROOT / 'reports/slice-10'
76 for name in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates']:
77 old = json.loads((previous / (name + '.json')).read_text())
78 new = json.loads((output / (name + '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old['cases'] == new['cases'], 'Retained object observations changed: ' + name
82 old_cxtm = json.loads((previous / 'cxtm.json').read_text())
83 new_cxtm = json.loads((output / 'cxtm.json').read_text())
84 assert old_cxtm['cases'] == new_cxtm['cases'][:55]
85 old_datetime = json.loads((previous / 'datetimes.json').read_text())
86 new_datetime = json.loads((output / 'datetimes.json').read_text())
87 assert old_datetime['cases'] == new_datetime['cases'][:4]
88 # Every previously passing imported contract remains PASS; the only status
89 # transition is the newly executed timezone2 baseline mismatch.
90 old_imported = json.loads((previous / 'validation.json').read_text())
91 new_imported = json.loads((output / 'validation.json').read_text())
92 transitions = []
93 for old, new in zip(old_imported['cases'], new_imported['cases']):
94 assert old['id'] == new['id']
95 if old['status'] == 'pass': assert new['status'] == 'pass'
96 if old['status'] != new['status']: transitions.append((old['id'], old['status'], new['status']))
97 assert transitions == [('ctm/in/occurrence-datetime-timezone2.ctm', 'not-run', 'fail')]
98 assert new_cxtm['contractCounts'] == {'pass': 54, 'fail': 2}
99 assert old_cxtm['checks'][:445] == new_cxtm['checks'][:445]
100 actual_dir = output / 'actual'
101 actual_dir.mkdir(exist_ok=True)
102 for case in new_cxtm['cases']:
103 actual = case['actualCXTM'].encode('utf-8')
104 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
105 if case['id'] in ['ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm']:
106 assert actual != baseline.read_bytes() and case['status'] == 'fail'
107 else:
108 assert actual == baseline.read_bytes() and case['status'] == 'pass', case['id']
109 (actual_dir / baseline.name).write_bytes(actual)
110 # Execute the exact public inspection example, suppressing only environment-
111 # dependent FASL location defaults. It uses no test-only API.
112 document = (ROOT / 'docs/reading-datetime-timezones.md').read_text()
113 example = document.split('```lisp\n', 1)[1].split('\n```', 1)[0]
114 setup = '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
115 source = setup + example + '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
116 command = ['/usr/bin/sandbox-exec', '-p', POLICY, args.sbcl_254, '--noinform', '--no-sysinit',
117 '--no-userinit', '--script', '/dev/stdin']
118 result = subprocess.run(command, cwd=ROOT, input=source.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
119 (output / 'describe.log').write_bytes(result.stdout)
120 records.append(dict(command=command, workdir=str(ROOT), exitCode=result.returncode, log='describe.log',
121 input='Lisp fence in docs/reading-datetime-timezones.md plus FASL setup/success marker',
122 inputSHA256=digest(source.encode('utf-8'))))
123 assert result.returncode == 0 and b'DOCUMENTED-INSPECTION-PASS' in result.stdout
124 for name, value in protected.items():
125 assert digest((ROOT / name).read_bytes()) == value, 'Protected file changed: ' + name
126 totals = {}
127 for key, marker in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
128 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
129 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS'), ('dateChecks', 'DATES'), ('datetimeChecks', 'DATETIMES'), ('timezoneChecks', 'TIMEZONES')]:
130 values = [int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for log in logs]
131 assert values[0] == values[1]
132 totals[key] = values[0]
133 assert [totals[k] for k in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
134 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks', 'decimalChecks', 'dateChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665, 715, 1806]
135 assert all('NFC-ORACLE-PASS: 16294' in log and 'Ran 2 tests' in log
136 and 'INTEGER-INDEPENDENT-PASS:' in log and 'DECIMAL-INDEPENDENT-PASS:' in log and 'DATE-INDEPENDENT-PASS:' in log and 'DATETIME-INDEPENDENT-PASS:' in log and 'TIMEZONE-INDEPENDENT-PASS:' in log for log in logs)
137 totals.update(totalLispChecks=sum(totals.values()), combinedCXTMReportChecks=new_cxtm['passed'],
138 unicodeOracleComparisons=16294, pythonIntegrityTests=2, regressionAssertionFailures=0, pinnedCanonicalCases=54, executedCanonicalCases=56, knownBaselineMismatches=2, documentedNormativeDiscrepancies=2, canonicalContractCounts=new_cxtm['contractCounts'],
139 newPositivePasses=0, newlyExecutedPositiveMismatches=1, newNegativeCases=0, independentIntegerOracleProbes=143, independentDecimalOracleProbes=157,
140 independentDateOracleProbes=len(json.loads((output / 'dates.json').read_text())['lexicalProbes']),
141 independentDateTimeOracleProbes=len(json.loads((output / 'datetimes.json').read_text())['lexicalProbes']),
142 importedStatuses=json.loads((output / 'validation.json').read_text())['statuses'],
143 independentTimezoneOracleProbes=len(json.loads((output / 'timezones.json').read_text())['lexicalProbes']),
144 allTwelveReportPairsByteIdentical=True, previous55CanonicalOutputsAnd54PassesUnchanged=True,
145 previousFourDatetimeObservationsUnchanged=True,
146 previous445CanonicalChecksUnchanged=True, originalCorpusAndReferencesUnchanged=True,
147 documentedInspectionPassed=True)
148 artifacts = {str(p.relative_to(output)): digest(p.read_bytes()) for p in sorted(output.rglob('*'))
149 if p.is_file() and p.name != 'execution.json'}
150 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-11', validatedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
151 pythonVersion=sys.version, unicodeOracleVersion=unicodedata.unidata_version, commands=records, results=totals,
152 qualification='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
153 normativeInterpretation='XSD UTC conversion implemented with exact calendar/fraction arithmetic. Original timezone2 now executes as a known normative baseline FAIL; datetime2 FAIL unchanged. TMDM values/equality/order remain lexical; qualified strings and retained handles unchanged. Nonzero-offset BCE day rollover and leap seconds are specific unsupported contexts; Date timezone canonicalization excluded.',
154 artifactsSHA256=artifacts)
155 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
156 print(json.dumps(totals, sort_keys=True), flush=True)
157 print('SLICE-11-VALIDATION-PASS: offline dual runtime, relocation, 12 identical report pairs; previous 55 outputs unchanged; two known normative baseline FAILs', flush=True)
158
159
160 if __name__ == '__main__':
161 main()