]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-10.py
Support DateTime occurrences and expose normative baseline conflicts
[topic-maps.git] / scripts / validate-slice-10.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 10: two fresh offline runtimes, relocation, report/retained-output checks.
3
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
6 """
7 import argparse
8 import datetime
9 import hashlib
10 import json
11 import os
12 import pathlib
13 import re
14 import shutil
15 import subprocess
16 import sys
17 import tempfile
18 import unicodedata
19
20 ROOT = pathlib.Path(__file__).resolve().parent.parent
21 BASE = '2ad41cd1a945a4ce97824464de40646bb899259e'
22 POLICY = '(version 1)(allow default)(deny network*)'
23 SUFFIXES = ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json', '.dates.json', '.datetimes.json']
25 NAMES = ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates', 'datetimes']
26
27
28 def digest(data):
29 return hashlib.sha256(data).hexdigest()
30
31
32 def main():
33 parser = argparse.ArgumentParser(description=__doc__)
34 parser.add_argument('--sbcl-254', required=True)
35 parser.add_argument('--sbcl-2410', required=True)
36 parser.add_argument('--output', default=str(ROOT / 'reports/slice-10'))
37 args = parser.parse_args()
38 output = pathlib.Path(args.output).resolve()
39 output.mkdir(parents=True, exist_ok=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected = {str(p.relative_to(ROOT)): digest(p.read_bytes())
43 for directory in ['vendor/cxtm', 'references']
44 for p in (ROOT / directory).rglob('*') if p.is_file()}
45 records = []
46 temporary = pathlib.Path(tempfile.mkdtemp(prefix='ctm-slice-10-validation-', dir='/private/tmp'))
47 relocated = temporary / 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil.copytree(ROOT, relocated, ignore=shutil.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-10'))
51 reports = []
52 logs = []
53 for version, runtime, checkout, log_name in [
54 ('2.5.4', args.sbcl_254, ROOT, 'tests-2.5.4.log'),
55 ('2.4.10', args.sbcl_2410, relocated, 'tests-2.4.10-relocated.log'),
56 ]:
57 version_result = subprocess.run(['/usr/bin/sandbox-exec', '-p', POLICY, runtime, '--version'],
58 check=True, capture_output=True, text=True)
59 assert version_result.stdout.strip() == 'SBCL ' + version, version_result.stdout
60 report = temporary / ('validation-' + version + '.json')
61 command = ['/usr/bin/sandbox-exec', '-p', POLICY, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime, 'TOPIC_MAPS_REPORT=' + str(report), str(checkout / 'scripts/test.sh')]
63 result = subprocess.run(command, cwd=checkout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64 (output / log_name).write_bytes(result.stdout)
65 records.append(dict(command=command, workdir=str(checkout), exitCode=result.returncode, log=log_name))
66 if result.returncode:
67 raise RuntimeError(f'{version} failed: see {output / log_name}')
68 reports.append(report)
69 logs.append(result.stdout.decode('utf-8'))
70 print(f'OFFLINE-SBCL-{version}-PASS: {log_name}', flush=True)
71 for name, suffix in zip(NAMES, SUFFIXES):
72 a, b = [pathlib.Path(str(p) + suffix).read_bytes() for p in reports]
73 assert a == b, 'Reports differ after runtime change/relocation: ' + name
74 (output / (name + '.json')).write_bytes(a)
75 previous = ROOT / 'reports/slice-9'
76 for name in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates']:
77 old = json.loads((previous / (name + '.json')).read_text())
78 new = json.loads((output / (name + '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old['cases'] == new['cases'], 'Retained object observations changed: ' + name
82 old_cxtm = json.loads((previous / 'cxtm.json').read_text())
83 new_cxtm = json.loads((output / 'cxtm.json').read_text())
84 assert old_cxtm['cases'] == new_cxtm['cases'][:51]
85 assert new_cxtm['contractCounts'] == {'pass': 54, 'fail': 1}
86 assert old_cxtm['checks'][:445] == new_cxtm['checks'][:445]
87 actual_dir = output / 'actual'
88 actual_dir.mkdir(exist_ok=True)
89 for case in new_cxtm['cases']:
90 actual = case['actualCXTM'].encode('utf-8')
91 baseline = ROOT / 'vendor/cxtm/ctm/baseline' / (pathlib.Path(case['id']).name + '.cxtm')
92 if case['id'] == 'ctm/in/occurrence-datetime2.ctm':
93 assert actual != baseline.read_bytes() and case['status'] == 'fail'
94 else:
95 assert actual == baseline.read_bytes() and case['status'] == 'pass', case['id']
96 (actual_dir / baseline.name).write_bytes(actual)
97 # Execute the exact public inspection example, suppressing only environment-
98 # dependent FASL location defaults. It uses no test-only API.
99 document = (ROOT / 'docs/reading-datetime-occurrences.md').read_text()
100 example = document.split('```lisp\n', 1)[1].split('\n```', 1)[0]
101 setup = '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
102 source = setup + example + '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
103 command = ['/usr/bin/sandbox-exec', '-p', POLICY, args.sbcl_254, '--noinform', '--no-sysinit',
104 '--no-userinit', '--script', '/dev/stdin']
105 result = subprocess.run(command, cwd=ROOT, input=source.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
106 (output / 'describe.log').write_bytes(result.stdout)
107 records.append(dict(command=command, workdir=str(ROOT), exitCode=result.returncode, log='describe.log',
108 input='Lisp fence in docs/reading-datetime-occurrences.md plus FASL setup/success marker',
109 inputSHA256=digest(source.encode('utf-8'))))
110 assert result.returncode == 0 and b'DOCUMENTED-INSPECTION-PASS' in result.stdout
111 for name, value in protected.items():
112 assert digest((ROOT / name).read_bytes()) == value, 'Protected file changed: ' + name
113 totals = {}
114 for key, marker in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
115 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
116 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS'), ('dateChecks', 'DATES'), ('datetimeChecks', 'DATETIMES')]:
117 values = [int(re.search(r'(?m)^' + marker + r'-PASS: (\d+)', log).group(1)) for log in logs]
118 assert values[0] == values[1]
119 totals[key] = values[0]
120 assert [totals[k] for k in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
121 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks', 'decimalChecks', 'dateChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665, 715, 1806]
122 assert all('NFC-ORACLE-PASS: 16294' in log and 'Ran 2 tests' in log
123 and 'INTEGER-INDEPENDENT-PASS:' in log and 'DECIMAL-INDEPENDENT-PASS:' in log and 'DATE-INDEPENDENT-PASS:' in log and 'DATETIME-INDEPENDENT-PASS:' in log for log in logs)
124 totals.update(totalLispChecks=sum(totals.values()), combinedCXTMReportChecks=new_cxtm['passed'],
125 unicodeOracleComparisons=16294, pythonIntegrityTests=2, regressionAssertionFailures=0, pinnedCanonicalCases=54, executedCanonicalCases=55, knownBaselineMismatches=1, documentedNormativeDiscrepancies=2, canonicalContractCounts=new_cxtm['contractCounts'],
126 newPositivePasses=3, newNegativeCases=0, independentIntegerOracleProbes=143, independentDecimalOracleProbes=157,
127 independentDateOracleProbes=len(json.loads((output / 'dates.json').read_text())['lexicalProbes']),
128 independentDateTimeOracleProbes=len(json.loads((output / 'datetimes.json').read_text())['lexicalProbes']),
129 importedStatuses=json.loads((output / 'validation.json').read_text())['statuses'],
130 allElevenReportPairsByteIdentical=True, previous51ObjectObservationsAndCanonicalOutputsUnchanged=True,
131 previous445CanonicalChecksUnchanged=True, originalCorpusAndReferencesUnchanged=True,
132 documentedInspectionPassed=True)
133 artifacts = {str(p.relative_to(output)): digest(p.read_bytes()) for p in sorted(output.rglob('*'))
134 if p.is_file() and p.name != 'execution.json'}
135 evidence = dict(baseCommit=BASE, branch='codex/ctm-slice-10', validatedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
136 pythonVersion=sys.version, unicodeOracleVersion=unicodedata.unidata_version, commands=records, results=totals,
137 qualification='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
138 normativeInterpretation='XSD 1.0 D.1 noncandidate leap second maps to following minute: original datetime2 FAIL retained as known baseline discrepancy. Absent/zero zones supported; nonzero offsets validate but canonical conversion unsupported, original timezone2 NOT-RUN. TMDM values/equality/order remain lexical, qualified strings unchanged, pure CXTM canonicalization.',
139 artifactsSHA256=artifacts)
140 (output / 'execution.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
141 print(json.dumps(totals, sort_keys=True), flush=True)
142 print('SLICE-10-VALIDATION-PASS: offline dual runtime, relocation, 11 identical report pairs; previous 51 outputs unchanged; one known normative baseline FAIL', flush=True)
143
144
145 if __name__ == '__main__':
146 main()