2 """Reproduce Slice 11: two fresh offline runtimes, relocation, report/retained-output checks.
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
20 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
21 BASE
= '8741d155d948c7317945eee423a977940cad995e'
22 POLICY
= '(version 1)(allow default)(deny network*)'
23 SUFFIXES
= ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json', '.dates.json', '.datetimes.json', '.timezones.json']
25 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones']
29 return hashlib
.sha256(data
).hexdigest()
33 parser
= argparse
.ArgumentParser(description
=__doc__
)
34 parser
.add_argument('--sbcl-254', required
=True)
35 parser
.add_argument('--sbcl-2410', required
=True)
36 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-11'))
37 args
= parser
.parse_args()
38 output
= pathlib
.Path(args
.output
).resolve()
39 output
.mkdir(parents
=True, exist_ok
=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected
= {str(p
.relative_to(ROOT
)): digest(p
.read_bytes())
43 for directory
in ['vendor/cxtm', 'references']
44 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
46 temporary
= pathlib
.Path(tempfile
.mkdtemp(prefix
='ctm-slice-11-validation-', dir='/private/tmp'))
47 relocated
= temporary
/ 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil
.copytree(ROOT
, relocated
, ignore
=shutil
.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-11'))
53 for version
, runtime
, checkout
, log_name
in [
54 ('2.5.4', args
.sbcl_254
, ROOT
, 'tests-2.5.4.log'),
55 ('2.4.10', args
.sbcl_2410
, relocated
, 'tests-2.4.10-relocated.log'),
57 version_result
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--version'],
58 check
=True, capture_output
=True, text
=True)
59 assert version_result
.stdout
.strip() == 'SBCL ' + version
, version_result
.stdout
60 report
= temporary
/ ('validation-' + version
+ '.json')
61 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(checkout
/ 'scripts/test.sh')]
63 result
= subprocess
.run(command
, cwd
=checkout
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
64 (output
/ log_name
).write_bytes(result
.stdout
)
65 records
.append(dict(command
=command
, workdir
=str(checkout
), exitCode
=result
.returncode
, log
=log_name
))
67 raise RuntimeError(f
'{version} failed: see {output / log_name}')
68 reports
.append(report
)
69 logs
.append(result
.stdout
.decode('utf-8'))
70 print(f
'OFFLINE-SBCL-{version}-PASS: {log_name}', flush
=True)
71 for name
, suffix
in zip(NAMES
, SUFFIXES
):
72 a
, b
= [pathlib
.Path(str(p
) + suffix
).read_bytes() for p
in reports
]
73 assert a
== b
, 'Reports differ after runtime change/relocation: ' + name
74 (output
/ (name
+ '.json')).write_bytes(a
)
75 previous
= ROOT
/ 'reports/slice-10'
76 for name
in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates']:
77 old
= json
.loads((previous
/ (name
+ '.json')).read_text())
78 new
= json
.loads((output
/ (name
+ '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old
['cases'] == new
['cases'], 'Retained object observations changed: ' + name
82 old_cxtm
= json
.loads((previous
/ 'cxtm.json').read_text())
83 new_cxtm
= json
.loads((output
/ 'cxtm.json').read_text())
84 assert old_cxtm
['cases'] == new_cxtm
['cases'][:55]
85 old_datetime
= json
.loads((previous
/ 'datetimes.json').read_text())
86 new_datetime
= json
.loads((output
/ 'datetimes.json').read_text())
87 assert old_datetime
['cases'] == new_datetime
['cases'][:4]
88 # Every previously passing imported contract remains PASS; the only status
89 # transition is the newly executed timezone2 baseline mismatch.
90 old_imported
= json
.loads((previous
/ 'validation.json').read_text())
91 new_imported
= json
.loads((output
/ 'validation.json').read_text())
93 for old
, new
in zip(old_imported
['cases'], new_imported
['cases']):
94 assert old
['id'] == new
['id']
95 if old
['status'] == 'pass': assert new
['status'] == 'pass'
96 if old
['status'] != new
['status']: transitions
.append((old
['id'], old
['status'], new
['status']))
97 assert transitions
== [('ctm/in/occurrence-datetime-timezone2.ctm', 'not-run', 'fail')]
98 assert new_cxtm
['contractCounts'] == {'pass': 54, 'fail': 2}
99 assert old_cxtm
['checks'][:445] == new_cxtm
['checks'][:445]
100 actual_dir
= output
/ 'actual'
101 actual_dir
.mkdir(exist_ok
=True)
102 for case
in new_cxtm
['cases']:
103 actual
= case
['actualCXTM'].encode('utf-8')
104 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')
105 if case
['id'] in ['ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm']:
106 assert actual
!= baseline
.read_bytes() and case
['status'] == 'fail'
108 assert actual
== baseline
.read_bytes() and case
['status'] == 'pass', case
['id']
109 (actual_dir
/ baseline
.name
).write_bytes(actual
)
110 # Execute the exact public inspection example, suppressing only environment-
111 # dependent FASL location defaults. It uses no test-only API.
112 document
= (ROOT
/ 'docs/reading-datetime-timezones.md').read_text()
113 example
= document
.split('```lisp\n', 1)[1].split('\n```', 1)[0]
114 setup
= '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
115 source
= setup
+ example
+ '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
116 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, args
.sbcl_254
, '--noinform', '--no-sysinit',
117 '--no-userinit', '--script', '/dev/stdin']
118 result
= subprocess
.run(command
, cwd
=ROOT
, input=source
.encode('utf-8'), stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
119 (output
/ 'describe.log').write_bytes(result
.stdout
)
120 records
.append(dict(command
=command
, workdir
=str(ROOT
), exitCode
=result
.returncode
, log
='describe.log',
121 input='Lisp fence in docs/reading-datetime-timezones.md plus FASL setup/success marker',
122 inputSHA256
=digest(source
.encode('utf-8'))))
123 assert result
.returncode
== 0 and b
'DOCUMENTED-INSPECTION-PASS' in result
.stdout
124 for name
, value
in protected
.items():
125 assert digest((ROOT
/ name
).read_bytes()) == value
, 'Protected file changed: ' + name
127 for key
, marker
in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
128 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
129 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS'), ('dateChecks', 'DATES'), ('datetimeChecks', 'DATETIMES'), ('timezoneChecks', 'TIMEZONES')]:
130 values
= [int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for log
in logs
]
131 assert values
[0] == values
[1]
132 totals
[key
] = values
[0]
133 assert [totals
[k
] for k
in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
134 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks', 'decimalChecks', 'dateChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665, 715, 1806]
135 assert all('NFC-ORACLE-PASS: 16294' in log
and 'Ran 2 tests' in log
136 and 'INTEGER-INDEPENDENT-PASS:' in log
and 'DECIMAL-INDEPENDENT-PASS:' in log
and 'DATE-INDEPENDENT-PASS:' in log
and 'DATETIME-INDEPENDENT-PASS:' in log
and 'TIMEZONE-INDEPENDENT-PASS:' in log
for log
in logs
)
137 totals
.update(totalLispChecks
=sum(totals
.values()), combinedCXTMReportChecks
=new_cxtm
['passed'],
138 unicodeOracleComparisons
=16294, pythonIntegrityTests
=2, regressionAssertionFailures
=0, pinnedCanonicalCases
=54, executedCanonicalCases
=56, knownBaselineMismatches
=2, documentedNormativeDiscrepancies
=2, canonicalContractCounts
=new_cxtm
['contractCounts'],
139 newPositivePasses
=0, newlyExecutedPositiveMismatches
=1, newNegativeCases
=0, independentIntegerOracleProbes
=143, independentDecimalOracleProbes
=157,
140 independentDateOracleProbes
=len(json
.loads((output
/ 'dates.json').read_text())['lexicalProbes']),
141 independentDateTimeOracleProbes
=len(json
.loads((output
/ 'datetimes.json').read_text())['lexicalProbes']),
142 importedStatuses
=json
.loads((output
/ 'validation.json').read_text())['statuses'],
143 independentTimezoneOracleProbes
=len(json
.loads((output
/ 'timezones.json').read_text())['lexicalProbes']),
144 allTwelveReportPairsByteIdentical
=True, previous55CanonicalOutputsAnd54PassesUnchanged
=True,
145 previousFourDatetimeObservationsUnchanged
=True,
146 previous445CanonicalChecksUnchanged
=True, originalCorpusAndReferencesUnchanged
=True,
147 documentedInspectionPassed
=True)
148 artifacts
= {str(p
.relative_to(output
)): digest(p
.read_bytes()) for p
in sorted(output
.rglob('*'))
149 if p
.is_file() and p
.name
!= 'execution.json'}
150 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-11', validatedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
151 pythonVersion
=sys
.version
, unicodeOracleVersion
=unicodedata
.unidata_version
, commands
=records
, results
=totals
,
152 qualification
='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
153 normativeInterpretation
='XSD UTC conversion implemented with exact calendar/fraction arithmetic. Original timezone2 now executes as a known normative baseline FAIL; datetime2 FAIL unchanged. TMDM values/equality/order remain lexical; qualified strings and retained handles unchanged. Nonzero-offset BCE day rollover and leap seconds are specific unsupported contexts; Date timezone canonicalization excluded.',
154 artifactsSHA256
=artifacts
)
155 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
156 print(json
.dumps(totals
, sort_keys
=True), flush
=True)
157 print('SLICE-11-VALIDATION-PASS: offline dual runtime, relocation, 12 identical report pairs; previous 55 outputs unchanged; two known normative baseline FAILs', flush
=True)
160 if __name__
== '__main__':