2 """Reproduce Slice 10: two fresh offline runtimes, relocation, report/retained-output checks.
4 Requires macOS sandbox-exec, installed SBCL 2.5.4/2.4.10 and Python UCD 15.0/15.1.
5 Does not provision software, change corpus/locks, use a server or commit files.
20 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
21 BASE
= '2ad41cd1a945a4ce97824464de40646bb899259e'
22 POLICY
= '(version 1)(allow default)(deny network*)'
23 SUFFIXES
= ['', '.semantics.json', '.merging.json', '.associations.json',
24 '.cxtm.json', '.occurrences.json', '.prefixes.json', '.integers.json', '.decimals.json', '.dates.json', '.datetimes.json']
25 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates', 'datetimes']
29 return hashlib
.sha256(data
).hexdigest()
33 parser
= argparse
.ArgumentParser(description
=__doc__
)
34 parser
.add_argument('--sbcl-254', required
=True)
35 parser
.add_argument('--sbcl-2410', required
=True)
36 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-10'))
37 args
= parser
.parse_args()
38 output
= pathlib
.Path(args
.output
).resolve()
39 output
.mkdir(parents
=True, exist_ok
=True)
40 # The pin is verified inside each full suite. Snapshot all reference/corpus
41 # bytes independently as a guard against any change during validation.
42 protected
= {str(p
.relative_to(ROOT
)): digest(p
.read_bytes())
43 for directory
in ['vendor/cxtm', 'references']
44 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
46 temporary
= pathlib
.Path(tempfile
.mkdtemp(prefix
='ctm-slice-10-validation-', dir='/private/tmp'))
47 relocated
= temporary
/ 'checkout'
48 # Historical reports are inputs for retained-result comparison, but the new
49 # output directory is not copied into its own test checkout.
50 shutil
.copytree(ROOT
, relocated
, ignore
=shutil
.ignore_patterns('.git', '__pycache__', '*.pyc', 'slice-10'))
53 for version
, runtime
, checkout
, log_name
in [
54 ('2.5.4', args
.sbcl_254
, ROOT
, 'tests-2.5.4.log'),
55 ('2.4.10', args
.sbcl_2410
, relocated
, 'tests-2.4.10-relocated.log'),
57 version_result
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--version'],
58 check
=True, capture_output
=True, text
=True)
59 assert version_result
.stdout
.strip() == 'SBCL ' + version
, version_result
.stdout
60 report
= temporary
/ ('validation-' + version
+ '.json')
61 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
62 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(checkout
/ 'scripts/test.sh')]
63 result
= subprocess
.run(command
, cwd
=checkout
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
64 (output
/ log_name
).write_bytes(result
.stdout
)
65 records
.append(dict(command
=command
, workdir
=str(checkout
), exitCode
=result
.returncode
, log
=log_name
))
67 raise RuntimeError(f
'{version} failed: see {output / log_name}')
68 reports
.append(report
)
69 logs
.append(result
.stdout
.decode('utf-8'))
70 print(f
'OFFLINE-SBCL-{version}-PASS: {log_name}', flush
=True)
71 for name
, suffix
in zip(NAMES
, SUFFIXES
):
72 a
, b
= [pathlib
.Path(str(p
) + suffix
).read_bytes() for p
in reports
]
73 assert a
== b
, 'Reports differ after runtime change/relocation: ' + name
74 (output
/ (name
+ '.json')).write_bytes(a
)
75 previous
= ROOT
/ 'reports/slice-9'
76 for name
in ['semantics', 'merging', 'associations', 'occurrences', 'prefixes', 'integers', 'decimals', 'dates']:
77 old
= json
.loads((previous
/ (name
+ '.json')).read_text())
78 new
= json
.loads((output
/ (name
+ '.json')).read_text())
79 # Checks change when previously unsupported syntax becomes implemented;
80 # the original sources and actual object observations must not change.
81 assert old
['cases'] == new
['cases'], 'Retained object observations changed: ' + name
82 old_cxtm
= json
.loads((previous
/ 'cxtm.json').read_text())
83 new_cxtm
= json
.loads((output
/ 'cxtm.json').read_text())
84 assert old_cxtm
['cases'] == new_cxtm
['cases'][:51]
85 assert new_cxtm
['contractCounts'] == {'pass': 54, 'fail': 1}
86 assert old_cxtm
['checks'][:445] == new_cxtm
['checks'][:445]
87 actual_dir
= output
/ 'actual'
88 actual_dir
.mkdir(exist_ok
=True)
89 for case
in new_cxtm
['cases']:
90 actual
= case
['actualCXTM'].encode('utf-8')
91 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')
92 if case
['id'] == 'ctm/in/occurrence-datetime2.ctm':
93 assert actual
!= baseline
.read_bytes() and case
['status'] == 'fail'
95 assert actual
== baseline
.read_bytes() and case
['status'] == 'pass', case
['id']
96 (actual_dir
/ baseline
.name
).write_bytes(actual
)
97 # Execute the exact public inspection example, suppressing only environment-
98 # dependent FASL location defaults. It uses no test-only API.
99 document
= (ROOT
/ 'docs/reading-datetime-occurrences.md').read_text()
100 example
= document
.split('```lisp\n', 1)[1].split('\n```', 1)[0]
101 setup
= '(require :asdf)\n(asdf:initialize-output-translations \'(:output-translations (t (:root "tmp" "topic-maps-fasls")) :ignore-inherited-configuration))\n'
102 source
= setup
+ example
+ '\n(format t "~&DOCUMENTED-INSPECTION-PASS~%")\n'
103 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, args
.sbcl_254
, '--noinform', '--no-sysinit',
104 '--no-userinit', '--script', '/dev/stdin']
105 result
= subprocess
.run(command
, cwd
=ROOT
, input=source
.encode('utf-8'), stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
106 (output
/ 'describe.log').write_bytes(result
.stdout
)
107 records
.append(dict(command
=command
, workdir
=str(ROOT
), exitCode
=result
.returncode
, log
='describe.log',
108 input='Lisp fence in docs/reading-datetime-occurrences.md plus FASL setup/success marker',
109 inputSHA256
=digest(source
.encode('utf-8'))))
110 assert result
.returncode
== 0 and b
'DOCUMENTED-INSPECTION-PASS' in result
.stdout
111 for name
, value
in protected
.items():
112 assert digest((ROOT
/ name
).read_bytes()) == value
, 'Protected file changed: ' + name
114 for key
, marker
in [('foundationChecks', 'FOUNDATION'), ('identityChecks', 'CTM-IDENTITY-SEMANTICS'),
115 ('mergeChecks', 'CTM-MERGING-SEMANTICS'), ('associationChecks', 'CTM-ASSOCIATIONS-SEMANTICS'),
116 ('retainedCXTMChecks', 'CXTM'), ('occurrenceChecks', 'OCCURRENCES'), ('prefixDatatypeChecks', 'PREFIX-DATATYPES'), ('integerChecks', 'INTEGERS'), ('decimalChecks', 'DECIMALS'), ('dateChecks', 'DATES'), ('datetimeChecks', 'DATETIMES')]:
117 values
= [int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) for log
in logs
]
118 assert values
[0] == values
[1]
119 totals
[key
] = values
[0]
120 assert [totals
[k
] for k
in ['foundationChecks', 'identityChecks', 'mergeChecks', 'associationChecks',
121 'retainedCXTMChecks', 'occurrenceChecks', 'prefixDatatypeChecks', 'integerChecks', 'decimalChecks', 'dateChecks']] == [54, 123, 646, 657, 445, 1101, 150, 665, 715, 1806]
122 assert all('NFC-ORACLE-PASS: 16294' in log
and 'Ran 2 tests' in log
123 and 'INTEGER-INDEPENDENT-PASS:' in log
and 'DECIMAL-INDEPENDENT-PASS:' in log
and 'DATE-INDEPENDENT-PASS:' in log
and 'DATETIME-INDEPENDENT-PASS:' in log
for log
in logs
)
124 totals
.update(totalLispChecks
=sum(totals
.values()), combinedCXTMReportChecks
=new_cxtm
['passed'],
125 unicodeOracleComparisons
=16294, pythonIntegrityTests
=2, regressionAssertionFailures
=0, pinnedCanonicalCases
=54, executedCanonicalCases
=55, knownBaselineMismatches
=1, documentedNormativeDiscrepancies
=2, canonicalContractCounts
=new_cxtm
['contractCounts'],
126 newPositivePasses
=3, newNegativeCases
=0, independentIntegerOracleProbes
=143, independentDecimalOracleProbes
=157,
127 independentDateOracleProbes
=len(json
.loads((output
/ 'dates.json').read_text())['lexicalProbes']),
128 independentDateTimeOracleProbes
=len(json
.loads((output
/ 'datetimes.json').read_text())['lexicalProbes']),
129 importedStatuses
=json
.loads((output
/ 'validation.json').read_text())['statuses'],
130 allElevenReportPairsByteIdentical
=True, previous51ObjectObservationsAndCanonicalOutputsUnchanged
=True,
131 previous445CanonicalChecksUnchanged
=True, originalCorpusAndReferencesUnchanged
=True,
132 documentedInspectionPassed
=True)
133 artifacts
= {str(p
.relative_to(output
)): digest(p
.read_bytes()) for p
in sorted(output
.rglob('*'))
134 if p
.is_file() and p
.name
!= 'execution.json'}
135 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-10', validatedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
136 pythonVersion
=sys
.version
, unicodeOracleVersion
=unicodedata
.unidata_version
, commands
=records
, results
=totals
,
137 qualification
='Pinned corpus targets CTM draft 2010-03-31; published standards remain authoritative.',
138 normativeInterpretation
='XSD 1.0 D.1 noncandidate leap second maps to following minute: original datetime2 FAIL retained as known baseline discrepancy. Absent/zero zones supported; nonzero offsets validate but canonical conversion unsupported, original timezone2 NOT-RUN. TMDM values/equality/order remain lexical, qualified strings unchanged, pure CXTM canonicalization.',
139 artifactsSHA256
=artifacts
)
140 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
141 print(json
.dumps(totals
, sort_keys
=True), flush
=True)
142 print('SLICE-10-VALIDATION-PASS: offline dual runtime, relocation, 11 identical report pairs; previous 51 outputs unchanged; one known normative baseline FAIL', flush
=True)
145 if __name__
== '__main__':