2 """Record Slice 16B's complete offline fresh-SBCL regression and source evidence."""
14 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
15 BASE
= 'f24ac27e1e1cc0118411b058d40af3c1c87f3ffb'
16 POLICY
= '(version 1)(allow default)(deny network*)'
17 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification']
23 return hashlib
.sha256(path
.read_bytes()).hexdigest()
27 parser
= argparse
.ArgumentParser(description
=__doc__
)
28 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
29 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-16b'))
30 args
= parser
.parse_args()
31 runtime
= shutil
.which(args
.sbcl
)
33 raise RuntimeError('SBCL executable not found: ' + args
.sbcl
)
34 output
= pathlib
.Path(args
.output
).resolve()
35 output
.mkdir(parents
=True, exist_ok
=True)
36 protected
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['vendor/cxtm', 'references']
37 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
38 sources
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['src', 'tests', 'scripts', 'docs']
39 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file() and
40 '__pycache__' not in p
.parts
and p
.suffix
in ['.lisp', '.py', '.sh', '.md']}
41 sources
.update({p
: sha(ROOT
/ p
) for p
in ['topic-maps.asd', 'README.md']})
42 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
43 version
= subprocess
.check_output([runtime
, '--version'], text
=True).strip()
44 with tempfile
.TemporaryDirectory(prefix
='ctm-slice-16b-validation-') as temp
:
45 report
= pathlib
.Path(temp
) / 'validation.json'
46 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
47 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(ROOT
/ 'scripts/test.sh')]
48 result
= subprocess
.run(command
, cwd
=ROOT
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
49 (output
/ 'tests.log').write_bytes(result
.stdout
)
51 raise RuntimeError('Full regression failed: see ' + str(output
/ 'tests.log'))
52 log
= result
.stdout
.decode('utf-8')
54 source
= report
if name
== 'validation' else pathlib
.Path(str(report
) + '.' + name
+ '.json')
55 (output
/ (name
+ '.json')).write_bytes(source
.read_bytes())
56 old
= ROOT
/ 'reports/slice-16a'
57 previous
= json
.loads((old
/ 'validation.json').read_text())
58 now
= json
.loads((output
/ 'validation.json').read_text())
59 assert now
['statuses'] == {'pass': 106, 'fail': 2, 'unsupported': 0, 'not-run': 165}
60 reification
= json
.loads((output
/ 'reification.json').read_text())
61 selected
= sorted([c
['id'] for c
in reification
['cases']] + [c
['id'] for c
in reification
['negativeCases']])
62 changes
= [(a
['id'], a
['status'], b
['status']) for a
, b
in zip(previous
['cases'], now
['cases']) if a
['status'] != b
['status']]
63 assert changes
== [(id, 'not-run', 'pass') for id in selected
]
64 for a
, b
in zip(previous
['cases'], now
['cases']):
65 if a
['id'] not in selected
:
66 assert a
== b
, a
['id']
67 for name
in NAMES
[1:-1]:
68 key
= 'lexicalProbes' if name
== 'timezones' else 'examples' if name
== 'identifiers' else 'cases'
69 prior
= json
.loads((old
/ (name
+ '.json')).read_text())[key
]
70 current
= json
.loads((output
/ (name
+ '.json')).read_text())[key
]
71 assert prior
== (current
[:len(prior
)] if name
== 'cxtm' else current
), name
72 actual_reifiers
= output
/ 'reification-actual'
73 actual_reifiers
.mkdir(exist_ok
=True)
74 for case
in json
.loads((output
/ 'cxtm.json').read_text())['cases']:
75 if case
['id'] in selected
:
76 actual
= case
['actualCXTM'].encode('utf-8')
77 assert actual
== (ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')).read_bytes()
78 (actual_reifiers
/ (pathlib
.Path(case
['id']).name
+ '.cxtm')).write_bytes(actual
)
79 identifiers
= json
.loads((output
/ 'identifiers.json').read_text())
80 actual_dir
= output
/ 'manual-model-actual'
81 actual_dir
.mkdir(exist_ok
=True)
82 for example
in identifiers
['examples']:
83 actual
= example
['actualCXTM'].encode('utf-8')
84 assert actual
== (ROOT
/ 'vendor/cxtm' / example
['baseline']).read_bytes()
85 (actual_dir
/ (pathlib
.Path(example
['source']).name
+ '.cxtm')).write_bytes(actual
)
86 document
= (ROOT
/ 'docs/reading-reification.md').read_text()
87 example
= document
.split('```lisp\n', 2)[2].split('\n```', 1)[0]
88 inspection
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
89 '--no-userinit', '--script', '/dev/stdin'], input=example
.encode(), cwd
=ROOT
,
90 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
91 (output
/ 'describe.log').write_bytes(inspection
.stdout
)
92 assert inspection
.returncode
== 0, 'Documented inspection forms failed'
93 for path
, value
in protected
.items():
94 assert sha(ROOT
/ path
) == value
, path
95 for path
, value
in sources
.items():
96 assert sha(ROOT
/ path
) == value
, path
97 previous_totals
= json
.loads((old
/ 'execution.json').read_text())['regressionChecks']
98 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1))
99 for marker
in [*previous_totals
, 'REIFICATION']}
100 assert {k
: totals
[k
] for k
in previous_totals
} == previous_totals
101 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-16b', runtime
=version
,
102 startedAtUTC
=started
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
103 command
=command
, exitCode
=result
.returncode
, freshSBCL
=True, network
='denied by sandbox-exec',
104 regressionChecks
=totals
, totalLispChecks
=sum(totals
.values()), importedStatuses
=now
['statuses'],
105 importedStatusChanges
=changes
, previous83CXTMOutputsAndAllModelObservationsUnchanged
=True,
106 reificationOriginalPositives
=len(reification
['cases']), reificationOriginalNegatives
=len(reification
['negativeCases']),
107 manualXTM2ModelBaselineComparisons
=len(identifiers
['examples']), xtmDeserializationImplemented
=False,
108 originalCorpusAndReferencesUnchanged
=True, sourceSHA256
=sources
, documentedInspectionPassed
=True,
109 qualification
='Same-map Reification; manual TMDM models are not XTM deserialization contracts. The two DateTime baseline FAILs remain unchanged.',
110 artifactsSHA256
={str(p
.relative_to(output
)): sha(p
) for p
in sorted(output
.rglob('*'))
111 if p
.is_file() and p
.name
!= 'execution.json'})
112 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
113 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys
=True))
114 print('SLICE-16B-VALIDATION-PASS: fresh offline SBCL; 11 exact original Reification comparisons and 2 specific rejections; retained results unchanged')
117 if __name__
== '__main__':