2 """Validate Slice 19C in a fresh SBCL 2.5.4 process with network access denied."""
14 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
15 BASE
= '5e716c078b885e97ebe858230ce7d8578e65ae4b'
16 POLICY
= '(version 1)(allow default)(deny network*)'
17 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity',
20 'relationships', 'templates', 'map-import', 'mergemap', 'include']
24 return hashlib
.sha256(path
.read_bytes()).hexdigest()
28 parser
= argparse
.ArgumentParser(description
=__doc__
)
29 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
30 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-19c'))
31 args
= parser
.parse_args()
32 runtime
= shutil
.which(args
.sbcl
)
33 assert runtime
is not None, 'SBCL executable not found'
34 output
= pathlib
.Path(args
.output
).resolve()
35 output
.mkdir(parents
=True, exist_ok
=True)
36 protected
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['vendor/cxtm', 'references']
37 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
38 sources
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['src', 'tests', 'scripts', 'docs']
39 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file() and
40 '__pycache__' not in p
.parts
and p
.suffix
in ['.lisp', '.py', '.sh', '.md']}
41 sources
.update({p
: sha(ROOT
/ p
) for p
in ['topic-maps.asd', 'README.md']})
42 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
43 version
= subprocess
.check_output([runtime
, '--version'], text
=True).strip()
44 assert version
== 'SBCL 2.5.4', version
45 with tempfile
.TemporaryDirectory(prefix
='tmdm-slice-19c-validation-') as temp
:
46 report
= pathlib
.Path(temp
) / 'validation.json'
47 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
48 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(ROOT
/ 'scripts/test.sh')]
49 result
= subprocess
.run(command
, cwd
=ROOT
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
50 (output
/ 'tests.log').write_bytes(result
.stdout
)
52 raise RuntimeError('Full regression failed: see ' + str(output
/ 'tests.log'))
53 log
= result
.stdout
.decode('utf-8')
55 source
= report
if name
== 'validation' else pathlib
.Path(str(report
) + '.' + name
+ '.json')
56 (output
/ (name
+ '.json')).write_bytes(source
.read_bytes())
57 old
= ROOT
/ 'reports/slice-19b'
58 previous
= json
.loads((old
/ 'validation.json').read_text())
59 now
= json
.loads((output
/ 'validation.json').read_text())
60 assert now
['statuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
61 selected
= sorted('ctm/in/' + n
+ '.ctm' for n
in [
62 'include', 'include-prefix', 'include-prefix2', 'include-template',
63 'include-template-wildcard', 'include-tm-reifier', 'include-topicref',
64 'include-wildcard', 'include-wildcard2'])
65 selected
+= sorted('ctm/invalid/' + n
+ '.ctm' for n
in ['include-prefix', 'include-prefix2', 'include-xtm'])
66 discrepancies
= {'ctm/in/include-template.ctm', 'ctm/in/include-wildcard2.ctm'}
67 changes
= [(a
['id'], a
['status'], b
['status']) for a
, b
in zip(previous
['cases'], now
['cases']) if a
['status'] != b
['status']]
68 assert changes
== [(id, 'not-run', 'fail' if id in discrepancies
else 'pass') for id in selected
]
69 assert [c
['id'] for c
in previous
['cases']] == [c
['id'] for c
in now
['cases']]
70 for a
, b
in zip(previous
['cases'], now
['cases']):
71 if a
['id'] not in selected
:
72 assert a
== b
, a
['id']
74 'semantics': ('identity-feature-boundary:%include <other.ctm>', 'identity:Include-requires-explicit-resolver'),
75 'prefixes': ('remaining-construct-unsupported:%include <other.ctm>', 'prefix:Include-requires-explicit-resolver'),
76 'references': ('references:feature-boundary:%include <file.ctm>', 'references:Include-requires-explicit-resolver'),
77 'templates': ('diagnostic:unchanged-excluded-feature:%include <other>', 'diagnostic:Include-requires-explicit-resolver')}
78 for name
in NAMES
[1:-1]:
79 prior
= json
.loads((old
/ (name
+ '.json')).read_text())
80 current
= json
.loads((output
/ (name
+ '.json')).read_text())
82 assert prior
['cases'] == current
['cases'][:len(prior
['cases'])]
83 assert current
['contractCounts'] == {'pass': 176, 'fail': 4}
84 actual
= output
/ 'actual'
85 actual
.mkdir(exist_ok
=True)
86 for case
in current
['cases'][len(prior
['cases']):]:
87 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')
88 raw
= case
['actualCXTM'].encode('utf-8')
89 assert (raw
== baseline
.read_bytes()) == (case
['id'] not in discrepancies
)
90 (actual
/ baseline
.name
).write_bytes(raw
)
92 old_check
, new_check
= renamed
[name
]
93 assert [new_check
if c
== old_check
else c
for c
in prior
['checks']] == current
['checks']
94 for key
in prior
.keys() - {'checks', 'rejections'}:
95 assert prior
[key
] == current
[key
], name
+ ':' + key
96 if name
== 'templates':
97 # Original rejection class/code/source/positions stay identical.
98 # Contexts now also preserve definition/invocation provenance.
99 for a
, b
in zip(prior
['rejections'], current
['rejections']):
100 assert {k
: v
for k
, v
in a
.items() if k
!= 'context'} == {k
: v
for k
, v
in b
.items() if k
!= 'context'}
101 assert a
['context'] == b
['context'] or ':DEFINITION-SOURCE' in b
['context']
103 assert prior
== current
, name
104 example
= (ROOT
/ 'docs/reading-ctm-include.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
105 inspection
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
106 '--no-userinit', '--script', '/dev/stdin'], input=example
.encode(), cwd
=ROOT
,
107 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
108 (output
/ 'describe.log').write_bytes(inspection
.stdout
)
109 assert inspection
.returncode
== 0, 'Documented inspection forms failed'
110 control
= """(require :asdf)
111 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
112 (asdf:load-asd (truename "topic-maps.asd"))
113 (asdf:load-system "topic-maps/ctm")
114 (load (make-string-input-stream (uiop:run-program
115 (list "git" "show" "BASE:src/ctm/lexer.lisp") :output :string)))
117 (progn (topic-maps.ctm:deserialize-string
119 :document-iri "urn:base-control")
120 (error "Base unexpectedly accepted Include"))
121 (topic-maps.ctm:ctm-unsupported (c)
122 (assert (eq :unsupported-directive (topic-maps.ctm:error-code c)))
123 (format t "BASE-CONTROL-PASS: committed lexer rejected Include as ~A~%" c)))
124 """.replace('BASE:', BASE
+ ':')
125 probe
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
126 '--no-userinit', '--script', '/dev/stdin'], input=control
.encode(), cwd
=ROOT
,
127 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
128 (output
/ 'base-control.log').write_bytes(probe
.stdout
)
129 assert probe
.returncode
== 0, 'Base lexer control failed'
130 for path
, value
in protected
.items():
131 assert sha(ROOT
/ path
) == value
, path
132 assert not subprocess
.check_output(['git', 'diff', BASE
, '--', 'vendor', 'references', 'src/tmdm', 'src/cxtm'], cwd
=ROOT
)
133 for path
, value
in sources
.items():
134 assert sha(ROOT
/ path
) == value
, path
135 previous_totals
= json
.loads((old
/ 'execution.json').read_text())['regressionChecks']
136 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1))
137 for marker
in [*previous_totals
, 'INCLUDE']}
138 assert {k
: totals
[k
] for k
in previous_totals
} == previous_totals
139 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-19c', runtime
=version
,
140 startedAtUTC
=started
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
141 command
=command
, exitCode
=result
.returncode
, freshSBCL
=True, network
='denied by sandbox-exec',
142 regressionChecks
=totals
, totalLispChecks
=sum(totals
.values()), importedStatuses
=now
['statuses'],
143 importedStatusChanges
=changes
, retainedOriginalCXTMOutputsAndModelObservationsUnchanged
=True,
144 originalCorpusAndReferencesUnchanged
=True, tmdmAndCxtmSourcesUnchanged
=True,
145 documentedInspectionPassed
=True, baseLexerCounterprobePassed
=True, sourceSHA256
=sources
,
146 qualification
='CTM Include with immediate-parent Document/Wildcard environments, per-level aliases and source-preserving Templates. All twelve originals execute: seven byte-equal positives, three rejections, two explicit mismatches. Two retained DateTime FAILs unchanged; XTM remains NOT-RUN.',
147 artifactsSHA256
={str(p
.relative_to(output
)): sha(p
) for p
in sorted(output
.rglob('*'))
148 if p
.is_file() and p
.name
!= 'execution.json'})
149 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
150 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys
=True))
151 print('SLICE-19C-VALIDATION-PASS: fresh offline regression; seven byte-exact Include originals, three rejections, two original mismatches; retained contracts unchanged')
154 if __name__
== '__main__':