2 """Validate Slice 19B in a fresh SBCL 2.5.4 process with network access denied."""
14 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
15 BASE
= '9ce2a2b5116edfd921a5240efcb2f604f9523693'
16 POLICY
= '(version 1)(allow default)(deny network*)'
17 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity',
20 'relationships', 'templates', 'map-import', 'mergemap']
24 return hashlib
.sha256(path
.read_bytes()).hexdigest()
28 parser
= argparse
.ArgumentParser(description
=__doc__
)
29 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
30 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-19b'))
31 args
= parser
.parse_args()
32 runtime
= shutil
.which(args
.sbcl
)
33 assert runtime
is not None, 'SBCL executable not found'
34 output
= pathlib
.Path(args
.output
).resolve()
35 output
.mkdir(parents
=True, exist_ok
=True)
36 protected
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['vendor/cxtm', 'references']
37 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
38 sources
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['src', 'tests', 'scripts', 'docs']
39 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file() and
40 '__pycache__' not in p
.parts
and p
.suffix
in ['.lisp', '.py', '.sh', '.md']}
41 sources
.update({p
: sha(ROOT
/ p
) for p
in ['topic-maps.asd', 'README.md']})
42 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
43 version
= subprocess
.check_output([runtime
, '--version'], text
=True).strip()
44 assert version
== 'SBCL 2.5.4', version
45 with tempfile
.TemporaryDirectory(prefix
='tmdm-slice-19b-validation-') as temp
:
46 report
= pathlib
.Path(temp
) / 'validation.json'
47 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
48 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(ROOT
/ 'scripts/test.sh')]
49 result
= subprocess
.run(command
, cwd
=ROOT
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
50 (output
/ 'tests.log').write_bytes(result
.stdout
)
52 raise RuntimeError('Full regression failed: see ' + str(output
/ 'tests.log'))
53 log
= result
.stdout
.decode('utf-8')
55 source
= report
if name
== 'validation' else pathlib
.Path(str(report
) + '.' + name
+ '.json')
56 (output
/ (name
+ '.json')).write_bytes(source
.read_bytes())
57 old
= ROOT
/ 'reports/slice-19a'
58 previous
= json
.loads((old
/ 'validation.json').read_text())
59 now
= json
.loads((output
/ 'validation.json').read_text())
60 assert now
['statuses'] == {'pass': 194, 'fail': 2, 'unsupported': 0, 'not-run': 77}
61 selected
= sorted('ctm/in/' + n
+ '.ctm' for n
in ['mergemap', 'mergemap-syntax-qname', 'mergemap-template', 'mergemap-tm-reifier'])
62 selected
+= sorted('ctm/invalid/' + n
+ '.ctm' for n
in ['mergemap-missing-syntax-iri', 'mergemap-template', 'mergemap-unknown-syntax'])
63 changes
= [(a
['id'], a
['status'], b
['status']) for a
, b
in zip(previous
['cases'], now
['cases']) if a
['status'] != b
['status']]
64 assert changes
== [(id, 'not-run', 'pass') for id in selected
]
65 assert [c
['id'] for c
in previous
['cases']] == [c
['id'] for c
in now
['cases']]
66 for a
, b
in zip(previous
['cases'], now
['cases']):
67 if a
['id'] not in selected
:
68 assert a
== b
, a
['id']
69 for name
in NAMES
[1:-1]:
70 prior
= json
.loads((old
/ (name
+ '.json')).read_text())
71 current
= json
.loads((output
/ (name
+ '.json')).read_text())
73 assert prior
['cases'] == current
['cases'][:len(prior
['cases'])]
74 assert current
['contractCounts'] == {'pass': 169, 'fail': 2}
75 actual
= output
/ 'actual'
76 actual
.mkdir(exist_ok
=True)
77 for case
in current
['cases'][len(prior
['cases']):]:
78 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')
79 raw
= case
['actualCXTM'].encode('utf-8')
80 assert raw
== baseline
.read_bytes()
81 (actual
/ baseline
.name
).write_bytes(raw
)
82 elif name
== 'templates':
83 # One former UNSUPPORTED-feature assertion becomes the specific
84 # unknown-notation diagnostic. No old original contract is dropped.
85 old_check
= 'diagnostic:unchanged-excluded-feature:%mergemap <other> <urn:ctm>'
86 new_check
= 'diagnostic:unknown-mergemap-notation'
87 assert old_check
in prior
['checks'] and new_check
in current
['checks']
88 assert set(prior
['checks']) - {old_check}
== set(current
['checks']) - {new_check}
89 for key
in prior
.keys() - {'checks'}:
90 assert prior
[key
] == current
[key
], name
+ ':' + key
92 assert prior
== current
, name
93 example
= (ROOT
/ 'docs/reading-ctm-mergemap.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
94 inspection
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
95 '--no-userinit', '--script', '/dev/stdin'], input=example
.encode(), cwd
=ROOT
,
96 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
97 (output
/ 'describe.log').write_bytes(inspection
.stdout
)
98 assert inspection
.returncode
== 0, 'Documented inspection forms failed'
99 control
= """(require :asdf)
100 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
101 (asdf:load-asd (truename "topic-maps.asd"))
102 (asdf:load-system "topic-maps/ctm")
103 (load (make-string-input-stream (uiop:run-program
104 (list "git" "show" "BASE:src/ctm/lexer.lisp") :output :string)))
106 (progn (topic-maps.ctm:deserialize-string
107 "%mergemap <child> <http://psi.topicmaps.org/iso13250/ctm>"
108 :document-iri "urn:base-control")
109 (error "Base unexpectedly accepted Mergemap"))
110 (topic-maps.ctm:ctm-unsupported (c)
111 (assert (eq :unsupported-directive (topic-maps.ctm:error-code c)))
112 (format t "BASE-CONTROL-PASS: committed lexer rejected Mergemap as ~A~%" c)))
113 """.replace('BASE:', BASE
+ ':')
114 probe
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
115 '--no-userinit', '--script', '/dev/stdin'], input=control
.encode(), cwd
=ROOT
,
116 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
117 (output
/ 'base-control.log').write_bytes(probe
.stdout
)
118 assert probe
.returncode
== 0, 'Base lexer control failed'
119 for path
, value
in protected
.items():
120 assert sha(ROOT
/ path
) == value
, path
121 assert not subprocess
.check_output(['git', 'diff', BASE
, '--', 'vendor', 'references', 'src/tmdm', 'src/cxtm'], cwd
=ROOT
)
122 for path
, value
in sources
.items():
123 assert sha(ROOT
/ path
) == value
, path
124 previous_totals
= json
.loads((old
/ 'execution.json').read_text())['regressionChecks']
125 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1))
126 for marker
in [*previous_totals
, 'MERGEMAP']}
127 assert {k
: totals
[k
] for k
in previous_totals
} == previous_totals
128 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-19b', runtime
=version
,
129 startedAtUTC
=started
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
130 command
=command
, exitCode
=result
.returncode
, freshSBCL
=True, network
='denied by sandbox-exec',
131 regressionChecks
=totals
, totalLispChecks
=sum(totals
.values()), importedStatuses
=now
['statuses'],
132 importedStatusChanges
=changes
, retainedOriginalCXTMOutputsAndModelObservationsUnchanged
=True,
133 originalCorpusAndReferencesUnchanged
=True, tmdmAndCxtmSourcesUnchanged
=True,
134 documentedInspectionPassed
=True, baseLexerCounterprobePassed
=True, sourceSHA256
=sources
,
135 qualification
='CTM Mergemap via an explicit finite bytes resolver and Slice 19A import. Four positives and three intended rejections newly executed. Include and XTM remain NOT-RUN; two DateTime FAILs unchanged.',
136 artifactsSHA256
={str(p
.relative_to(output
)): sha(p
) for p
in sorted(output
.rglob('*'))
137 if p
.is_file() and p
.name
!= 'execution.json'})
138 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
139 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys
=True))
140 print('SLICE-19B-VALIDATION-PASS: fresh offline regression; four byte-exact Mergemap originals and three intended rejections; prior results unchanged')
143 if __name__
== '__main__':