2 """Validate Slice 18 in a fresh offline process and preserve all prior evidence."""
14 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
15 BASE
= '963f2a09444100d37e49cccbd6634879bd6742b5'
16 POLICY
= '(version 1)(allow default)(deny network*)'
17 NAMES
= ['validation', 'semantics', 'merging', 'associations', 'cxtm', 'occurrences',
18 'prefixes', 'integers', 'decimals', 'dates', 'datetimes', 'timezones',
19 'typed-names', 'scopes', 'variants', 'references', 'identifiers', 'reification', 'arity', 'relationships', 'templates']
20 OPEN
= ['ctm/in/' + n
+ '.ctm' for n
in ['template-subject-identifier2', 'template-ambiguous-parameter5', 'template-ambiguous-parameter6']]
21 OPEN
+= ['ctm/invalid/' + n
+ '.ctm' for n
in ['template-item-identifier3', 'template-item-identifier-2nd-arg',
22 'template-subject-locator3', 'template-subject-locator-2nd-arg']]
23 IMPORTS
= ['ctm/in/include-template.ctm', 'ctm/in/include-template-wildcard.ctm',
24 'ctm/in/mergemap-template.ctm', 'ctm/invalid/mergemap-template.ctm']
28 return hashlib
.sha256(path
.read_bytes()).hexdigest()
32 parser
= argparse
.ArgumentParser(description
=__doc__
)
33 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
34 parser
.add_argument('--output', default
=str(ROOT
/ 'reports/slice-18'))
35 args
= parser
.parse_args()
36 runtime
= shutil
.which(args
.sbcl
)
38 raise RuntimeError('SBCL executable not found: ' + args
.sbcl
)
39 output
= pathlib
.Path(args
.output
).resolve()
40 output
.mkdir(parents
=True, exist_ok
=True)
41 protected
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['vendor/cxtm', 'references']
42 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
43 sources
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['src', 'tests', 'scripts', 'docs']
44 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file() and
45 '__pycache__' not in p
.parts
and p
.suffix
in ['.lisp', '.py', '.sh', '.md']}
46 sources
.update({p
: sha(ROOT
/ p
) for p
in ['topic-maps.asd', 'README.md']})
47 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
48 version
= subprocess
.check_output([runtime
, '--version'], text
=True).strip()
49 assert version
== 'SBCL 2.5.4', version
50 with tempfile
.TemporaryDirectory(prefix
='ctm-slice-18-validation-') as temp
:
51 report
= pathlib
.Path(temp
) / 'validation.json'
52 command
= ['/usr/bin/sandbox-exec', '-p', POLICY
, '/usr/bin/env', 'PYTHONDONTWRITEBYTECODE=1',
53 'SBCL=' + runtime
, 'TOPIC_MAPS_REPORT=' + str(report
), str(ROOT
/ 'scripts/test.sh')]
54 result
= subprocess
.run(command
, cwd
=ROOT
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
55 (output
/ 'tests.log').write_bytes(result
.stdout
)
57 raise RuntimeError('Full regression failed: see ' + str(output
/ 'tests.log'))
58 log
= result
.stdout
.decode('utf-8')
60 source
= report
if name
== 'validation' else pathlib
.Path(str(report
) + '.' + name
+ '.json')
61 (output
/ (name
+ '.json')).write_bytes(source
.read_bytes())
62 old
= ROOT
/ 'reports/slice-17b'
63 previous
= json
.loads((old
/ 'validation.json').read_text())
64 now
= json
.loads((output
/ 'validation.json').read_text())
65 templates
= json
.loads((output
/ 'templates.json').read_text())
66 selected
= sorted(c
['id'] for c
in templates
['cases'] + templates
['rejections'])
67 assert len(selected
) == len(set(selected
)) == 67
68 assert now
['statuses'] == {'pass': 187, 'fail': 2, 'unsupported': 0, 'not-run': 84}
69 assert [c
['id'] for c
in previous
['cases']] == [c
['id'] for c
in now
['cases']]
70 changes
= [(a
['id'], a
['status'], b
['status']) for a
, b
in zip(previous
['cases'], now
['cases']) if a
['status'] != b
['status']]
71 assert changes
== [(id, 'not-run', 'pass') for id in selected
]
72 for a
, b
in zip(previous
['cases'], now
['cases']):
73 if a
['id'] not in selected
:
74 assert a
== b
, a
['id']
75 if b
['id'] in OPEN
+ IMPORTS
:
76 assert b
['status'] == 'not-run' and b
['actualOutcome'] is None
77 for name
in NAMES
[1:-1]:
78 key
= 'lexicalProbes' if name
== 'timezones' else 'examples' if name
== 'identifiers' else 'original' if name
== 'arity' else 'cases'
79 prior
= json
.loads((old
/ (name
+ '.json')).read_text())[key
]
80 current
= json
.loads((output
/ (name
+ '.json')).read_text())[key
]
81 assert prior
== (current
[:len(prior
)] if name
== 'cxtm' else current
), name
82 extra
= 'negativeCases' if name
== 'reification' else 'witnesses' if name
== 'arity' else None
84 assert json
.loads((old
/ (name
+ '.json')).read_text())[extra
] == json
.loads((output
/ (name
+ '.json')).read_text())[extra
], name
85 cxtm
= json
.loads((output
/ 'cxtm.json').read_text())
86 assert cxtm
['contractCounts'] == {'pass': 165, 'fail': 2}
87 actual_dir
= output
/ 'actual'
88 actual_dir
.mkdir(exist_ok
=True)
89 for case
in cxtm
['cases']:
90 if case
['id'] in selected
:
91 actual
= case
['actualCXTM'].encode('utf-8')
92 baseline
= ROOT
/ 'vendor/cxtm/ctm/baseline' / (pathlib
.Path(case
['id']).name
+ '.cxtm')
93 assert actual
== baseline
.read_bytes()
94 (actual_dir
/ baseline
.name
).write_bytes(actual
)
95 example
= (ROOT
/ 'docs/reading-ctm-templates.md').read_text().split('```lisp\n', 1)[1].split('\n```', 1)[0]
96 inspection
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
97 '--no-userinit', '--script', '/dev/stdin'], input=example
.encode(), cwd
=ROOT
,
98 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
99 (output
/ 'describe.log').write_bytes(inspection
.stdout
)
100 assert inspection
.returncode
== 0, 'Documented inspection forms failed'
101 # Load committed BASE definitions in a separate process. No checkout/cache
102 # changes, no second model and no persistent runtime are involved.
103 control
= '''(require :asdf)
104 (asdf:initialize-source-registry '(:source-registry :ignore-inherited-configuration))
105 (asdf:load-asd (truename "topic-maps.asd"))
106 (asdf:load-system "topic-maps/tmdm")
107 (dolist (path '("package" "conditions" "unicode" "iri" "lexer" "reader"))
108 (load (make-string-input-stream
109 (uiop:run-program (list "git" "show" (concatenate 'string "BASE:src/ctm/" path ".lisp")) :output :string))))
110 (dolist (text '("def f() a. end f()" "a f()." "$x."))
112 (progn (topic-maps.ctm:deserialize-string text :document-iri "https://example.invalid/base.ctm")
113 (error "Base unexpectedly supports Templates"))
114 (topic-maps.ctm:ctm-unsupported (condition)
115 (format t "BASE-CONTROL-PASS: ~S => ~A~%" text (topic-maps.ctm:error-code condition)))))
116 '''.replace('BASE:', BASE
+ ':')
117 probe
= subprocess
.run(['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
, '--noinform', '--no-sysinit',
118 '--no-userinit', '--script', '/dev/stdin'], input=control
.encode(), cwd
=ROOT
,
119 stdout
=subprocess
.PIPE
, stderr
=subprocess
.STDOUT
)
120 (output
/ 'base-control.log').write_bytes(probe
.stdout
)
121 assert probe
.returncode
== 0, 'Base control failed'
122 for path
, value
in protected
.items():
123 assert sha(ROOT
/ path
) == value
, path
124 assert not subprocess
.check_output(['git', 'diff', BASE
, '--', 'vendor', 'references', 'src/tmdm', 'src/cxtm'], cwd
=ROOT
)
125 for path
, value
in sources
.items():
126 assert sha(ROOT
/ path
) == value
, path
127 previous_totals
= json
.loads((old
/ 'execution.json').read_text())['regressionChecks']
128 totals
= {marker
: int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1))
129 for marker
in [*previous_totals
, 'TEMPLATES']}
130 assert {k
: totals
[k
] for k
in previous_totals
} == previous_totals
131 evidence
= dict(baseCommit
=BASE
, branch
='codex/ctm-slice-18', runtime
=version
,
132 startedAtUTC
=started
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
133 command
=command
, exitCode
=result
.returncode
, freshSBCL
=True, network
='denied by sandbox-exec',
134 regressionChecks
=totals
, totalLispChecks
=sum(totals
.values()), importedStatuses
=now
['statuses'],
135 importedStatusChanges
=changes
, prior108PassingCXTMAndTwoKnownFailuresUnchanged
=True,
136 allRetainedModelObservationsUnchanged
=True, originalCorpusAndReferencesUnchanged
=True,
137 tmdmAndCxtmSourcesUnchanged
=True, documentedInspectionPassed
=True, baseControlPassed
=True,
138 openGrammarContracts
=OPEN
, unimplementedImportContracts
=IMPORTS
, sourceSHA256
=sources
,
139 qualification
='Templates expand supported CTM statements into the existing TMDM. Seven grammar cases and Import-dependent contracts remain NOT-RUN. The two DateTime baseline FAILs remain unchanged.',
140 artifactsSHA256
={str(p
.relative_to(output
)): sha(p
) for p
in sorted(output
.rglob('*'))
141 if p
.is_file() and p
.name
!= 'execution.json'})
142 (output
/ 'execution.json').write_text(json
.dumps(evidence
, ensure_ascii
=False, indent
=2) + '\n')
143 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'regressionChecks', 'totalLispChecks', 'importedStatuses']}, sort_keys
=True))
144 print('SLICE-18-VALIDATION-PASS: fresh offline regression, 57 original byte comparisons, 10 specific rejections; prior results unchanged')
147 if __name__
== '__main__':