2 """Reproduce documentary observations and the unchanged offline regression."""
9 from pathlib
import Path
15 ROOT
= Path(__file__
).resolve().parents
[2]
16 EVIDENCE
= Path(__file__
).resolve().parent
17 BASE
= 'bc04a1657f0b548ec5cdf2862aeed6a1ad0630b8'
18 POLICY
= '(version 1)(allow default)(deny network*)'
19 READ_ONLY
= POLICY
+ '(deny file-write* (subpath "' + str(ROOT
) + '"))'
23 return json
.loads(path
.read_text())
26 def save(path
, value
):
27 path
.write_text(json
.dumps(value
, indent
=2, ensure_ascii
=False) + '\n')
31 return hashlib
.sha256(path
.read_bytes()).hexdigest()
34 def command(args
, **kwargs
):
35 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True, **kwargs
).strip()
38 def load_module(name
, path
):
39 spec
= importlib
.util
.spec_from_file_location(name
, path
)
40 module
= importlib
.util
.module_from_spec(spec
)
41 spec
.loader
.exec_module(module
)
45 def prepare_pinned(target
):
46 # Reuse the existing verifier/patcher with the already recorded Store
47 # manifest. No Nix evaluation or daemon/network connection is needed.
48 prior
= read(ROOT
/ 'reports/slice-20h/source-preparation.json')
49 module
= load_module('prepare_cxml', ROOT
/ 'scripts/prepare-cxml.py')
50 environment
= {'manifest': prior
['sources'],
51 'localPatches': [{'package': p
['package'], 'sha256': p
['sha256']} for p
in prior
['patches']]}
52 return module
.prepare(environment
, target
)
56 parser
= argparse
.ArgumentParser(description
=__doc__
)
57 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
58 parser
.add_argument('--output', type=Path
, default
=EVIDENCE
)
59 parser
.add_argument('--prepare', type=Path
)
60 args
= parser
.parse_args()
62 print(json
.dumps(prepare_pinned(args
.prepare
)))
64 runtime
= shutil
.which(args
.sbcl
)
65 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
66 subprocess
.run(['git', 'merge-base', '--is-ancestor', BASE
, 'HEAD'], cwd
=ROOT
, check
=True)
67 output
= args
.output
.resolve()
68 output
.mkdir(parents
=True, exist_ok
=True)
69 assert not (output
/ 'execution.json').exists(), 'Choose an unused output directory'
70 tracked
= command(['git', 'ls-tree', '-r', '--name-only', BASE
]).splitlines()
71 assert not command(['git', 'diff', BASE
, '--'] + tracked
)
72 protected
= {p
: sha(ROOT
/ p
) for p
in tracked
}
73 historical
= {str(p
.relative_to(ROOT
)): sha(p
) for p
in (ROOT
/ 'reports').rglob('*')
74 if p
.is_file() and output
not in p
.parents
}
75 inputs
= {str(p
.relative_to(ROOT
)): sha(p
) for p
in [*EVIDENCE
.glob('*.py'), *EVIDENCE
.glob('*.lisp'),
76 EVIDENCE
/ 'import.json', ROOT
/ 'docs/slice-20j-template-tail-grammar.md', *sorted((EVIDENCE
/ 'slice20i').glob('*'))]}
77 imported
= read(EVIDENCE
/ 'import.json')
78 assert imported
['baseCommit'] == BASE
79 for name
, record
in imported
['files'].items():
80 path
= EVIDENCE
/ 'slice20i' / name
81 assert path
.stat().st_size
== record
['bytes'] and sha(path
) == record
['sha256']
82 old
= read(EVIDENCE
/ 'slice20i/investigation.json')
83 old_controls
= read(EVIDENCE
/ 'slice20i/controls.json')
84 detailed
= read(EVIDENCE
/ 'slice20i/detailed-inventory.json')
85 assert read(EVIDENCE
/ 'slice20i/before-sha256.json') == protected
86 assert old
['harnessCaseStatesUnchanged'] and len(old
['cases']) == len(detailed
['cases']) == 7
87 retained
= output
/ 'retained'
90 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
92 def run(label
, invocation
, env
, policy
=POLICY
):
93 invocation
= ['/usr/bin/sandbox-exec', '-p', policy
] + invocation
94 entry
= {'label': label
, 'command': invocation
, 'network': 'denied',
95 'freshProcess': True, 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
96 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
97 stderr
=subprocess
.STDOUT
, timeout
=300)
98 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
99 entry
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
101 assert result
.returncode
== 0, f
'{label}: see {label}.log'
102 return result
.stdout
.decode()
104 with tempfile
.TemporaryDirectory(prefix
='topic-maps-20j-') as temp_name
:
105 temp
= Path(temp_name
)
106 env
= {k
: v
for k
, v
in os
.environ
.items() if not k
.startswith('TOPIC_MAPS_')}
107 env
.update(PYTHONDONTWRITEBYTECODE
='1', GIT_OPTIONAL_LOCKS
='0', SBCL
=runtime
)
108 provenance
= json
.loads(run('source-preparation',
109 ['python3', str(Path(__file__
)), '--prepare', str(temp
/ 'sources')], env
, READ_ONLY
))
110 save(output
/ 'source-preparation.json', provenance
)
111 env
.update(CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'],
112 TOPIC_MAPS_REPORT
=str(retained
/ 'core.json'), TOPIC_MAPS_XTM_REPORT
=str(retained
/ 'xtm.json'),
113 TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(retained
/ 'xtm-mergemap'),
114 TOPIC_MAPS_REGISTERED_ORIGINALS_REPORT
=str(retained
/ 'slice20a'),
115 TOPIC_MAPS_REGISTERED_NEGATIVES_REPORT
=str(retained
/ 'slice20b'),
116 TOPIC_MAPS_PROLOG_REPORT
=str(retained
/ 'slice20c'), TOPIC_MAPS_EMPTY2_REPORT
=str(retained
/ 'slice20e'),
117 TOPIC_MAPS_CTM_INTEGER_REPORT
=str(retained
/ 'slice20f'), TOPIC_MAPS_TRIPLE_REPORT
=str(retained
/ 'slice20g'),
118 TOPIC_MAPS_ENCODING_REPORT
=str(output
/ 'current'))
119 for label
, script
, marker
, count
in [('xml', 'tests/cxml-probe.lisp', 'CXML', 146),
120 ('xtm', 'scripts/fresh-xtm.lisp', 'XTM', 327)]:
121 log
= run(label
, [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', script
], env
)
122 labels
= re
.findall(r
'^' + marker
+ r
'-CHECK-PASS (.+)$', log
, re
.M
)
123 assert len(labels
) == len(set(labels
)) == count
124 assert f
'{marker}-SUMMARY {count} PASS 0 FAIL' in log
125 log
= run('regression', [str(ROOT
/ 'scripts/test.sh')], env
)
126 obs_env
= {k
: v
for k
, v
in env
.items() if not k
.startswith('TOPIC_MAPS_')}
127 obs_env
.update(TOPIC_MAPS_XTM_MERGEMAP_REPORT
=str(temp
/ 'probe-xtm'),
128 TOPIC_MAPS_20J_ROOT
=str(ROOT
), TOPIC_MAPS_20J_OUTPUT
=str(temp
))
129 driver
= temp
/ 'observe.lisp'
130 driver
.write_text('(load ' + json
.dumps(str(EVIDENCE
/ 'observations.lisp')) + ')\n'
131 '(load ' + json
.dumps(str(EVIDENCE
/ 'controls.lisp')) + ')\n')
132 run('observations', [runtime
, '--noinform', '--no-sysinit', '--no-userinit', '--script', str(driver
)], obs_env
, READ_ONLY
)
133 current
= read(temp
/ 'investigation.json')
134 controls
= read(temp
/ 'controls.json')
135 assert current
== old
, 'Original observations differ from verified Slice 20I'
136 assert controls
== old_controls
, 'Explorative controls differ from verified Slice 20I'
137 save(output
/ 'observed-originals.json', current
)
138 save(output
/ 'observed-controls.json', controls
)
140 verified_reports
= []
141 for prior
in sorted((ROOT
/ 'reports/slice-20h/retained').glob('*.json')):
142 assert read(prior
) == read(retained
/ prior
.name
), str(prior
)
143 verified_reports
.append(str(prior
.relative_to(ROOT
)))
144 for suffix
in ['validation', 'cxtm', 'encoding']:
145 prior
= ROOT
/ f
'reports/slice-20h/current.{suffix}.json'
146 assert read(prior
) == read(output
/ f
'current.{suffix}.json'), str(prior
)
147 verified_reports
.append(str(prior
.relative_to(ROOT
)))
148 final
= read(output
/ 'current.validation.json')
149 expected_statuses
= {'pass': 262, 'fail': 4, 'unsupported': 0, 'not-run': 7}
150 assert final
['statuses'] == current
['harnessStatuses'] == expected_statuses
151 pending
= [c
for c
in final
['cases'] if c
['status'] == 'not-run']
152 assert [c
['id'] for c
in pending
] == [c
['id'] for c
in current
['cases']]
154 for original
, detail
in zip(current
['cases'], detailed
['cases']):
155 assert original
['id'] == detail
['id']
156 path
= ROOT
/ 'vendor/cxtm' / original
['id']
157 raw
= path
.read_bytes()
158 assert raw
== bytes
.fromhex(original
['sourceHex']) == original
['sourceText'].encode('ascii')
159 assert len(raw
) == original
['sourceBytes'] and sha(path
) == detail
['sourceSHA256']
160 matches
= list(re
.finditer(r
'\$[A-Za-z_][A-Za-z0-9_]*', original
['sourceText']))
161 assert len(matches
) == len(detail
['variables'])
163 for match
, variable
in zip(matches
, detail
['variables']):
165 assert match
.group() == variable
['variable'] and pos
== variable
['position']
166 assert variable
['line'] == raw
[:pos
].count(b
'\n') + 1
167 assert variable
['column'] == pos
- raw
.rfind(b
'\n', 0, pos
)
168 variables
.append(dict(variable
, roleOrigin
='observed' if variable
['role'] == 'Parameterdeklaration' else 'inferred'))
169 condition
= original
['condition']
170 assert condition
['condition'] == 'INVALID-TEMPLATE-BODY' and condition
['code'] == 'template-identity-tail'
171 assert condition
['positionUnit'] == 'character' and condition
['cause'] is None and condition
['context'] is None
172 assert original
['model'] is None and original
['actualCXTM'] is None
173 case
= dict(detail
, status
='not-run', variables
=variables
, observation
=original
,
174 originalComparisonPerformed
=False, argumentCheckReached
=False,
175 evidenceOrigins
={'sourceAndCondition': 'observed', 'intendedRoles': 'inferred',
176 'grammar': 'norm-derived', 'compatibilityRule': 'hypothesized'},
177 compatibilityRule
='H-SI-tail', normativeReferences
=['N-tail', 'N-variables', 'N-templates'])
178 if original
['baselineText']:
179 baseline
= Path(original
['baselinePath'])
180 assert baseline
.read_bytes() == original
['baselineText'].encode('utf-8') and sha(baseline
) == detail
['baselineSHA256']
181 control
= detail
['expandedControl']
182 assert control
in controls
and control
['byteExact'] and control
['condition'] is None
183 actual
= control
['actualCXTM'].encode('utf-8')
184 assert actual
== baseline
.read_bytes() and len(actual
) == control
['actualBytes'] == control
['expectedBytes']
185 case
['expandedComparison'] = {'epistemicOrigin': 'observed', 'originalTemplateExecutedSuccessfully': False,
186 'actualBytes': len(actual
), 'expectedBytes': len(actual
), 'actualSHA256': hashlib
.sha256(actual
).hexdigest(),
187 'expectedSHA256': sha(baseline
), 'firstDifferentByte': None}
189 second
= '2nd-arg' in original
['id']
190 case
['intendedArgumentCheck'] = {'epistemicOrigin': 'inferred', 'argumentIndex': 2 if second
else 1,
191 'parameter': '$iri2' if second
else '$iri', 'identityKind': 'item-identifier' if 'item-identifier' in original
['id'] else 'subject-locator',
192 'targetContext': 'unmarked Subject Identifier Tail slot',
193 'basis': 'invalid corpus category, source argument marker and, for two arguments, original invalid comment'}
195 report
= {'kind': 'ctm-template-tail-investigation', 'baseCommit': BASE
, 'runtime': 'SBCL 2.5.4',
196 'corpusCommit': final
['commit'], 'statuses': expected_statuses
, 'caseStatusChanges': [],
197 'originalComparisonsPerformed': 0, 'expandedControlComparisons': 3,
198 'normativeReference': detailed
['normativeReference'],
199 'normativeFindings': [
200 {'id': 'N-tail', 'epistemicOrigin': 'norm-derived', 'sections': {'3.9': [50, 51, 52], '3.3.6': [17], '3.3.5': [12]},
201 'derivation': '[50] -> [51] -> [52] -> [17] -> [12]', 'result': 'No variable alternative in this unmarked Tail slot.'},
202 {'id': 'N-variables', 'epistemicOrigin': 'norm-derived', 'sections': {'3.3.6': [13, 18, 19], '3.3.7': [20], '3.4.1': [28]},
203 'result': 'Explicit variable uses in Topic identity/reference, marked SL/II and literal contexts.'},
204 {'id': 'N-templates', 'epistemicOrigin': 'norm-derived', 'sections': {'3.14': [63, 64, 65, 66, 67], '3.15': [68, 69, 70]},
205 'result': 'Template-only variables, positional binding and use-context constraints.'}],
206 'compatibilityHypothesis': {'id': 'H-SI-tail', 'epistemicOrigin': 'hypothesized', 'implemented': False,
207 'scope': 'Only the Subject Identifier branch of [52] inside a Template Tail.',
208 'additionalRule': 'template-tail-subject-identifier ::= iri-ref | variable',
209 'bindingRequirement': 'Variable must denote an IRI reference (IRI or resolved QName), added as an SI of the owner.',
210 'rejectedBindings': ['marked item identifier', 'marked subject locator', 'Topic-only argument', 'non-IRI literal'],
211 'identityKindsMustBePreserved': True, 'textualMarkerStrippingAllowed': False,
212 'doesNotChange': ['Topic-head variables', 'Type/Player Topic references', 'literal substitution'],
213 'historicalIntentEstablished': False},
214 'historicalQualification': detailed
['historicalQualification'], 'cases': cases
}
215 save(output
/ 'current.investigation.json', report
)
216 totals
= read(ROOT
/ 'reports/slice-19d1/regression-execution.json')['regressionChecks']
217 assert sum(totals
.values()) == 12100
218 for marker
, count
in totals
.items():
219 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == count
220 stage_counts
= {'XTM-MERGEMAP': 217, 'REGISTERED-ORIGINALS': 286, 'REGISTERED-NEGATIVES': 25,
221 'PROLOG': 171, 'EMPTY2': 13, 'CTM-INTEGER': 112, 'TRIPLE': 226, 'ENCODING': 722}
222 for marker
, count
in stage_counts
.items():
223 assert int(re
.search(r
'(?m)^' + marker
+ r
'-PASS: (\d+)', log
).group(1)) == count
224 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
225 xtm
= read(retained
/ 'xtm.json')
226 assert xtm
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
227 assert xtm
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
228 lock
= read(ROOT
/ 'references/corpus-lock.json')
229 for entry
in lock
['files']:
230 p
= ROOT
/ 'vendor/cxtm' / entry
['path']
231 assert p
.stat().st_size
== entry
['size'] and sha(p
) == entry
['sha256']
232 for snapshot
in [protected
, historical
, inputs
]:
233 assert snapshot
== {p
: sha(ROOT
/ p
) for p
in snapshot
}
234 assert not command(['git', 'diff', BASE
, '--'] + tracked
)
235 save(output
/ 'integrity.json', {'baseTrackedFilesSHA256': protected
, 'historicalReportsSHA256': historical
,
236 'documentaryInputsSHA256': inputs
})
237 save(output
/ 'execution.json', {'kind': 'ctm-template-tail-documentation-validation', 'baseCommit': BASE
,
238 'branch': command(['git', 'branch', '--show-current']), 'startedAtUTC': started
,
239 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(), 'runtime': 'SBCL 2.5.4',
240 'runs': runs
, 'sourcePreparation': provenance
, 'retainedLispChecks': 12100, 'retainedChecks': totals
,
241 'stageChecks': stage_counts
, 'xmlQualificationChecks': 146, 'standaloneXTMChecks': 327,
242 'pythonUnitTests': 2, 'nfcOracleComparisons': 16294, 'originalObservationsReproduced': 7,
243 'explorativeControlsReproduced': len(controls
), 'expandedBaselineComparisons': 3,
244 'ctmStatusesBefore': expected_statuses
, 'ctmStatusesAfter': final
['statuses'], 'ctmStatusChanges': [],
245 'xtmCorpusStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 118},
246 'allRetainedReportsEqualToSlice20H': verified_reports
, 'allBaseTrackedFileBytesUnchanged': True,
247 'baseTrackedFilesVerified': len(protected
), 'historicalReportsVerified': len(historical
),
248 'pinnedCorpusFilesVerified': len(lock
['files']), 'newDependencyDownloads': 0,
249 'languageOrRegistrationChanges': False, 'persistentLispImagesTouched': False,
250 'fullOfflineNixRebuild': 'NOT-RUN; verified local pinned Store sources and unchanged XML patches.',
251 'executionAuthorization': 'Regular require_escalated approval; no symlink or sandbox configuration changes.',
252 'acceptancePassed': True})
253 print(json
.dumps({'acceptancePassed': True, 'ctm': final
['statuses'], 'xtm': {'pass': 6, 'fail': 0, 'not-run': 118},
254 'originalObservations': 7, 'expandedComparisons': 3, 'protectedBaseFiles': len(protected
)}))
257 if __name__
== '__main__':