]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-19d1.py
2 """Fresh offline XTM identity checks plus the unchanged XML and CTM regressions."""
8 from pathlib
import Path
14 ROOT
= Path(__file__
).resolve().parent
.parent
15 BASE
= '1691c8e0712e30666de2e667a5050eccb0086f71'
16 POLICY
= '(version 1)(allow default)(deny network*)'
20 return subprocess
.check_output(args
, cwd
=ROOT
, text
=True).strip()
24 return hashlib
.sha256(path
.read_bytes()).hexdigest()
28 parser
= argparse
.ArgumentParser(description
=__doc__
)
29 parser
.add_argument('--sbcl', default
=os
.environ
.get('SBCL', 'sbcl'))
30 parser
.add_argument('--output', type=Path
, default
=ROOT
/ 'reports/slice-19d1')
31 args
= parser
.parse_args()
32 runtime
= shutil
.which(args
.sbcl
)
33 assert runtime
and command([runtime
, '--version']) == 'SBCL 2.5.4'
34 output
= args
.output
.resolve()
35 output
.mkdir(parents
=True, exist_ok
=True)
36 protected_paths
= ['src/tmdm', 'src/ctm', 'src/cxtm', 'src/xml', 'patches', 'flake.nix',
37 'flake.lock', 'vendor', 'references', 'scripts/prepare-cxml.py',
38 'tests/cxml-probe.lisp', 'tests/cxml-encoding-regressions.lisp', 'scripts/test.sh']
39 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
40 protected
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['vendor/cxtm', 'references', 'patches']
41 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file()}
42 sources
= {str(p
.relative_to(ROOT
)): sha(p
) for directory
in ['src', 'tests', 'scripts', 'docs']
43 for p
in (ROOT
/ directory
).rglob('*') if p
.is_file() and p
.suffix
in ['.lisp', '.py', '.sh', '.md']}
44 sources
.update({p
: sha(ROOT
/ p
) for p
in ['topic-maps.asd', 'README.md', 'flake.nix', 'flake.lock']})
45 started
= datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
46 # The XTM source must have exactly one XML parse entry, the qualified boundary.
47 xtm_source
= '\n'.join(p
.read_text() for p
in sorted((ROOT
/ 'src/xtm').glob('*.lisp')))
48 assert xtm_source
.count('(topic-maps.xml:parse-octets ') == 1
49 assert not re
.search(r
'\(cxml:parse[-\w]*\s', xtm_source
, re
.I
)
51 with tempfile
.TemporaryDirectory(prefix
='topic-maps-19d1-') as tmp
:
53 preparation
= ['python3', 'scripts/prepare-cxml.py', '--directory', str(tmp
/ 'sources')]
54 provenance
= json
.loads(command(preparation
))
55 environment
= dict(os
.environ
, CL_SOURCE_REGISTRY
=provenance
['sourceRegistry'], PYTHONDONTWRITEBYTECODE
='1')
56 for label
, script
in [('xml', 'tests/cxml-probe.lisp'), ('xtm-1', 'scripts/fresh-xtm.lisp'),
57 ('xtm-2', 'scripts/fresh-xtm.lisp')]:
58 env
= dict(environment
)
59 if label
.startswith('xtm'):
60 env
['TOPIC_MAPS_XTM_REPORT'] = str(output
/ (label
+ '.json'))
61 invocation
= ['/usr/bin/sandbox-exec', '-p', POLICY
, runtime
,
62 '--noinform', '--no-sysinit', '--no-userinit', '--script', script
]
63 context
= {'label': label
, 'command': invocation
, 'freshProcess': True,
64 'network': 'denied by sandbox-exec',
65 'startedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()}
66 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=env
, stdout
=subprocess
.PIPE
,
67 stderr
=subprocess
.STDOUT
, timeout
=120)
68 (output
/ (label
+ '.log')).write_bytes(result
.stdout
)
69 context
.update(exitCode
=result
.returncode
, finishedAtUTC
=datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat())
70 assert result
.returncode
== 0, label
+ ' failed; see saved log'
71 prefix
= 'CXML' if label
== 'xml' else 'XTM'
72 passed
= re
.findall(r
'^' + prefix
+ r
'-CHECK-PASS (.+)$', result
.stdout
.decode(), re
.M
)
73 assert len(passed
) == len(set(passed
))
74 assert prefix
+ '-SUMMARY ' + str(len(passed
)) + ' PASS 0 FAIL' in result
.stdout
.decode()
75 context
['passedChecks'] = len(passed
)
77 assert len(passed
) == 146
79 report
= json
.loads((output
/ (label
+ '.json')).read_text())
80 assert report
['checks'] == passed
81 assert report
['statuses'] == {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0}
82 assert report
['helperSource']['status'] == 'pass'
83 assert len(report
['cases']) == 6
84 for case
in report
['actualAndExpected']:
85 assert case
['actual'].encode() == case
['expected'].encode()
86 assert report
['otherCorpusInputs'] == {'positiveUnexecuted': 107, 'negativeUnregisteredUnexecuted': 11}
88 assert (output
/ 'xtm-1.json').read_bytes() == (output
/ 'xtm-2.json').read_bytes()
89 regression_dir
= tmp
/ 'regression'
90 invocation
= ['python3', 'scripts/validate-slice-19c.py', '--sbcl', runtime
, '--output', str(regression_dir
)]
91 result
= subprocess
.run(invocation
, cwd
=ROOT
, env
=environment
, stdout
=subprocess
.PIPE
,
92 stderr
=subprocess
.STDOUT
, timeout
=180)
93 (output
/ 'regression-driver.log').write_bytes(result
.stdout
)
94 assert result
.returncode
== 0, 'Retained regression failed'
95 regression
= json
.loads((regression_dir
/ 'execution.json').read_text())
96 assert regression
['totalLispChecks'] == 12100
97 assert regression
['importedStatuses'] == {'pass': 204, 'fail': 4, 'unsupported': 0, 'not-run': 65}
99 for previous
in sorted((ROOT
/ 'reports/slice-19c').glob('*.json')):
100 if previous
.name
!= 'execution.json':
101 assert (regression_dir
/ previous
.name
).read_bytes() == previous
.read_bytes(), previous
.name
102 identical
.append(previous
.name
)
103 for name
in ['tests.log', 'execution.json']:
104 shutil
.copyfile(regression_dir
/ name
, output
/ ('regression-' + name
))
105 assert protected
== {p
: sha(ROOT
/ p
) for p
in protected
}
106 assert sources
== {p
: sha(ROOT
/ p
) for p
in sources
}
107 assert not command(['git', 'diff', BASE
, '--'] + protected_paths
)
108 evidence
= {'baseCommit': BASE
, 'branch': command(['git', 'branch', '--show-current']),
109 'startedAtUTC': started
, 'finishedAtUTC': datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat(),
110 'runtime': 'SBCL 2.5.4', 'newDependencyDownloads': 0, 'sourcePreparationCommand': preparation
,
111 'qualifiedSourceProvenance': provenance
, 'sourceSHA256': sources
,
112 'protectedCorpusAndDecoderPatchesUnchanged': True, 'xmlBoundaryStaticAndRuntimeChecks': 'PASS',
113 'runs': runs
, 'ctmStatuses': regression
['importedStatuses'],
114 'retainedLispChecks': regression
['totalLispChecks'], 'retainedChecks': regression
['regressionChecks'],
115 'retainedRegressionCommand': invocation
, 'retainedReportsByteIdentical': identical
,
116 'xtmStatuses': {'pass': 6, 'fail': 0, 'unsupported': 0, 'not-run': 0},
117 'unexecutedOtherXTMInputs': {'positive': 107, 'negative': 11},
118 'directXTMHelperSource': 'PASS; 5 Topics, 1 Association, 2 Roles; not a CTM Mergemap PASS',
119 'deterministicXTMReports': True, 'persistentLispImagesTouched': False,
120 'fullOfflineNixRebuild': 'NOT-RUN; original Store sources, verified existing local patches and fresh offline ASDF loads.'}
121 (output
/ 'execution.json').write_text(json
.dumps(evidence
, indent
=2) + '\n')
122 print(json
.dumps({k
: evidence
[k
] for k
in ['runtime', 'runs', 'ctmStatuses', 'retainedLispChecks', 'xtmStatuses']}, indent
=2))
125 if __name__
== '__main__':