]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/corpus.py
Support CTM associations with arbitrarily many roles
[topic-maps.git] / scripts / corpus.py
1 #!/usr/bin/env python3
2 """Acquisition/integrity only. No CTM parsing, XML comparison or network in verify."""
3 import argparse
4 import hashlib
5 import io
6 import json
7 import pathlib
8 import subprocess
9 import tarfile
10 import tempfile
11
12 ROOT = pathlib.Path(__file__).resolve().parent.parent
13 LOCK = ROOT / 'references/corpus-lock.json'
14 DESTINATION = ROOT / 'vendor/cxtm'
15
16
17 def inventory(root):
18 return [dict(path=p.relative_to(root).as_posix(), size=p.stat().st_size,
19 sha256=hashlib.sha256(p.read_bytes()).hexdigest())
20 for p in sorted(root.rglob('*')) if p.is_file()]
21
22
23 def recount(root):
24 ctm = root / 'ctm'
25 valid = sorted((ctm / 'in').rglob('*.ctm'))
26 invalid = sorted((ctm / 'invalid').rglob('*.ctm'))
27 baselines = sorted((ctm / 'baseline').rglob('*.cxtm'))
28 used = set(valid + invalid + baselines)
29 auxiliary = sorted(p for p in ctm.rglob('*') if p.is_file() and p not in used)
30 expected = {ctm / 'baseline' / (p.relative_to(ctm / 'in').as_posix()+'.cxtm') for p in valid}
31 return dict(valid=len(valid), invalid=len(invalid), baselines=len(baselines),
32 auxiliary=len(auxiliary),
33 missingBaselines=sorted(p.relative_to(root).as_posix() for p in expected if not p.is_file()),
34 unmatchedBaselines=sorted(p.relative_to(root).as_posix() for p in baselines if p not in expected))
35
36
37 def verify(lock, root=DESTINATION):
38 if not root.is_dir():
39 raise SystemExit('Corpus absent; run scripts/corpus.py acquire once with network access.')
40 actual = inventory(root)
41 if actual != lock['files']:
42 expected_map = {e['path']: e for e in lock['files']}
43 actual_map = {e['path']: e for e in actual}
44 changed = sorted(k for k in expected_map.keys() | actual_map.keys()
45 if expected_map.get(k) != actual_map.get(k))
46 raise SystemExit('CORPUS-INTEGRITY-FAIL: ' + ', '.join(changed[:20]))
47 return recount(root)
48
49
50 def acquire(lock):
51 if DESTINATION.exists():
52 verify(lock)
53 print('Pinned corpus already present and verified; no files overwritten.')
54 return
55 with tempfile.TemporaryDirectory(prefix='topic-maps-corpus-') as temporary:
56 checkout = pathlib.Path(temporary) / 'upstream'
57 subprocess.run(['git', 'clone', '--quiet', lock['repository'] + '.git', str(checkout)], check=True)
58 subprocess.run(['git', 'cat-file', '-e', lock['commit'] + '^{commit}'], cwd=checkout, check=True)
59 archive = subprocess.check_output(['git', 'archive', '--format=tar', lock['commit']], cwd=checkout)
60 if hashlib.sha256(archive).hexdigest() != lock['archiveSHA256']:
61 raise SystemExit('Pinned archive hash mismatch; acquisition stopped.')
62 staging = pathlib.Path(temporary) / 'corpus'
63 staging.mkdir()
64 with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
65 for member in tar.getmembers():
66 parts = pathlib.PurePosixPath(member.name)
67 if parts.is_absolute() or '..' in parts.parts or not (member.isfile() or member.isdir()):
68 raise SystemExit('Unsafe upstream archive entry: ' + member.name)
69 tar.extractall(staging)
70 verify(lock, staging)
71 DESTINATION.parent.mkdir(exist_ok=True)
72 # Copy into the absent destination; never merge over existing contents.
73 import shutil
74 shutil.copytree(staging, DESTINATION)
75 verify(lock)
76
77
78 def main():
79 parser = argparse.ArgumentParser(description=__doc__)
80 parser.add_argument('operation', choices=['verify', 'acquire'])
81 args = parser.parse_args()
82 lock = json.loads(LOCK.read_text())
83 if args.operation == 'acquire':
84 acquire(lock)
85 result = verify(lock)
86 print('CORPUS-INTEGRITY-PASS: ' + json.dumps(result, sort_keys=True))
87
88
89 if __name__ == '__main__':
90 main()