]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/corpus.py
2 """Acquisition/integrity only. No CTM parsing, XML comparison or network in verify."""
12 ROOT
= pathlib
.Path(__file__
).resolve().parent
.parent
13 LOCK
= ROOT
/ 'references/corpus-lock.json'
14 DESTINATION
= ROOT
/ 'vendor/cxtm'
18 return [dict(path
=p
.relative_to(root
).as_posix(), size
=p
.stat().st_size
,
19 sha256
=hashlib
.sha256(p
.read_bytes()).hexdigest())
20 for p
in sorted(root
.rglob('*')) if p
.is_file()]
25 valid
= sorted((ctm
/ 'in').rglob('*.ctm'))
26 invalid
= sorted((ctm
/ 'invalid').rglob('*.ctm'))
27 baselines
= sorted((ctm
/ 'baseline').rglob('*.cxtm'))
28 used
= set(valid
+ invalid
+ baselines
)
29 auxiliary
= sorted(p
for p
in ctm
.rglob('*') if p
.is_file() and p
not in used
)
30 expected
= {ctm
/ 'baseline' / (p
.relative_to(ctm
/ 'in').as_posix()+'.cxtm') for p
in valid
}
31 return dict(valid
=len(valid
), invalid
=len(invalid
), baselines
=len(baselines
),
32 auxiliary
=len(auxiliary
),
33 missingBaselines
=sorted(p
.relative_to(root
).as_posix() for p
in expected
if not p
.is_file()),
34 unmatchedBaselines
=sorted(p
.relative_to(root
).as_posix() for p
in baselines
if p
not in expected
))
37 def verify(lock
, root
=DESTINATION
):
39 raise SystemExit('Corpus absent; run scripts/corpus.py acquire once with network access.')
40 actual
= inventory(root
)
41 if actual
!= lock
['files']:
42 expected_map
= {e
['path']: e
for e
in lock
['files']}
43 actual_map
= {e
['path']: e
for e
in actual
}
44 changed
= sorted(k
for k
in expected_map
.keys() | actual_map
.keys()
45 if expected_map
.get(k
) != actual_map
.get(k
))
46 raise SystemExit('CORPUS-INTEGRITY-FAIL: ' + ', '.join(changed
[:20]))
51 if DESTINATION
.exists():
53 print('Pinned corpus already present and verified; no files overwritten.')
55 with tempfile
.TemporaryDirectory(prefix
='topic-maps-corpus-') as temporary
:
56 checkout
= pathlib
.Path(temporary
) / 'upstream'
57 subprocess
.run(['git', 'clone', '--quiet', lock
['repository'] + '.git', str(checkout
)], check
=True)
58 subprocess
.run(['git', 'cat-file', '-e', lock
['commit'] + '^{commit}'], cwd
=checkout
, check
=True)
59 archive
= subprocess
.check_output(['git', 'archive', '--format=tar', lock
['commit']], cwd
=checkout
)
60 if hashlib
.sha256(archive
).hexdigest() != lock
['archiveSHA256']:
61 raise SystemExit('Pinned archive hash mismatch; acquisition stopped.')
62 staging
= pathlib
.Path(temporary
) / 'corpus'
64 with tarfile
.open(fileobj
=io
.BytesIO(archive
)) as tar
:
65 for member
in tar
.getmembers():
66 parts
= pathlib
.PurePosixPath(member
.name
)
67 if parts
.is_absolute() or '..' in parts
.parts
or not (member
.isfile() or member
.isdir()):
68 raise SystemExit('Unsafe upstream archive entry: ' + member
.name
)
69 tar
.extractall(staging
)
71 DESTINATION
.parent
.mkdir(exist_ok
=True)
72 # Copy into the absent destination; never merge over existing contents.
74 shutil
.copytree(staging
, DESTINATION
)
79 parser
= argparse
.ArgumentParser(description
=__doc__
)
80 parser
.add_argument('operation', choices
=['verify', 'acquire'])
81 args
= parser
.parse_args()
82 lock
= json
.loads(LOCK
.read_text())
83 if args
.operation
== 'acquire':
86 print('CORPUS-INTEGRITY-PASS: ' + json
.dumps(result
, sort_keys
=True))
89 if __name__
== '__main__':