]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/check-report.py
test(topic-maps): establish pinned CTM test foundation
[topic-maps.git] / scripts / check-report.py
1 #!/usr/bin/env python3
2 """Check the Lisp-generated JSON with an independent standard-library decoder."""
3 import json
4 import pathlib
5 import sys
6
7 root = pathlib.Path(__file__).resolve().parent.parent
8 report = json.loads(pathlib.Path(sys.argv[1]).read_text())
9 lock = json.loads((root / 'references/corpus-lock.json').read_text())
10 assert report['repository'] == lock['repository'] and report['commit'] == lock['commit']
11 assert report['qualification'] == lock['draftQualification']
12 assert report['counts'] == lock['ctmCounts']
13 assert report['implemented'] == dict(ctmParsing=False, tmdmDeserialization=False, cxtmCanonicalization=False)
14 assert report['countsMatchPin'] is True
15 assert report['correspondence'] == dict(ok=True, missingInputs=[], missingBaselines=[], unmatchedBaselines=[], duplicateIDs=[])
16 cases = report['cases']
17 assert len(cases) == report['counts']['valid'] + report['counts']['invalid']
18 assert report['statuses'] == {'pass': 0, 'fail': 0, 'unsupported': 0, 'not-run': len(cases)}
19 assert [c['id'] for c in cases] == sorted(c['id'] for c in cases)
20 assert len({c['id'] for c in cases}) == len(cases)
21 assert all(c['status'] == 'not-run' and c['actualOutcome'] is None and c['diagnostics'] == [] for c in cases)
22 assert all(c['documentIRI'] == 'https://ctm-tests.example.invalid/' + lock['commit'] + '/' + c['id'] for c in cases)
23 assert sum(c['category'] == 'valid' for c in cases) == report['counts']['valid']
24 assert sum(c['category'] == 'invalid' for c in cases) == report['counts']['invalid']
25 assert all(c['expectedOutcome'] == 'ctm-rejection' and c['expectedCXTM'] is None for c in cases if c['category'] == 'invalid')
26 assert all(c['expectedOutcome'] == 'cxtm-equal' and c['expectedCXTM'].endswith('.ctm.cxtm') for c in cases if c['category'] == 'valid')
27 assert all((root/'vendor/cxtm'/c['sourcePathname']).is_file() for c in cases)
28 assert all((root/'vendor/cxtm'/c['expectedCXTM']).is_file() for c in cases if c['expectedCXTM'])
29 assert len(report['auxiliaryResources']) == report['counts']['auxiliary']
30 assert len(report['baselines']) == report['counts']['baselines']
31 assert len(report['infrastructureChecks']) == len(set(report['infrastructureChecks'])) > 0
32 print('REPORT-JSON-PASS: deterministic schema, draft qualification, case contracts and zero CTM passes')