]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/check-report.py
Support explicitly typed Topic Names with merge and CXTM evidence
[topic-maps.git] / scripts / check-report.py
1 #!/usr/bin/env python3
2 """Check the Lisp-generated JSON with an independent standard-library decoder."""
3 import json
4 import pathlib
5 import sys
6
7 root = pathlib.Path(__file__).resolve().parent.parent
8 report = json.loads(pathlib.Path(sys.argv[1]).read_text())
9 lock = json.loads((root / 'references/corpus-lock.json').read_text())
10 assert report['repository'] == lock['repository'] and report['commit'] == lock['commit']
11 assert report['qualification'] == lock['draftQualification']
12 assert report['counts'] == lock['ctmCounts']
13 slice12a = len(sys.argv) > 2 and sys.argv[2] == '--slice-12a'
14 slice11 = slice12a or len(sys.argv) > 2 and sys.argv[2] == '--slice-11'
15 slice10 = slice11 or len(sys.argv) > 2 and sys.argv[2] == '--slice-10'
16 slice9 = slice10 or len(sys.argv) > 2 and sys.argv[2] == '--slice-9'
17 slice8 = slice9 or len(sys.argv) > 2 and sys.argv[2] == '--slice-8'
18 slice7 = slice8 or len(sys.argv) > 2 and sys.argv[2] == '--slice-7'
19 slice6 = slice7 or len(sys.argv) > 2 and sys.argv[2] == '--slice-6'
20 slice5 = slice6 or len(sys.argv) > 2 and sys.argv[2] == '--slice-5'
21 slice4 = slice5 or len(sys.argv) > 2 and sys.argv[2] == '--slice-4'
22 slice1 = slice4 or (len(sys.argv) > 2 and sys.argv[2] == '--slice-1')
23 assert report['implemented'] == dict(ctmParsing=slice1, tmdmDeserialization=slice1, cxtmCanonicalization=slice4)
24 assert report['countsMatchPin'] is True
25 assert report['correspondence'] == dict(ok=True, missingInputs=[], missingBaselines=[], unmatchedBaselines=[], duplicateIDs=[])
26 cases = report['cases']
27 assert len(cases) == report['counts']['valid'] + report['counts']['invalid']
28 passes = 64 if slice12a else 62 if slice10 else 59 if slice9 else 49 if slice8 else 42 if slice7 else 36 if slice6 else 29 if slice5 else 25 if slice4 else int(slice1)
29 assert report['statuses'] == {'pass': passes, 'fail': (2 if slice11 else int(slice10)), 'unsupported': 0, 'not-run': len(cases)-passes-(2 if slice11 else int(slice10))}
30 assert [c['id'] for c in cases] == sorted(c['id'] for c in cases)
31 assert len({c['id'] for c in cases}) == len(cases)
32 tested = [c for c in cases if c['status'] == 'pass']
33 if slice4:
34 positives = [c for c in tested if c['category'] == 'valid']
35 assert len(positives) == (56 if slice12a else 54 if slice10 else 51 if slice9 else 46 if slice8 else 39 if slice7 else 33 if slice6 else 28 if slice5 else 24) and all(c['actualOutcome'] == 'cxtm-match' for c in positives)
36 failed = [c for c in cases if c['status'] == 'fail']
37 if slice10:
38 assert len(failed) == (2 if slice11 else 1) and {c['id'] for c in failed} == ({'ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm'} if slice11 else {'ctm/in/occurrence-datetime2.ctm'})
39 assert failed[0]['actualOutcome'] == 'cxtm-mismatch' and 'normative discrepancy' in failed[0]['diagnostics'][0]
40 offset = next(c for c in cases if c['id'] == 'ctm/in/occurrence-datetime-timezone2.ctm')
41 assert offset['status'] == ('fail' if slice11 else 'not-run')
42 else:
43 assert not failed
44 untested = [c for c in cases if c['status'] == 'not-run']
45 assert all(c['status'] == 'not-run' and c['actualOutcome'] is None and c['diagnostics'] == [] for c in untested)
46 if slice1:
47 negative = next(c for c in cases if c['id'] == 'ctm/invalid/prefix-undefined.ctm')
48 assert negative['status'] == 'pass' and negative['actualOutcome'] == 'ctm-rejection'
49 assert len(negative['diagnostics']) == 1 and 'UNDEFINED-PREFIX' in negative['diagnostics'][0]
50 assert all(c['documentIRI'] == 'https://ctm-tests.example.invalid/' + lock['commit'] + '/' + c['id'] for c in cases)
51 assert sum(c['category'] == 'valid' for c in cases) == report['counts']['valid']
52 assert sum(c['category'] == 'invalid' for c in cases) == report['counts']['invalid']
53 assert all(c['expectedOutcome'] == 'ctm-rejection' and c['expectedCXTM'] is None for c in cases if c['category'] == 'invalid')
54 assert all(c['expectedOutcome'] == 'cxtm-equal' and c['expectedCXTM'].endswith('.ctm.cxtm') for c in cases if c['category'] == 'valid')
55 assert all((root/'vendor/cxtm'/c['sourcePathname']).is_file() for c in cases)
56 assert all((root/'vendor/cxtm'/c['expectedCXTM']).is_file() for c in cases if c['expectedCXTM'])
57 assert len(report['auxiliaryResources']) == report['counts']['auxiliary']
58 assert len(report['baselines']) == report['counts']['baselines']
59 assert len(report['infrastructureChecks']) == len(set(report['infrastructureChecks'])) > 0
60 print('REPORT-JSON-PASS: deterministic schema, draft qualification, case contracts and only warranted imported status changes')
61
62 if slice6:
63 for name, reason in [('prefix-collision', 'PREFIX-COLLISION'), ('prefix-undefined2', 'UNDEFINED-PREFIX')]:
64 c = next(c for c in cases if c['id'] == f'ctm/invalid/{name}.ctm')
65 assert c['status'] == 'pass' and c['actualOutcome'] == 'ctm-rejection'
66 assert len(c['diagnostics']) == 1 and reason in c['diagnostics'][0]
67
68 if slice9:
69 for name, reason in [('date-invalid-day', 'INVALID-DATE-DAY'), ('date-invalid-month', 'INVALID-DATE-MONTH'),
70 ('date-invalid-year', 'INVALID-DATE-YEAR'), ('date-invalid-year2', 'INVALID-DATE-YEAR'), ('date-invalid-year3', 'INVALID-DATE-YEAR')]:
71 c = next(c for c in cases if c['id'] == f'ctm/invalid/{name}.ctm')
72 assert c['status'] == 'pass' and c['actualOutcome'] == 'ctm-rejection'
73 assert len(c['diagnostics']) == 1 and reason in c['diagnostics'][0]