]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/check-report.py
Support DateTime occurrences and expose normative baseline conflicts
[topic-maps.git] / scripts / check-report.py
1 #!/usr/bin/env python3
2 """Check the Lisp-generated JSON with an independent standard-library decoder."""
3 import json
4 import pathlib
5 import sys
6
7 root = pathlib.Path(__file__).resolve().parent.parent
8 report = json.loads(pathlib.Path(sys.argv[1]).read_text())
9 lock = json.loads((root / 'references/corpus-lock.json').read_text())
10 assert report['repository'] == lock['repository'] and report['commit'] == lock['commit']
11 assert report['qualification'] == lock['draftQualification']
12 assert report['counts'] == lock['ctmCounts']
13 slice10 = len(sys.argv) > 2 and sys.argv[2] == '--slice-10'
14 slice9 = slice10 or len(sys.argv) > 2 and sys.argv[2] == '--slice-9'
15 slice8 = slice9 or len(sys.argv) > 2 and sys.argv[2] == '--slice-8'
16 slice7 = slice8 or len(sys.argv) > 2 and sys.argv[2] == '--slice-7'
17 slice6 = slice7 or len(sys.argv) > 2 and sys.argv[2] == '--slice-6'
18 slice5 = slice6 or len(sys.argv) > 2 and sys.argv[2] == '--slice-5'
19 slice4 = slice5 or len(sys.argv) > 2 and sys.argv[2] == '--slice-4'
20 slice1 = slice4 or (len(sys.argv) > 2 and sys.argv[2] == '--slice-1')
21 assert report['implemented'] == dict(ctmParsing=slice1, tmdmDeserialization=slice1, cxtmCanonicalization=slice4)
22 assert report['countsMatchPin'] is True
23 assert report['correspondence'] == dict(ok=True, missingInputs=[], missingBaselines=[], unmatchedBaselines=[], duplicateIDs=[])
24 cases = report['cases']
25 assert len(cases) == report['counts']['valid'] + report['counts']['invalid']
26 passes = 62 if slice10 else 59 if slice9 else 49 if slice8 else 42 if slice7 else 36 if slice6 else 29 if slice5 else 25 if slice4 else int(slice1)
27 assert report['statuses'] == {'pass': passes, 'fail': int(slice10), 'unsupported': 0, 'not-run': len(cases)-passes-int(slice10)}
28 assert [c['id'] for c in cases] == sorted(c['id'] for c in cases)
29 assert len({c['id'] for c in cases}) == len(cases)
30 tested = [c for c in cases if c['status'] == 'pass']
31 if slice4:
32 positives = [c for c in tested if c['category'] == 'valid']
33 assert len(positives) == (54 if slice10 else 51 if slice9 else 46 if slice8 else 39 if slice7 else 33 if slice6 else 28 if slice5 else 24) and all(c['actualOutcome'] == 'cxtm-match' for c in positives)
34 failed = [c for c in cases if c['status'] == 'fail']
35 if slice10:
36 assert len(failed) == 1 and failed[0]['id'] == 'ctm/in/occurrence-datetime2.ctm'
37 assert failed[0]['actualOutcome'] == 'cxtm-mismatch' and 'normative discrepancy' in failed[0]['diagnostics'][0]
38 offset = next(c for c in cases if c['id'] == 'ctm/in/occurrence-datetime-timezone2.ctm')
39 assert offset['status'] == 'not-run'
40 else:
41 assert not failed
42 untested = [c for c in cases if c['status'] == 'not-run']
43 assert all(c['status'] == 'not-run' and c['actualOutcome'] is None and c['diagnostics'] == [] for c in untested)
44 if slice1:
45 negative = next(c for c in cases if c['id'] == 'ctm/invalid/prefix-undefined.ctm')
46 assert negative['status'] == 'pass' and negative['actualOutcome'] == 'ctm-rejection'
47 assert len(negative['diagnostics']) == 1 and 'UNDEFINED-PREFIX' in negative['diagnostics'][0]
48 assert all(c['documentIRI'] == 'https://ctm-tests.example.invalid/' + lock['commit'] + '/' + c['id'] for c in cases)
49 assert sum(c['category'] == 'valid' for c in cases) == report['counts']['valid']
50 assert sum(c['category'] == 'invalid' for c in cases) == report['counts']['invalid']
51 assert all(c['expectedOutcome'] == 'ctm-rejection' and c['expectedCXTM'] is None for c in cases if c['category'] == 'invalid')
52 assert all(c['expectedOutcome'] == 'cxtm-equal' and c['expectedCXTM'].endswith('.ctm.cxtm') for c in cases if c['category'] == 'valid')
53 assert all((root/'vendor/cxtm'/c['sourcePathname']).is_file() for c in cases)
54 assert all((root/'vendor/cxtm'/c['expectedCXTM']).is_file() for c in cases if c['expectedCXTM'])
55 assert len(report['auxiliaryResources']) == report['counts']['auxiliary']
56 assert len(report['baselines']) == report['counts']['baselines']
57 assert len(report['infrastructureChecks']) == len(set(report['infrastructureChecks'])) > 0
58 print('REPORT-JSON-PASS: deterministic schema, draft qualification, case contracts and only warranted imported status changes')
59
60 if slice6:
61 for name, reason in [('prefix-collision', 'PREFIX-COLLISION'), ('prefix-undefined2', 'UNDEFINED-PREFIX')]:
62 c = next(c for c in cases if c['id'] == f'ctm/invalid/{name}.ctm')
63 assert c['status'] == 'pass' and c['actualOutcome'] == 'ctm-rejection'
64 assert len(c['diagnostics']) == 1 and reason in c['diagnostics'][0]
65
66 if slice9:
67 for name, reason in [('date-invalid-day', 'INVALID-DATE-DAY'), ('date-invalid-month', 'INVALID-DATE-MONTH'),
68 ('date-invalid-year', 'INVALID-DATE-YEAR'), ('date-invalid-year2', 'INVALID-DATE-YEAR'), ('date-invalid-year3', 'INVALID-DATE-YEAR')]:
69 c = next(c for c in cases if c['id'] == f'ctm/invalid/{name}.ctm')
70 assert c['status'] == 'pass' and c['actualOutcome'] == 'ctm-rejection'
71 assert len(c['diagnostics']) == 1 and reason in c['diagnostics'][0]