]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/check-report.py
2 """Check the Lisp-generated JSON with an independent standard-library decoder."""
7 root
= pathlib
.Path(__file__
).resolve().parent
.parent
8 report
= json
.loads(pathlib
.Path(sys
.argv
[1]).read_text())
9 lock
= json
.loads((root
/ 'references/corpus-lock.json').read_text())
10 assert report
['repository'] == lock
['repository'] and report
['commit'] == lock
['commit']
11 assert report
['qualification'] == lock
['draftQualification']
12 assert report
['counts'] == lock
['ctmCounts']
13 slice11
= len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-11'
14 slice10
= slice11
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-10'
15 slice9
= slice10
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-9'
16 slice8
= slice9
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-8'
17 slice7
= slice8
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-7'
18 slice6
= slice7
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-6'
19 slice5
= slice6
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-5'
20 slice4
= slice5
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-4'
21 slice1
= slice4
or (len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-1')
22 assert report
['implemented'] == dict(ctmParsing
=slice1
, tmdmDeserialization
=slice1
, cxtmCanonicalization
=slice4
)
23 assert report
['countsMatchPin'] is True
24 assert report
['correspondence'] == dict(ok
=True, missingInputs
=[], missingBaselines
=[], unmatchedBaselines
=[], duplicateIDs
=[])
25 cases
= report
['cases']
26 assert len(cases
) == report
['counts']['valid'] + report
['counts']['invalid']
27 passes
= 62 if slice10
else 59 if slice9
else 49 if slice8
else 42 if slice7
else 36 if slice6
else 29 if slice5
else 25 if slice4
else int(slice1
)
28 assert report
['statuses'] == {'pass': passes
, 'fail': (2 if slice11
else int(slice10
)), 'unsupported': 0, 'not-run': len(cases
)-passes
-(2 if slice11
else int(slice10
))}
29 assert [c
['id'] for c
in cases
] == sorted(c
['id'] for c
in cases
)
30 assert len({c
['id'] for c
in cases
}) == len(cases
)
31 tested
= [c
for c
in cases
if c
['status'] == 'pass']
33 positives
= [c
for c
in tested
if c
['category'] == 'valid']
34 assert len(positives
) == (54 if slice10
else 51 if slice9
else 46 if slice8
else 39 if slice7
else 33 if slice6
else 28 if slice5
else 24) and all(c
['actualOutcome'] == 'cxtm-match' for c
in positives
)
35 failed
= [c
for c
in cases
if c
['status'] == 'fail']
37 assert len(failed
) == (2 if slice11
else 1) and {c
['id'] for c
in failed
} == ({'ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm'} if slice11
else {'ctm/in/occurrence-datetime2.ctm'})
38 assert failed
[0]['actualOutcome'] == 'cxtm-mismatch' and 'normative discrepancy' in failed
[0]['diagnostics'][0]
39 offset
= next(c
for c
in cases
if c
['id'] == 'ctm/in/occurrence-datetime-timezone2.ctm')
40 assert offset
['status'] == ('fail' if slice11
else 'not-run')
43 untested
= [c
for c
in cases
if c
['status'] == 'not-run']
44 assert all(c
['status'] == 'not-run' and c
['actualOutcome'] is None and c
['diagnostics'] == [] for c
in untested
)
46 negative
= next(c
for c
in cases
if c
['id'] == 'ctm/invalid/prefix-undefined.ctm')
47 assert negative
['status'] == 'pass' and negative
['actualOutcome'] == 'ctm-rejection'
48 assert len(negative
['diagnostics']) == 1 and 'UNDEFINED-PREFIX' in negative
['diagnostics'][0]
49 assert all(c
['documentIRI'] == 'https://ctm-tests.example.invalid/' + lock
['commit'] + '/' + c
['id'] for c
in cases
)
50 assert sum(c
['category'] == 'valid' for c
in cases
) == report
['counts']['valid']
51 assert sum(c
['category'] == 'invalid' for c
in cases
) == report
['counts']['invalid']
52 assert all(c
['expectedOutcome'] == 'ctm-rejection' and c
['expectedCXTM'] is None for c
in cases
if c
['category'] == 'invalid')
53 assert all(c
['expectedOutcome'] == 'cxtm-equal' and c
['expectedCXTM'].endswith('.ctm.cxtm') for c
in cases
if c
['category'] == 'valid')
54 assert all((root
/'vendor/cxtm'/c
['sourcePathname']).is_file() for c
in cases
)
55 assert all((root
/'vendor/cxtm'/c
['expectedCXTM']).is_file() for c
in cases
if c
['expectedCXTM'])
56 assert len(report
['auxiliaryResources']) == report
['counts']['auxiliary']
57 assert len(report
['baselines']) == report
['counts']['baselines']
58 assert len(report
['infrastructureChecks']) == len(set(report
['infrastructureChecks'])) > 0
59 print('REPORT-JSON-PASS: deterministic schema, draft qualification, case contracts and only warranted imported status changes')
62 for name
, reason
in [('prefix-collision', 'PREFIX-COLLISION'), ('prefix-undefined2', 'UNDEFINED-PREFIX')]:
63 c
= next(c
for c
in cases
if c
['id'] == f
'ctm/invalid/{name}.ctm')
64 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
65 assert len(c
['diagnostics']) == 1 and reason
in c
['diagnostics'][0]
68 for name
, reason
in [('date-invalid-day', 'INVALID-DATE-DAY'), ('date-invalid-month', 'INVALID-DATE-MONTH'),
69 ('date-invalid-year', 'INVALID-DATE-YEAR'), ('date-invalid-year2', 'INVALID-DATE-YEAR'), ('date-invalid-year3', 'INVALID-DATE-YEAR')]:
70 c
= next(c
for c
in cases
if c
['id'] == f
'ctm/invalid/{name}.ctm')
71 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
72 assert len(c
['diagnostics']) == 1 and reason
in c
['diagnostics'][0]