]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/check-report.py
2 """Check the Lisp-generated JSON with an independent standard-library decoder."""
7 root
= pathlib
.Path(__file__
).resolve().parent
.parent
8 report
= json
.loads(pathlib
.Path(sys
.argv
[1]).read_text())
9 lock
= json
.loads((root
/ 'references/corpus-lock.json').read_text())
10 assert report
['repository'] == lock
['repository'] and report
['commit'] == lock
['commit']
11 assert report
['qualification'] == lock
['draftQualification']
12 assert report
['counts'] == lock
['ctmCounts']
13 slice19b
= len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-19b'
14 slice18
= slice19b
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-18'
15 slice17b
= slice18
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-17b'
16 slice17a
= slice17b
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-17a'
17 slice16b
= slice17a
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-16b'
18 slice15
= slice16b
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-15'
19 slice14
= slice15
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-14'
20 slice13
= slice14
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-13'
21 slice12a
= slice13
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-12a'
22 slice11
= slice12a
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-11'
23 slice10
= slice11
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-10'
24 slice9
= slice10
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-9'
25 slice8
= slice9
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-8'
26 slice7
= slice8
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-7'
27 slice6
= slice7
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-6'
28 slice5
= slice6
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-5'
29 slice4
= slice5
or len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-4'
30 slice1
= slice4
or (len(sys
.argv
) > 2 and sys
.argv
[2] == '--slice-1')
31 assert report
['implemented'] == dict(ctmParsing
=slice1
, tmdmDeserialization
=slice1
, cxtmCanonicalization
=slice4
)
32 assert report
['countsMatchPin'] is True
33 assert report
['correspondence'] == dict(ok
=True, missingInputs
=[], missingBaselines
=[], unmatchedBaselines
=[], duplicateIDs
=[])
34 cases
= report
['cases']
35 assert len(cases
) == report
['counts']['valid'] + report
['counts']['invalid']
36 passes
= 194 if slice19b
else 187 if slice18
else 120 if slice17b
else 107 if slice17a
else 106 if slice16b
else 93 if slice15
else 71 if slice14
else 67 if slice13
else 64 if slice12a
else 62 if slice10
else 59 if slice9
else 49 if slice8
else 42 if slice7
else 36 if slice6
else 29 if slice5
else 25 if slice4
else int(slice1
)
37 assert report
['statuses'] == {'pass': passes
, 'fail': (2 if slice11
else int(slice10
)), 'unsupported': 0, 'not-run': len(cases
)-passes
-(2 if slice11
else int(slice10
))}
38 assert [c
['id'] for c
in cases
] == sorted(c
['id'] for c
in cases
)
39 assert len({c
['id'] for c
in cases
}) == len(cases
)
40 tested
= [c
for c
in cases
if c
['status'] == 'pass']
42 positives
= [c
for c
in tested
if c
['category'] == 'valid']
43 assert len(positives
) == (169 if slice19b
else 165 if slice18
else 108 if slice17b
else 95 if slice17a
else 94 if slice16b
else 83 if slice15
else 62 if slice14
else 59 if slice13
else 56 if slice12a
else 54 if slice10
else 51 if slice9
else 46 if slice8
else 39 if slice7
else 33 if slice6
else 28 if slice5
else 24) and all(c
['actualOutcome'] == 'cxtm-match' for c
in positives
)
44 failed
= [c
for c
in cases
if c
['status'] == 'fail']
46 assert len(failed
) == (2 if slice11
else 1) and {c
['id'] for c
in failed
} == ({'ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm'} if slice11
else {'ctm/in/occurrence-datetime2.ctm'})
47 assert failed
[0]['actualOutcome'] == 'cxtm-mismatch' and 'normative discrepancy' in failed
[0]['diagnostics'][0]
48 offset
= next(c
for c
in cases
if c
['id'] == 'ctm/in/occurrence-datetime-timezone2.ctm')
49 assert offset
['status'] == ('fail' if slice11
else 'not-run')
52 untested
= [c
for c
in cases
if c
['status'] == 'not-run']
53 assert all(c
['status'] == 'not-run' and c
['actualOutcome'] is None and c
['diagnostics'] == [] for c
in untested
)
55 negative
= next(c
for c
in cases
if c
['id'] == 'ctm/invalid/prefix-undefined.ctm')
56 assert negative
['status'] == 'pass' and negative
['actualOutcome'] == 'ctm-rejection'
57 assert len(negative
['diagnostics']) == 1 and 'UNDEFINED-PREFIX' in negative
['diagnostics'][0]
58 assert all(c
['documentIRI'] == 'https://ctm-tests.example.invalid/' + lock
['commit'] + '/' + c
['id'] for c
in cases
)
59 assert sum(c
['category'] == 'valid' for c
in cases
) == report
['counts']['valid']
60 assert sum(c
['category'] == 'invalid' for c
in cases
) == report
['counts']['invalid']
61 assert all(c
['expectedOutcome'] == 'ctm-rejection' and c
['expectedCXTM'] is None for c
in cases
if c
['category'] == 'invalid')
62 assert all(c
['expectedOutcome'] == 'cxtm-equal' and c
['expectedCXTM'].endswith('.ctm.cxtm') for c
in cases
if c
['category'] == 'valid')
63 assert all((root
/'vendor/cxtm'/c
['sourcePathname']).is_file() for c
in cases
)
64 assert all((root
/'vendor/cxtm'/c
['expectedCXTM']).is_file() for c
in cases
if c
['expectedCXTM'])
65 assert len(report
['auxiliaryResources']) == report
['counts']['auxiliary']
66 assert len(report
['baselines']) == report
['counts']['baselines']
67 assert len(report
['infrastructureChecks']) == len(set(report
['infrastructureChecks'])) > 0
68 print('REPORT-JSON-PASS: deterministic schema, draft qualification, case contracts and only warranted imported status changes')
71 for name
, reason
in [('prefix-collision', 'PREFIX-COLLISION'), ('prefix-undefined2', 'UNDEFINED-PREFIX')]:
72 c
= next(c
for c
in cases
if c
['id'] == f
'ctm/invalid/{name}.ctm')
73 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
74 assert len(c
['diagnostics']) == 1 and reason
in c
['diagnostics'][0]
77 for name
, reason
in [('date-invalid-day', 'INVALID-DATE-DAY'), ('date-invalid-month', 'INVALID-DATE-MONTH'),
78 ('date-invalid-year', 'INVALID-DATE-YEAR'), ('date-invalid-year2', 'INVALID-DATE-YEAR'), ('date-invalid-year3', 'INVALID-DATE-YEAR')]:
79 c
= next(c
for c
in cases
if c
['id'] == f
'ctm/invalid/{name}.ctm')
80 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
81 assert len(c
['diagnostics']) == 1 and reason
in c
['diagnostics'][0]
84 c
= next(c
for c
in cases
if c
['id'] == 'ctm/invalid/variant-no-scope.ctm')
85 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
86 assert len(c
['diagnostics']) == 1 and 'EXPECTED-VARIANT-SCOPE' in c
['diagnostics'][0]
89 c
= next(c
for c
in cases
if c
['id'] == 'ctm/invalid/embedded-topic-global-scope.ctm')
90 assert c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection'
91 assert len(c
['diagnostics']) == 1 and 'EMBEDDED-TOPIC-AT-TOP-LEVEL' in c
['diagnostics'][0]
92 c
= next(c
for c
in cases
if c
['id'] == 'ctm/invalid/embedded-topic-template.ctm')
93 assert (c
['status'] == 'pass' and c
['actualOutcome'] == 'ctm-rejection') if slice18
else (c
['status'] == 'not-run' and c
['actualOutcome'] is None)