X-Git-Url: https://dreyeck.freedombox.rocks/gitweb/topic-maps.git/blobdiff_plain/8741d155d948c7317945eee423a977940cad995e..6a2f8446afca0915882f0d4a06cb891ccfcdc96d:/scripts/check-report.py diff --git a/scripts/check-report.py b/scripts/check-report.py index e12e967..11faf21 100644 --- a/scripts/check-report.py +++ b/scripts/check-report.py @@ -10,7 +10,8 @@ lock = json.loads((root / 'references/corpus-lock.json').read_text()) assert report['repository'] == lock['repository'] and report['commit'] == lock['commit'] assert report['qualification'] == lock['draftQualification'] assert report['counts'] == lock['ctmCounts'] -slice10 = len(sys.argv) > 2 and sys.argv[2] == '--slice-10' +slice11 = len(sys.argv) > 2 and sys.argv[2] == '--slice-11' +slice10 = slice11 or len(sys.argv) > 2 and sys.argv[2] == '--slice-10' slice9 = slice10 or len(sys.argv) > 2 and sys.argv[2] == '--slice-9' slice8 = slice9 or len(sys.argv) > 2 and sys.argv[2] == '--slice-8' slice7 = slice8 or len(sys.argv) > 2 and sys.argv[2] == '--slice-7' @@ -24,7 +25,7 @@ assert report['correspondence'] == dict(ok=True, missingInputs=[], missingBaseli cases = report['cases'] assert len(cases) == report['counts']['valid'] + report['counts']['invalid'] passes = 62 if slice10 else 59 if slice9 else 49 if slice8 else 42 if slice7 else 36 if slice6 else 29 if slice5 else 25 if slice4 else int(slice1) -assert report['statuses'] == {'pass': passes, 'fail': int(slice10), 'unsupported': 0, 'not-run': len(cases)-passes-int(slice10)} +assert report['statuses'] == {'pass': passes, 'fail': (2 if slice11 else int(slice10)), 'unsupported': 0, 'not-run': len(cases)-passes-(2 if slice11 else int(slice10))} assert [c['id'] for c in cases] == sorted(c['id'] for c in cases) assert len({c['id'] for c in cases}) == len(cases) tested = [c for c in cases if c['status'] == 'pass'] @@ -33,10 +34,10 @@ if slice4: assert len(positives) == (54 if slice10 else 51 if slice9 else 46 if slice8 else 39 if slice7 else 33 if slice6 else 28 if slice5 else 24) and all(c['actualOutcome'] == 'cxtm-match' for c in positives) failed = [c for c in cases if c['status'] == 'fail'] if slice10: - assert len(failed) == 1 and failed[0]['id'] == 'ctm/in/occurrence-datetime2.ctm' + assert len(failed) == (2 if slice11 else 1) and {c['id'] for c in failed} == ({'ctm/in/occurrence-datetime2.ctm', 'ctm/in/occurrence-datetime-timezone2.ctm'} if slice11 else {'ctm/in/occurrence-datetime2.ctm'}) assert failed[0]['actualOutcome'] == 'cxtm-mismatch' and 'normative discrepancy' in failed[0]['diagnostics'][0] offset = next(c for c in cases if c['id'] == 'ctm/in/occurrence-datetime-timezone2.ctm') - assert offset['status'] == 'not-run' + assert offset['status'] == ('fail' if slice11 else 'not-run') else: assert not failed untested = [c for c in cases if c['status'] == 'not-run']