]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22e.py
2 """Run/audit Slice 22E through the unchanged 22D offline executor and ASDF tests.
3 --output must be outside the repository. --audit-only verifies saved executions,
4 without pretending they are fresh runs. Local pinned dependencies only.
11 from pathlib
import Path
15 sys
.dont_write_bytecode
=True
16 ROOT
=Path(__file__
).resolve().parents
[1]
17 TOOLS
=ROOT
/'reports/slice-22e'
18 BASE
='791969ca59922d61e967198eaee4c1fe172b5a0d'
19 PRIOR
=ROOT
/'reports/slice-22d/after'
20 ALLOWED
={'src/xtm/package.lisp','src/xtm/reader.lisp','tests/xtm-identity.lisp','tests/xtm-map-reification.lisp','tests/ctm-xtm-mergemap.lisp','topic-maps.asd'}
21 NEW
={'tests/xtm-mergemap.lisp','scripts/validate-slice-22e.py'}
22 TARGET
='xtm2/in/mergemap-itemid.xtm'
24 def read(p
):return json
.loads(p
.read_text())
25 def save(p
,v
):p
.write_text(json
.dumps(v
,indent
=2,ensure_ascii
=False)+'\n')
26 def sha(p
):return hashlib
.sha256(p
.read_bytes()).hexdigest()
27 def git(*a
,cwd
=ROOT
):return subprocess
.check_output(['git',*a
],cwd
=cwd
,text
=True).strip()
28 def now():return datetime
.datetime
.now(datetime
.timezone
.utc
).isoformat()
29 def previous_validator():
30 spec
=importlib
.util
.spec_from_file_location('slice22d',ROOT
/'scripts/validate-slice-22d.py')
31 v
=importlib
.util
.module_from_spec(spec
);spec
.loader
.exec_module(v
)
34 def audit(out
,baseline
,initial
):
35 v
=previous_validator();start
=read(initial
)
36 assert start
['baseCommit']==BASE
and start
['indexClean'] and start
['worktreeClean']
37 assert set(git('diff','--name-only',BASE
).splitlines())&set(start
['trackedFileSHA256'])<=ALLOWED
38 protected
={p
:h
for p
,h
in start
['trackedFileSHA256'].items() if p
not in ALLOWED
}
39 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
40 runs
=read(out
/'runs.json');assert len(runs
['runs'])==119
41 assert all(r
['exitCode']==r
['expectedExitCode'] and sha(out
/(r
['label']+'.log'))==r
['logSHA256'] for r
in runs
['runs'])
42 assert runs
['executionSourcesSHA256']=={p
:sha(ROOT
/p
) for p
in runs
['executionSourcesSHA256']}
43 assert runs
['rawReportSHA256']=={p
:sha(out
/p
) for p
in runs
['rawReportSHA256']}
44 before
=read(baseline
/'xtm.json');xtm
=read(out
/'xtm.json')
45 assert before
==read(PRIOR
/'xtm.json') and len(before
['checks'])==2522
46 assert read(baseline
/'diagnostics.json')==read(PRIOR
/'after-diagnostics.json')
47 assert xtm
['cases']==before
['cases'] and len(xtm
['cases'])==72
48 assert xtm
['statuses']==before
['statuses']=={'pass':72,'fail':0,'unsupported':0,'not-run':0}
49 assert xtm
['otherCorpusInputs']==before
['otherCorpusInputs']
50 labels
=re
.findall(r
'^XTM-CHECK-PASS (.+)$',(out
/'xtm.log').read_text(),re
.M
)
51 assert labels
==xtm
['checks'] and len(labels
)==len(set(labels
))
52 assert [x
for x
in labels
if x
in set(before
['checks'])]==before
['checks']
53 assert all(x
.startswith(('mm:','mm-error:','mm-original:')) for x
in set(labels
)-set(before
['checks']))
55 for k
,value
in before
.items():
56 if k
=='checks':continue
57 if k
=='mapReifierControls':
58 assert len(value
)==len(xtm
[k
])
59 for i
,(a
,b
) in enumerate(zip(value
,xtm
[k
])):
61 assert a
['label']==b
['label']=='mergeMap'
62 assert a
['condition']['class']==b
['condition']['class']=='XTM-UNSUPPORTED'
63 assert a
['condition']['code']=='unsupported-construct' and b
['condition']['code']=='resource-resolver-required'
64 assert all(d
['path'] in {'/condition/code','/condition/detail','/condition/message','/expectedCode'} for d
in v
.changes(a
,b
))
65 retargets
.append({'field':k
,'index':i
,'before':a
,'after':b
})
66 else:assert xtm
[k
]==value
,k
67 save(out
/'retargeted-observations.json',retargets
)
68 olddiag
=read(baseline
/'diagnostics.json');newdiag
=read(out
/'after-diagnostics.json')
69 assert len(olddiag
)==len(newdiag
)==124
70 changes
=[{'id':a
['id'],'before':a
,'after':b
} for a
,b
in zip(olddiag
,newdiag
) if a
!=b
]
71 merge_ids
={x
['id'] for x
in olddiag
if x
.get('condition') and x
['condition'].get('detail')=='XTM mergeMap is not implemented in this subset.'}
72 assert len(merge_ids
)==7 and {x
['id'] for x
in changes
}==merge_ids
73 assert all(b
'<mergeMap ' in (ROOT
/'vendor/cxtm'/id).read_bytes() for id in merge_ids
)
75 assert x
['before']['condition']['code']=='unsupported-construct'
76 assert x
['after']['condition']['class']=='XTM-UNSUPPORTED' and x
['after']['condition']['code']=='resource-resolver-required'
77 assert not x
['after']['byteExact']
78 save(out
/'diagnostic-changes.json',{'kind':'all-124-originals-without-resolver','changed':7,'unchanged':117,'changes':changes
,
79 'separateExplicitResolverObservations':'original-observations.json','registeredStatusesChanged':False})
81 for r
in runs
['runs']:
83 if '-control-' not in label
:continue
84 a
=read(PRIOR
/(label
+'.json'));b
=read(out
/(label
+'.json'))
85 assert a
['cases']==b
['cases'] and a
['statuses']==b
['statuses'],label
86 ca
=dict(v
.condition_nodes(a
));cb
=dict(v
.condition_nodes(b
));assert ca
.keys()==cb
.keys(),label
88 for path
,c
in ca
.items():
89 d
=cb
[path
];assert c
['class']==d
['class'],(label
,path
,c
,d
)
90 if c
.get('code')!=d
.get('code'):
91 assert path
.startswith('/mapReifierControls/') and path
.endswith('/condition')
92 assert c
.get('code')=='unsupported-construct' and d
.get('code')=='resource-resolver-required'
93 if c
!=d
:ds
.append({'path':path
,'before':c
,'after':d
})
94 assert all(x
['status']!='pass' for x
in b
['cases'] if x
['status'] in ('fail','unsupported'))
95 faults
.append({'label':label
,'expectedASDFFailureObserved':True,'caseRecordsAndOutcomesIdentical':True,
96 'previousCorrectPASSRetained':sum(x
['status']=='pass' for x
in b
['cases']),
97 'conditionClassesUnchanged':True,'conditionChanges':ds
,'faultCountedAsPass':False})
98 assert len(faults
)==113
99 save(out
/'failure-controls.json',{'retained':113,'new':0,'controls':faults
,'allExpectedFailureExitsObserved':True})
100 unchanged
=[];adapter
=[]
101 for p
in [*sorted((PRIOR
/'retained').glob('*.json')),*[PRIOR
/f
'ctm.{s}.json' for s
in ['validation','cxtm','encoding']]]:
102 rel
=p
.relative_to(PRIOR
);q
=out
/rel
103 if p
.read_bytes()!=q
.read_bytes():
104 assert str(rel
)=='retained/xtm-mergemap.integration.json'
105 adapter
=v
.changes(read(p
),read(q
));assert len(adapter
)==1,adapter
106 assert adapter
[0]['before']=='UNSUPPORTED-CONSTRUCT' and adapter
[0]['after']=='RESOURCE-RESOLVER-REQUIRED',adapter
107 else:unchanged
.append({'report':str(rel
),'sha256':sha(p
),'byteIdentical':True})
108 assert len(unchanged
)==47
109 save(out
/'retained-reports.json',unchanged
);save(out
/'adapter-retargeting.json',{'differences':adapter
,'adapterProductionCodeChanged':False})
110 regression
=(out
/'regression.log').read_text();prior
=read(PRIOR
/'execution.json')
111 for marker
,count
in dict(prior
['retainedChecks'],**prior
['stageChecks']).items():
112 assert int(re
.search(r
'(?m)^'+marker
+r
'-PASS: (\d+)',regression
).group(1))==count
113 assert 'Ran 2 tests' in regression
and 'NFC-ORACLE-PASS: 16294' in regression
114 xml
=(out
/'xml.log').read_text();assert 'CXML-SUMMARY 146 PASS 0 FAIL' in xml
115 assert len(set(re
.findall(r
'^CXML-CHECK-PASS (.+)$',xml
,re
.M
)))==146
116 assert read(out
/'ctm.validation.json')['statuses']==read(PRIOR
/'ctm.validation.json')['statuses']
117 originals
=xtm
['mergemapOriginalObservations'];assert len(originals
)==2
118 item
=originals
[0];assert item
['id']==TARGET
and item
['condition'] is None and item
['registeredStatus']=='not-run' and not item
['registeredAsPASS']
119 raw
=ROOT
/'vendor/cxtm'/TARGET
;child
=ROOT
/'vendor/cxtm/xtm2/in/mergemap-itemid.sub';expected
=ROOT
/'vendor/cxtm/xtm2/baseline/mergemap-itemid.xtm.cxtm'
120 assert raw
.read_bytes()==bytes
.fromhex(item
['sourceHex']) and child
.read_bytes()==bytes
.fromhex(item
['childHex'])
121 actual
=item
['actualCXTM'].encode();assert expected
.read_bytes()==item
['expectedCXTM'].encode() and actual
!=expected
.read_bytes()
122 mismatch
=next(i
for i
,(a
,b
) in enumerate(zip(actual
,expected
.read_bytes())) if a
!=b
)
123 assert mismatch
==item
['firstDifferentByte'] and item
['onlyBaselineRootMapItemIdentifiersDiffer']
124 assert item
['targetMapItemIdentifiers']==[] and item
['sourceMapItemIdentifiers']==[item
['childIRI']+'#the-topic-map']
125 item
=dict(item
,sourceSHA256
=sha(raw
),childSHA256
=sha(child
),baselineSHA256
=sha(expected
),actualSHA256
=hashlib
.sha256(actual
).hexdigest())
126 save(out
/'original-observations.json',[item
,originals
[1]])
127 (out
/'actual').mkdir(exist_ok
=True);(out
/'actual/mergemap-itemid.xtm.cxtm').write_bytes(actual
)
128 assert originals
[1]['condition']['code']=='cyclic-mergemap' and originals
[1]['registeredStatus']=='not-run'
129 lock
=read(ROOT
/'references/corpus-lock.json')
130 for e
in lock
['files']:
131 p
=ROOT
/'vendor/cxtm'/e
['path'];assert p
.stat().st_size
==e
['size'] and sha(p
)==e
['sha256']
132 assert protected
=={p
:sha(ROOT
/p
) for p
in protected
}
133 hd
=Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch');assert git('rev-parse','HEAD',cwd
=hd
)==start
['hyperdocHEAD'] and git('status','--porcelain=v1',cwd
=hd
)==start
['hyperdocStatus']
134 save(out
/'integrity.json',{'baseCommit':BASE
,'protectedBaseFilesSHA256':protected
,'executionSourcesSHA256':runs
['executionSourcesSHA256'],
135 'pinnedCorpusFilesVerified':len(lock
['files']),'historicalReportFilesVerified':sum(x
.startswith('reports/') for x
in protected
),
136 'historicalReportsOriginalsBaselinesCorpusLockUnchanged':True,'hyperdocUnchanged':True,'productionChanges':['src/xtm/package.lisp','src/xtm/reader.lisp']})
137 save(out
/'execution.json',{'kind':'slice-22e-noncyclic-xtm-mergemap','baseCommit':BASE
,'branch':git('branch','--show-current'),
138 'runtime':'SBCL 2.5.4','startedAtUTC':runs
['startedAtUTC'],'finishedAtUTC':runs
['finishedAtUTC'],'auditedAtUTC':now(),
139 'runs':runs
['runs'],'collector':'scripts/validate-slice-22d.py::collect (unchanged)','collectorSHA256':sha(ROOT
/'scripts/validate-slice-22d.py'),
140 'auditToolSHA256':sha(Path(__file__
)),'standaloneXTMChecks':len(labels
),'retainedXTMChecks':2522,'newXTMChecks':len(labels
)-2522,
141 'registeredXTMOriginals':72,'newPositiveOriginals':0,'originalObservationsNotRegistrations':2,
142 'retainedLispChecks':12100,'retainedChecks':prior
['retainedChecks'],'stageChecks':prior
['stageChecks'],'xmlQualificationChecks':146,
143 'faultControls':113,'newFaultControls':0,'originalDiagnoses':124,'diagnosticChangesWithinSlice':7,'diagnosesOutsideScopeUnchanged':117,
144 'statusesBefore':prior
['statusesAfter'],'statusesAfter':prior
['statusesAfter'],'pythonUnitTests':2,'nfcOracleComparisons':16294,
145 'nonXTMReportsByteIdentical':47,'adapterControlCodeRetargetings':1,'pinnedCorpusFilesVerified':len(lock
['files']),
146 'historicalReportFilesVerified':sum(x
.startswith('reports/') for x
in protected
),'knownCTMFailuresUnchanged':prior
['knownCTMFailuresUnchanged'],
147 'mapItemMetadataImported':False,'normativeCycleSemanticsImplemented':False,'anyTypeImplemented':False,'anyTypePlannedSlice':'22I',
148 'newDependencyDownloads':0,'persistentImagesTouched':False,'acceptancePassed':True})
149 print(json
.dumps({'acceptancePassed':True,'xtmChecks':len(labels
),'statuses':prior
['statusesAfter'],'faultControls':113,'actualBytes':len(actual
),'baselineBytes':len(expected
.read_bytes()),'firstDifferentByte':mismatch
}))
152 ap
=argparse
.ArgumentParser(description
=__doc__
);ap
.add_argument('--output',type=Path
,required
=True)
153 ap
.add_argument('--baseline',type=Path
,default
=TOOLS
/'baseline');ap
.add_argument('--initial',type=Path
,default
=TOOLS
/'initial-state.json');ap
.add_argument('--audit-only',action
='store_true');a
=ap
.parse_args()
154 out
=a
.output
.resolve();assert not out
.is_relative_to(ROOT
);out
.mkdir(parents
=True,exist_ok
=True)
156 v
=previous_validator();v
.EXECUTION_FILES
=ALLOWED|
{'tests/xtm-mergemap.lisp','scripts/validate-slice-22d.py','reports/slice-22d/failure-controls.lisp'}
157 # Its historical 108 labels plus the five datatype controls are exactly
158 # the 113 accepted controls, using the unchanged 22D executor driver.
159 v
.collect(out
,ROOT
/'reports/slice-22d/baseline')
160 audit(out
,a
.baseline
,a
.initial
)
161 if __name__
=='__main__':main()