]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22g.py
2 """Slice 22G: reuse the existing 119-process offline collector; audit saved evidence.
3 --audit-only starts no tests. Output must be outside the repository.
13 from pathlib
import Path
14 sys
. dont_write_bytecode
= True
15 ROOT
= Path ( __file__
). resolve (). parents
[ 1 ]
16 D
= ROOT
/ 'reports/slice-22g' ; PRIOR
= ROOT
/ 'reports/slice-22f/after'
17 BASE
= 'b50a9d7950c5407b2ec02e1caccd76f73018b45f'
18 ALLOWED
={ 'src/xtm/reader.lisp' , 'topic-maps.asd' , 'tests/xtm-identity.lisp' , 'tests/xtm-name-scopes.lisp' , 'tests/xtm-resource-datatypes.lisp' , 'tests/xtm-mergemap.lisp' }
19 SOURCES
= ALLOWED|
{ 'tests/xtm-occurrences.lisp' , 'scripts/validate-slice-22d.py' , 'reports/slice-22d/failure-controls.lisp' }
20 def read ( p
): return json
. loads ( p
. read_text ())
21 def sha ( p
): return hashlib
. sha256 ( p
. read_bytes ()). hexdigest ()
22 def save ( p
, x
): p
. write_text ( json
. dumps ( x
, indent
= 2 , ensure_ascii
= False )+ ' \n ' )
23 def git (* args
): return subprocess
. check_output ([ 'git' ,* args
], cwd
= ROOT
, text
= True ). strip ()
24 def diff ( a
, b
, path
= '' ):
26 if isinstance ( a
, dict ) and isinstance ( b
, dict ): return [ d
for k
in sorted ( a
. keys () |b
. keys ()) for d
in diff ( a
. get ( k
), b
. get ( k
), path
+ '/' + k
)]
27 if isinstance ( a
, list ) and isinstance ( b
, list ) and len ( a
)== len ( b
): return [ d
for i
,( x
, y
) in enumerate ( zip ( a
, b
)) for d
in diff ( x
, y
, path
+ '/' + str ( i
))]
28 return [{ 'path' : path
, 'before' : a
, 'after' : b
}]
30 initial
= read ( D
/ 'initial-state.json' ); assert initial
[ 'baseCommit' ]== BASE
and initial
[ 'initialIndexAndWorktreeClean' ]
31 assert set ( git ( 'diff' , '--name-only' , BASE
). splitlines ())& set ( initial
[ 'trackedFileSHA256' ])<= ALLOWED
32 protected
={ p
: h
for p
, h
in initial
[ 'trackedFileSHA256' ]. items () if p
not in ALLOWED
}
33 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
34 runs
= read ( out
/ 'runs.json' ); assert len ( runs
[ 'runs' ])== 119
35 assert all ( x
[ 'exitCode' ]== x
[ 'expectedExitCode' ] and sha ( out
/( x
[ 'label' ]+ '.log' ))== x
[ 'logSHA256' ] for x
in runs
[ 'runs' ])
36 assert runs
[ 'executionSourcesSHA256' ]=={ p
: sha ( ROOT
/ p
) for p
in runs
[ 'executionSourcesSHA256' ]}
37 assert runs
[ 'rawReportSHA256' ]=={ p
: sha ( out
/ p
) for p
in runs
[ 'rawReportSHA256' ]}
38 old
= read ( PRIOR
/ 'xtm.json' ); new
= read ( out
/ 'xtm.json' ); targets
={ x
[ 'id' ] for x
in read ( D
/ 'originals-before.json' )}
39 assert new
[ 'statuses' ]=={ 'pass' : 83 , 'fail' : 0 , 'unsupported' : 0 , 'not-run' : 0 } and len ( new
[ 'cases' ])== 83
40 assert new
[ 'otherCorpusInputs' ]=={ 'positiveUnexecuted' : 38 , 'negativeUnregisteredUnexecuted' : 3 }
41 assert new
[ 'cases' ][: 72 ]== old
[ 'cases' ] and { x
[ 'id' ] for x
in new
[ 'cases' ][ 72 :]}== targets
42 assert len ({ x
[ 'id' ] for x
in new
[ 'cases' ]})== 83 and all ( x
[ 'status' ]== 'pass' and x
[ 'actualOutcome' ]== 'cxtm-match' for x
in new
[ 'cases' ][ 72 :])
43 assert new
[ 'checks' ][: len ( old
[ 'checks' ])]!=[]
44 assert [ x
for x
in new
[ 'checks' ] if x
in set ( old
[ 'checks' ])]== old
[ 'checks' ]
45 labels
= re
. findall ( r
'^XTM-CHECK-PASS (.+)$' ,( out
/ 'xtm.log' ). read_text (), re
. M
)
46 assert labels
== new
[ 'checks' ] and len ( labels
)== len ( set ( labels
))
47 actualdir
= out
/ 'actual' ; actualdir
. mkdir ( exist_ok
= True ); originals
=[]
48 sourceinfo
={ x
[ 'id' ]: x
for x
in read ( D
/ 'originals-before.json' )}
49 assert len ( new
[ 'occurrenceOriginals' ])== 11
50 for row
in new
[ 'occurrenceOriginals' ]:
51 source
= sourceinfo
[ row
[ 'id' ]]; assert row
[ 'condition' ] is None and row
[ 'status' ]== 'pass'
52 assert row
[ 'qualifiedParserCalls' ]== 1 and row
[ 'originalOctetsEQ' ] and row
[ 'oldHandlesCanonicalAndLive' ]
53 assert row
[ 'model' ][ 'canonicalOccurrenceReferencesEQ' ] and row
[ 'firstDifferentByte' ] is None
54 raw
= bytes
. fromhex ( row
[ 'sourceHex' ]); ac
= row
[ 'actualCXTM' ]. encode (); expected
=( ROOT
/ 'vendor/cxtm/xtm2/baseline' /( Path ( row
[ 'id' ]). name
+ '.cxtm' )). read_bytes ()
55 assert raw
==( ROOT
/ 'vendor/cxtm' / row
[ 'id' ]). read_bytes () and hashlib
. sha256 ( raw
). hexdigest ()== source
[ 'sourceSHA256' ]
56 assert ac
== expected
== row
[ 'expectedCXTM' ]. encode () and len ( ac
)== row
[ 'actualBytes' ]== row
[ 'expectedBytes' ]== source
[ 'baselineBytes' ]
57 p
= actualdir
/( Path ( row
[ 'id' ]). name
+ '.cxtm' ); p
. write_bytes ( ac
)
58 originals
. append ( dict ( row
, sourceSHA256
= source
[ 'sourceSHA256' ], baselineSHA256
= sha ( p
), actualSHA256
= sha ( p
), completeByteExact
= True ))
59 save ( out
/ 'original-contracts.json' , originals
)
60 controls
= new
[ 'occurrenceControls' ]; positive
=[ c
for c
in controls
if c
. get ( 'model' )]; negative
=[ c
for c
in controls
if c
. get ( 'condition' )]
61 assert len ( positive
)== 19 and len ( negative
)== 27 and len ( controls
)== 46
62 assert all ( c
[ 'model' ][ 'canonicalOccurrenceReferencesEQ' ] for c
in positive
)
63 merges
=[ c
for c
in positive
if c
[ 'label' ]. startswith ( 'merge-' )]
66 h
= c
[ 'retainedHandleEvidence' ]; assert c
[ 'occurrenceCountsAfterEachRead' ]==[ 1 , 2 ] and c
[ 'model' ][ 'occurrenceCount' ]== 1
67 assert h
[ 'readHandleCount' ]== h
[ 'distinctReadHandles' ]== 2 and h
[ 'canonicalGroupsInFinalMap' ]==[ 1 , 1 ] and h
[ 'replacedReadHandles' ]==[ True , True ]
68 assert { c
[ 'condition' ][ 'class' ] for c
in negative
}=={ 'XTM-REJECTION' , 'XTM-UNSUPPORTED' , 'XTM-XML-ERROR' , 'OCCURRENCE-ERROR' , 'OCCURRENCE-IDENTIFIER-ERROR' }
69 save ( out
/ 'native-and-negative-controls.json' ,{ 'positiveControls' : 19 , 'negativeControls' : 27 , 'mergingDimensions' :[ 'parent' , 'type' , 'scope' ], 'identityKinds' :[ 'SI' , 'SL' , 'II' ], 'controls' : controls
})
70 # Compare all original diagnostics, including unregistered capability-boundary effects.
71 before
={ x
[ 'id' ]: x
for x
in read ( PRIOR
/ 'after-diagnostics.json' )}; after
={ x
[ 'id' ]: x
for x
in read ( out
/ 'after-diagnostics.json' )}; assert len ( before
)== len ( after
)== 124
73 for id in sorted ( before
):
74 if before
[ id ]== after
[ id ]: continue
76 assert after
[ id ][ 'condition' ] is None and after
[ id ][ 'byteExact' ]; reason
= 'Newly implemented supported occurrence core; full original baseline match.'
78 assert id in { 'xtm2/in/itemid-occurrence.xtm' , 'xtm2/in/occurrence-duplicate-iid.xtm' , 'xtm2/in/occurrence-duplicate-iid2.xtm' , 'xtm2/in/occurrence-reifier.xtm' , 'xtm2/in/occurrence-duplicate-reifier.xtm' , 'xtm2/invalid/reifier-collision.xtm' }
79 assert after
[ id ][ 'condition' ][ 'class' ]== 'XTM-UNSUPPORTED' and after
[ id ][ 'condition' ][ 'code' ]== 'unsupported-construct'
80 assert 'occurrence' in after
[ id ][ 'condition' ][ 'detail' ]; reason
= 'Precise excluded occurrence II/reification boundary, not a registered PASS.'
81 changes
. append ({ 'id' : id , 'reason' : reason
, 'before' : before
[ id ], 'after' : after
[ id ], 'differences' : diff ( before
[ id ], after
[ id ])})
82 assert { x
[ 'id' ] for x
in changes
}== targets|
{ 'xtm2/in/itemid-occurrence.xtm' , 'xtm2/in/occurrence-duplicate-iid.xtm' , 'xtm2/in/occurrence-duplicate-iid2.xtm' , 'xtm2/in/occurrence-reifier.xtm' , 'xtm2/in/occurrence-duplicate-reifier.xtm' , 'xtm2/invalid/reifier-collision.xtm' }
83 save ( out
/ 'diagnostic-changes.json' ,{ 'originalDiagnoses' : 124 , 'changed' : len ( changes
), 'unchanged' : 124 - len ( changes
), 'changes' : changes
})
85 for x
in runs
[ 'runs' ]:
86 if '-control-' not in x
[ 'label' ]: continue
87 a
= read ( PRIOR
/( x
[ 'label' ]+ '.json' )); b
= read ( out
/( x
[ 'label' ]+ '.json' ))
88 assert a
[ 'cases' ]== b
[ 'cases' ] and a
[ 'statuses' ]== b
[ 'statuses' ], x
[ 'label' ]
89 assert x
[ 'exitCode' ]== 1 and len ( a
[ 'checks' ])== len ( b
[ 'checks' ])
91 permitted
={ '/scope' , '/occurrenceControls' , '/occurrenceOriginals' , '/scopeControls/35/sourceText' ,
92 '/scopeControls/35/condition/column' , '/scopeControls/35/condition/detail' , '/scopeControls/35/condition/message' ,
93 '/mapReifierControls/31/condition/detail' , '/mapReifierControls/31/condition/message' ,
94 '/variantReifierControls/35/condition/detail' , '/variantReifierControls/35/condition/message' }
95 assert all ( d
[ 'path' ] in permitted
for d
in differences
), x
[ 'label' ]
96 faults
. append ({ 'label' : x
[ 'label' ], 'exitCode' : 1 , 'expectedASDFFailureObserved' : True , 'caseRecordsAndOutcomesIdentical' : True , 'checkCoveragePreserved' : True , 'behavioralDifferences' : differences
})
97 assert len ( faults
)== 113
98 save ( out
/ 'failure-controls.json' ,{ 'retained' : 113 , 'new' : 0 , 'controls' : faults
, 'noFaultCountedAsPASS' : True })
99 # Preserve all non-XTM reports. The old equivalence gap has a more precise message.
101 for p
in sorted (( PRIOR
/ 'retained' ). glob ( '*.json' )):
102 rel
= p
. relative_to ( PRIOR
)
103 if p
. name
== 'xtm-mergemap.equivalence.json' : continue
104 assert p
. read_bytes ()==( out
/ rel
). read_bytes (), str ( rel
)
105 retained
. append ({ 'report' : str ( rel
), 'sha256' : sha ( p
), 'byteIdentical' : True })
106 for name
in [ 'ctm.validation.json' , 'ctm.cxtm.json' , 'ctm.encoding.json' ]:
107 assert ( PRIOR
/ name
). read_bytes ()==( out
/ name
). read_bytes (); retained
. append ({ 'report' : name
, 'sha256' : sha ( out
/ name
), 'byteIdentical' : True })
108 assert len ( retained
)== 48
109 save ( out
/ 'retained-reports.json' , retained
)
110 eqold
= read ( PRIOR
/ 'retained/xtm-mergemap.equivalence.json' ); eq
= read ( out
/ 'retained/xtm-mergemap.equivalence.json' )
111 assert eq
[ 'condition' ] is None and eq
[ 'checks' ]== eqold
[ 'checks' ] and len ( eq
[ 'checks' ])== 487
112 assert eq
[ 'pairs' ][: 30 ]== eqold
[ 'pairs' ][: 30 ] and len ( eq
[ 'pairs' ])== 33
113 eqdiff
= diff ( eqold
, eq
)
114 assert all ( x
[ 'path' ]. startswith ( '/pairs/30/xtmCondition/' ) and x
[ 'path' ]. split ( '/' )[- 1 ] in { 'message' , 'detail' } for x
in eqdiff
)
115 save ( out
/ 'equivalence-preservation.json' ,{ 'checks' : 487 , 'readerPairs' : 27 , 'originalPairSourcesModelsAndFullCXTMUnchanged' : True , 'differences' : eqdiff
, 'scope' : 'Only the existing unsupported occurrence-reification gap message is refined.' })
116 prior
= read ( PRIOR
/ 'execution.json' ); log
=( out
/ 'regression.log' ). read_text ()
117 for marker
, count
in dict ( prior
[ 'retainedChecks' ],** prior
[ 'stageChecks' ]). items (): assert int ( re
. search ( r
'(?m)^' + marker
+ r
'-PASS: (\d+)' , log
). group ( 1 ))== count
118 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
119 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in ( out
/ 'xml.log' ). read_text ()
120 lock
= read ( ROOT
/ 'references/corpus-lock.json' )
121 for x
in lock
[ 'files' ]:
122 p
= ROOT
/ 'vendor/cxtm' / x
[ 'path' ]; assert p
. stat (). st_size
== x
[ 'size' ] and sha ( p
)== x
[ 'sha256' ]
123 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
124 save ( out
/ 'integrity.json' ,{ 'protectedBaseFilesSHA256' : protected
, 'pinnedCorpusFilesVerified' : len ( lock
[ 'files' ]), 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
), 'originalSourcesBaselinesHistoricalReportsUnchanged' : True , 'onlyProductionChange' : 'src/xtm/reader.lisp' })
125 save ( out
/ 'retargeted-observations.json' ,{ 'retargetings' : read ( D
/ 'retargetings.json' ), 'existingTestReportDifferences' : diff ( old
,{ k
: v
for k
, v
in new
. items () if k
not in { 'occurrenceOriginals' , 'occurrenceControls' }})})
126 save ( out
/ 'execution.json' ,{ 'kind' : 'slice-22g-XTM-occurrence-core' , 'baseCommit' : BASE
, 'branch' : git ( 'branch' , '--show-current' ), 'runtime' : 'SBCL 2.5.4' ,
127 'startedAtUTC' : runs
[ 'startedAtUTC' ], 'finishedAtUTC' : runs
[ 'finishedAtUTC' ], 'auditedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (), 'runs' : runs
[ 'runs' ],
128 'collector' : 'scripts/validate-slice-22d.py::collect (unchanged)' , 'auditToolSHA256' : sha ( Path ( __file__
)), 'standaloneXTMChecks' : len ( labels
), 'retainedStandaloneXTMChecks' : len ( old
[ 'checks' ]),
129 'newXTMChecks' : len ( labels
)- len ( old
[ 'checks' ]), 'retainedLispChecks' : 12100 , 'retainedChecks' : prior
[ 'retainedChecks' ], 'stageChecks' : prior
[ 'stageChecks' ],
130 'equivalenceChecks' : 487 , 'equivalenceReaderPairs' : 27 , 'faultControls' : 113 , 'originalDiagnoses' : 124 , 'diagnosticChanges' : len ( changes
),
131 'statusesBefore' : prior
[ 'statusesAfter' ], 'statusesAfter' :{ 'XTM' :{ 'pass' : 83 , 'fail' : 0 , 'not-run' : 41 }, 'CTM' :{ 'pass' : 262 , 'fail' : 4 , 'not-run' : 7 }, 'total' :{ 'pass' : 345 , 'fail' : 4 , 'not-run' : 48 }},
132 'originalRegistrationsAdded' : 11 , 'xmlQualificationChecks' : 146 , 'pythonUnitTests' : 2 , 'nfcOracleComparisons' : 16294 , 'pinnedCorpusFilesVerified' : len ( lock
[ 'files' ]),
133 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
), 'knownCTMFailuresUnchanged' : prior
[ 'knownCTMFailuresUnchanged' ],
134 'productionChanges' :[ 'src/xtm/reader.lisp' ], 'newDependencyDownloads' : 0 , 'persistentImagesTouched' : False , 'acceptancePassed' : True })
135 print ( json
. dumps ({ 'acceptancePassed' : True , 'XTMChecks' : len ( labels
), 'newOriginalPASS' : 11 , 'diagnosticChanges' : len ( changes
), 'faultControls' : 113 }))
137 ap
= argparse
. ArgumentParser ( description
= __doc__
); ap
. add_argument ( '--output' , type = Path
, required
= True ); ap
. add_argument ( '--audit-only' , action
= 'store_true' ); a
= ap
. parse_args (); out
= a
. output
. resolve (); assert not out
. is_relative_to ( ROOT
); out
. mkdir ( parents
= True , exist_ok
= True )
139 spec
= importlib
. util
. spec_from_file_location ( 'v22d' , ROOT
/ 'scripts/validate-slice-22d.py' ); v
= importlib
. util
. module_from_spec ( spec
); spec
. loader
. exec_module ( v
); v
. EXECUTION_FILES
= SOURCES
; v
. collect ( out
, ROOT
/ 'reports/slice-22d/baseline' )
141 if __name__
== '__main__' : main ()