]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-22f.py
2 """Validate direct CTM/XTM semantic pairs via existing ASDF/offline executors.
3 --audit-only verifies saved executions; it starts no test processes.
10 from pathlib
import Path
14 sys
. dont_write_bytecode
= True
15 ROOT
= Path ( __file__
). resolve (). parents
[ 1 ]
16 D
= ROOT
/ 'reports/slice-22f' ; PRIOR
= ROOT
/ 'reports/slice-22e/after'
17 BASE
= 'aeebbc7c6fdbf26642db0b7b5a358a3d04d62cca'
18 ALLOWED
={ 'topic-maps.asd' }
19 SOURCES
={ 'topic-maps.asd' , 'tests/ctm-xtm-equivalence.lisp' , 'reports/slice-22f/fresh-equivalence.lisp' , 'scripts/validate-slice-22d.py' , 'reports/slice-22d/failure-controls.lisp' }
20 def read ( p
): return json
. loads ( p
. read_text ())
21 def sha ( p
): return hashlib
. sha256 ( p
. read_bytes ()). hexdigest ()
22 def save ( p
, x
): p
. write_text ( json
. dumps ( x
, indent
= 2 , ensure_ascii
= False )+ ' \n ' )
23 def git (* a
, cwd
= ROOT
): return subprocess
. check_output ([ 'git' ,* a
], cwd
= cwd
, text
= True ). strip ()
25 spec
= importlib
. util
. spec_from_file_location ( 'slice22d' , ROOT
/ 'scripts/validate-slice-22d.py' ); v
= importlib
. util
. module_from_spec ( spec
); spec
. loader
. exec_module ( v
); return v
27 def audit ( out
, initial
):
28 state
= read ( initial
); assert state
[ 'baseCommit' ]== BASE
and state
[ 'initialIndexAndWorktreeClean' ]
29 assert set ( git ( 'diff' , '--name-only' , BASE
). splitlines ())& set ( state
[ 'trackedFileSHA256' ])<= ALLOWED
30 protected
={ p
: h
for p
, h
in state
[ 'trackedFileSHA256' ]. items () if p
not in ALLOWED
}
31 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
32 run
= read ( out
/ 'runs.json' ); assert len ( run
[ 'runs' ])== 119
33 assert all ( x
[ 'exitCode' ]== x
[ 'expectedExitCode' ] and sha ( out
/( x
[ 'label' ]+ '.log' ))== x
[ 'logSHA256' ] for x
in run
[ 'runs' ])
34 assert run
[ 'executionSourcesSHA256' ]=={ p
: sha ( ROOT
/ p
) for p
in run
[ 'executionSourcesSHA256' ]}
35 assert run
[ 'rawReportSHA256' ]=={ p
: sha ( out
/ p
) for p
in run
[ 'rawReportSHA256' ]}
36 # Existing standalone XTM behavior and registry are completely unchanged.
37 assert ( out
/ 'xtm.json' ). read_bytes ()==( PRIOR
/ 'xtm.json' ). read_bytes ()
38 assert read ( out
/ 'xtm.json' )[ 'statuses' ]=={ 'pass' : 72 , 'fail' : 0 , 'unsupported' : 0 , 'not-run' : 0 }
39 assert ( out
/ 'after-diagnostics.json' ). read_bytes ()==( PRIOR
/ 'after-diagnostics.json' ). read_bytes ()
40 assert len ( read ( out
/ 'after-diagnostics.json' ))== 124
43 if '-control-' not in x
[ 'label' ]: continue
44 p
= x
[ 'label' ]+ '.json' ; assert ( out
/ p
). read_bytes ()==( PRIOR
/ p
). read_bytes (), p
45 faults
. append ({ 'label' : x
[ 'label' ], 'exitCode' : x
[ 'exitCode' ], 'expectedASDFFailureObserved' : True , 'behavioralReportByteIdentical' : True , 'sha256' : sha ( out
/ p
)})
46 assert len ( faults
)== 113
47 save ( out
/ 'failure-controls.json' ,{ 'retained' : 113 , 'new' : 0 , 'controls' : faults
, 'allFaultReportsByteIdentical' : True })
49 for old
in [* sorted (( PRIOR
/ 'retained' ). glob ( '*.json' )),*[ PRIOR
/ f
'ctm. {s} .json' for s
in [ 'validation' , 'cxtm' , 'encoding' ]]]:
50 rel
= old
. relative_to ( PRIOR
); assert old
. read_bytes ()==( out
/ rel
). read_bytes (), str ( rel
)
51 unchanged
. append ({ 'report' : str ( rel
), 'sha256' : sha ( old
), 'byteIdentical' : True })
52 assert len ( unchanged
)== 48
53 save ( out
/ 'retained-reports.json' , unchanged
)
54 prior
= read ( PRIOR
/ 'execution.json' ); log
=( out
/ 'regression.log' ). read_text ()
55 for marker
, count
in dict ( prior
[ 'retainedChecks' ],** prior
[ 'stageChecks' ]). items ():
56 assert int ( re
. search ( r
'(?m)^' + marker
+ r
'-PASS: (\d+)' , log
). group ( 1 ))== count
57 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
58 assert 'CXML-SUMMARY 146 PASS 0 FAIL' in ( out
/ 'xml.log' ). read_text ()
59 eq
= read ( out
/ 'retained/xtm-mergemap.equivalence.json' ); assert eq
[ 'condition' ] is None and eq
[ 'originalCorpusRegistrationsAdded' ]== 0
60 labels
= re
. findall ( r
'^XTM-MERGEMAP-CHECK-PASS (equivalence:.+)$' , log
, re
. M
)
61 assert labels
== eq
[ 'checks' ] and len ( labels
)== len ( set ( labels
))== 487
62 assert 'CTM-XTM-EQUIVALENCE-PASS: 487 checks; 27 actual reader pairs; 3 contrasts; 3 explicit capability gaps.' in log
63 pairs
=[ p
for p
in eq
[ 'pairs' ] if p
[ 'kind' ]== 'direct-reader-equivalence-pair' ]
64 contrasts
=[ p
for p
in eq
[ 'pairs' ] if p
[ 'kind' ]== 'intentional-semantic-difference' ]
65 gaps
=[ p
for p
in eq
[ 'pairs' ] if p
[ 'kind' ]== 'reader-capability-gap-not-equivalence-PASS' ]
66 assert len ( pairs
)== 27 and len ( contrasts
)== 3 and len ( gaps
)== 3
67 allids
=[ p
[ 'id' ] for p
in eq
[ 'pairs' ]]; assert len ( allids
)== len ( set ( allids
))== 33
68 ( out
/ 'actual' ). mkdir ( exist_ok
= True ); summary
=[]
70 assert p
[ 'nativePropertiesEqual' ] and p
[ 'ctmCanonicalEQ' ] and p
[ 'xtmCanonicalEQ' ] and not p
[ 'crossMapEQRequired' ]
71 assert p
[ 'ctmCondition' ] is None and p
[ 'xtmCondition' ] is None and p
[ 'byteExact' ] and p
[ 'firstDifferentByte' ] is None
72 assert p
[ 'ctmTrace' ][ 'entryPoint' ]== 'topic-maps.ctm:deserialize-string' and p
[ 'xtmTrace' ][ 'entryPoint' ]== 'topic-maps.xtm:deserialize-octets'
73 assert p
[ 'ctmTrace' ][ 'sourceKey' ]!= p
[ 'xtmTrace' ][ 'sourceKey' ]
74 assert p
[ 'ctmTrace' ][ 'documentIRI' ]== p
[ 'xtmTrace' ][ 'documentIRI' ]== p
[ 'documentIRI' ]== p
[ 'canonicalBaseIRI' ]
75 assert p
[ 'ctmModel' ]== p
[ 'xtmModel' ]
76 ac
= p
[ 'ctmCXTM' ]. encode (); bc
= p
[ 'xtmCXTM' ]. encode (); assert ac
== bc
and len ( ac
)== p
[ 'ctmBytes' ]== p
[ 'xtmBytes' ]
77 name
= p
[ 'id' ];( out
/ 'actual' /( name
+ '.ctm.cxtm' )). write_bytes ( ac
);( out
/ 'actual' /( name
+ '.xtm.cxtm' )). write_bytes ( bc
)
78 summary
. append ({ 'id' : name
, 'documentIRI' : p
[ 'documentIRI' ], 'nativeCounts' : p
[ 'ctmModel' ][ 'counts' ], 'ctmBytes' : len ( ac
), 'xtmBytes' : len ( bc
),
79 'ctmSHA256' : hashlib
. sha256 ( ac
). hexdigest (), 'xtmSHA256' : hashlib
. sha256 ( bc
). hexdigest (), 'sourceCTMSHA256' : hashlib
. sha256 ( p
[ 'ctmSource' ]. encode ()). hexdigest (),
80 'sourceXTMSHA256' : hashlib
. sha256 ( p
[ 'xtmSource' ]. encode ()). hexdigest (), 'firstDifferentByte' : None , 'completeByteExact' : True })
82 a
= p
[ 'ctmCXTM' ]. encode (); b
= p
[ 'xtmCXTM' ]. encode (); assert a
!= b
and not p
[ 'nativePropertiesEqual' ] and not p
[ 'byteExact' ]
83 assert p
[ 'firstDifferentByte' ]== next ( i
for i
,( x
, y
) in enumerate ( zip ( a
, b
)) if x
!= y
)
85 assert p
[ 'ctmCondition' ] is None and p
[ 'ctmModel' ] and p
[ 'xtmModel' ] is None and not p
[ 'successfulReaderPair' ]
86 assert p
[ 'xtmCondition' ][ 'class' ]== 'XTM-UNSUPPORTED' and p
[ 'xtmCondition' ][ 'code' ]== 'unsupported-construct'
87 save ( out
/ 'equivalence-summary.json' ,{ 'checks' : 487 , 'actualPairs' : 27 , 'intentionalDifferences' : 3 , 'readerCapabilityGaps' : 3 , 'pairs' : summary
, 'contrasts' : contrasts
, 'gaps' : gaps
, 'originalCorpusRegistrationsAdded' : 0 })
88 save ( out
/ 'diagnostic-changes.json' ,{ 'originalDiagnoses' : 124 , 'changed' : 0 , 'byteIdenticalToSlice22E' : True , 'registeredStatusesChanged' : False })
89 lock
= read ( ROOT
/ 'references/corpus-lock.json' )
90 for e
in lock
[ 'files' ]:
91 p
= ROOT
/ 'vendor/cxtm' / e
[ 'path' ]; assert p
. stat (). st_size
== e
[ 'size' ] and sha ( p
)== e
[ 'sha256' ]
92 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
93 hd
= Path ( '/Users/rgb/workspace/hyperdoc-dreyeck-ch' ); assert git ( 'rev-parse' , 'HEAD' , cwd
= hd
)== state
[ 'hyperdocHEAD' ] and git ( 'status' , '--porcelain=v1' , cwd
= hd
)== state
[ 'hyperdocStatus' ]
94 save ( out
/ 'integrity.json' ,{ 'baseCommit' : BASE
, 'protectedBaseFilesSHA256' : protected
, 'executionSourcesSHA256' : run
[ 'executionSourcesSHA256' ],
95 'pinnedCorpusFilesVerified' : len ( lock
[ 'files' ]), 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
),
96 'productionSourcesUnchanged' : True , 'originalSourcesBaselinesCorpusLockHistoricalReportsUnchanged' : True , 'hyperdocUnchanged' : True })
97 save ( out
/ 'execution.json' ,{ 'kind' : 'slice-22f-direct-CTM-XTM-semantic-equivalence' , 'baseCommit' : BASE
, 'branch' : git ( 'branch' , '--show-current' ),
98 'runtime' : 'SBCL 2.5.4' , 'startedAtUTC' : run
[ 'startedAtUTC' ], 'finishedAtUTC' : run
[ 'finishedAtUTC' ], 'auditedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (),
99 'runs' : run
[ 'runs' ], 'collector' : 'scripts/validate-slice-22d.py::collect (unchanged)' , 'auditToolSHA256' : sha ( Path ( __file__
)),
100 'equivalenceChecks' : 487 , 'actualReaderPairs' : 27 , 'intentionalSemanticDifferences' : 3 , 'documentedReaderCapabilityGaps' : 3 ,
101 'retainedStandaloneXTMChecks' : 2704 , 'retainedLispChecks' : 12100 , 'retainedChecks' : prior
[ 'retainedChecks' ], 'stageChecks' : prior
[ 'stageChecks' ],
102 'xmlQualificationChecks' : 146 , 'faultControls' : 113 , 'newFaultControls' : 0 , 'originalDiagnoses' : 124 , 'diagnosticChanges' : 0 ,
103 'statusesBefore' : prior
[ 'statusesAfter' ], 'statusesAfter' : prior
[ 'statusesAfter' ], 'originalCorpusRegistrationsAdded' : 0 ,
104 'pythonUnitTests' : 2 , 'nfcOracleComparisons' : 16294 , 'retainedReportsByteIdentical' : 48 ,
105 'pinnedCorpusFilesVerified' : len ( lock
[ 'files' ]), 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
),
106 'knownCTMFailuresUnchanged' : prior
[ 'knownCTMFailuresUnchanged' ], 'productionChanges' :[], 'newDependencyDownloads' : 0 , 'persistentImagesTouched' : False , 'acceptancePassed' : True })
107 print ( json
. dumps ({ 'acceptancePassed' : True , 'readerPairs' : 27 , 'equivalenceChecks' : 487 , 'faultControls' : 113 , 'statuses' : prior
[ 'statusesAfter' ]}))
110 ap
= argparse
. ArgumentParser ( description
= __doc__
); ap
. add_argument ( '--output' , type = Path
, required
= True ); ap
. add_argument ( '--initial' , type = Path
, default
= D
/ 'initial-state.json' ); ap
. add_argument ( '--audit-only' , action
= 'store_true' ); a
= ap
. parse_args ()
111 out
= a
. output
. resolve (); assert not out
. is_relative_to ( ROOT
); out
. mkdir ( parents
= True , exist_ok
= True )
113 v
= legacy (); v
. EXECUTION_FILES
= SOURCES
; v
. collect ( out
, ROOT
/ 'reports/slice-22d/baseline' )
115 if __name__
== '__main__' : main ()