]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22q/reproduce.py
2 """Run the existing offline collector, then audit Slice 22Q characterization.
3 Output must be a new directory outside the repository. This is not a C14N engine.
5 import argparse
, datetime
, hashlib
, importlib
. util
, json
, re
, subprocess
, sys
6 from pathlib
import Path
7 sys
. dont_write_bytecode
= True
8 ROOT
= Path ( __file__
). resolve (). parents
[ 2 ]
9 D
= ROOT
/ 'reports/slice-22q' ; PRIOR
= ROOT
/ 'reports/slice-22n/after'
10 BASE
= '1418be83ad013fe6d69e6036e6786be8100b9355'
11 ALLOWED
={ 'topic-maps.asd' , 'tests/xtm-identity.lisp' }
12 def read ( p
): return json
. loads ( p
. read_text ())
13 def sha ( p
): return hashlib
. sha256 ( p
. read_bytes ()). hexdigest ()
14 def save ( p
, x
): p
. write_text ( json
. dumps ( x
, indent
= 2 , ensure_ascii
= False )+ ' \n ' )
16 initial
= read ( D
/ 'initial-state.json' ); assert initial
[ 'baseCommit' ]== BASE
and initial
[ 'initialIndexAndWorktreeClean' ]
17 protected
={ p
: h
for p
, h
in initial
[ 'trackedFileSHA256' ]. items () if p
not in ALLOWED
}
18 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
19 run
= read ( out
/ 'runs.json' ); assert len ( run
[ 'runs' ])== 119
20 assert all ( r
[ 'exitCode' ]== r
[ 'expectedExitCode' ] and r
[ 'network' ]== r
[ 'repositoryWrites' ]== 'denied' and sha ( out
/( r
[ 'label' ]+ '.log' ))== r
[ 'logSHA256' ] for r
in run
[ 'runs' ])
21 assert run
[ 'executionSourcesSHA256' ]=={ p
: sha ( ROOT
/ p
) for p
in run
[ 'executionSourcesSHA256' ]}
22 assert run
[ 'rawReportSHA256' ]=={ p
: sha ( out
/ p
) for p
in run
[ 'rawReportSHA256' ]}
23 old
= read ( PRIOR
/ 'xtm.json' ); new
= read ( out
/ 'xtm.json' )
25 if key
!= 'checks' : assert old
[ key
]== new
[ key
], key
26 assert [ c
for c
in new
[ 'checks' ] if c
in set ( old
[ 'checks' ])]== old
[ 'checks' ]
27 additions
=[ c
for c
in new
[ 'checks' ] if c
not in set ( old
[ 'checks' ])]
28 assert additions
and all ( c
. startswith (( 'c14n-probe:' , 'c14n-native:' , 'c14n-api:' , 'c14n-characterization:' )) for c
in additions
)
29 labels
= re
. findall ( r
'^XTM-CHECK-PASS (.+)$' ,( out
/ 'xtm.log' ). read_text (), re
. M
)
30 assert labels
== new
[ 'checks' ] and len ( labels
)== len ( set ( labels
))
31 x
= new
[ 'c14nPreflight' ]; explore
= read ( D
/ 'exploration.json' )
32 assert x
[ 'executed' ] and not x
[ 'W3CC14NQualified' ] and not x
[ 'XTMReaderIntegration' ]
33 assert x
[ 'probes' ]== explore
[ 'probes' ] and x
[ 'nativeValues' ]== explore
[ 'nativeValues' ]
35 for row
in x
[ 'probes' ]:
36 actual
= bytes
. fromhex ( row
[ 'actualHex' ]); expected
= bytes
. fromhex ( row
[ 'expectedHex' ]); raw
= bytes
. fromhex ( row
[ 'inputHex' ])
37 assert actual
== row
[ 'actualXML' ]. encode ( 'utf-8' ) and expected
== row
[ 'expectedXML' ]. encode ( 'utf-8' )
38 assert len ( actual
)== row
[ 'actualBytes' ] and len ( expected
)== row
[ 'expectedBytes' ]
39 match
= actual
== expected
; assert match
==( row
[ 'normativeComparison' ]== 'PASS-for-this-probe' )
40 delta
= next (( i
for i
,( a
, b
) in enumerate ( zip ( actual
, expected
)) if a
!= b
), None )
41 if delta
is None and len ( actual
)!= len ( expected
): delta
= min ( len ( actual
), len ( expected
))
42 assert delta
== row
[ 'firstDifferentByte' ] and not row
[ 'W3C-C14N-1.0-qualified' ]
43 comparisons
. append ({ k
: row
[ k
] for k
in [ 'id' , 'expectedBytes' , 'actualBytes' , 'firstDifferentByte' , 'normativeComparison' ]})
44 assert len ({ r
[ 'id' ] for r
in comparisons
})== len ( comparisons
)
45 assert any ( r
[ 'normativeComparison' ]== 'BYTE-DIFFERENCE' for r
in comparisons
)
46 for item
in x
[ 'nativeValues' ]:
47 assert item
[ 'occurrenceValue' ]== item
[ 'variantValue' ] and item
[ 'datatype' ]== 'http://www.w3.org/2001/XMLSchema#anyType'
48 assert item
[ 'canonicalParentTypeScopeEQ' ] and item
[ 'repeatedCXTMIdentical' ] and item
[ 'modelUnchanged' ] and not item
[ 'readerIntegration' ]
49 assert item
[ 'topicCountBefore' ]== item
[ 'topicCountAfter' ]
50 olddiag
= read ( PRIOR
/ 'after-diagnostics.json' ); newdiag
= read ( out
/ 'after-diagnostics.json' ); assert olddiag
== newdiag
52 for p
in sorted (( out
/ 'retained' ). rglob ( '*' )):
54 rel
= p
. relative_to ( out
); assert ( PRIOR
/ rel
). exists () and p
. read_bytes ()==( PRIOR
/ rel
). read_bytes (), str ( rel
)
55 retained
. append ({ 'report' : str ( rel
), 'SHA256' : sha ( p
)})
58 if '-control-' not in r
[ 'label' ]: continue
59 name
= r
[ 'label' ]+ '.json' ; a
= read ( PRIOR
/ name
); b
= read ( out
/ name
)
60 assert r
[ 'exitCode' ]== 1
61 for key
in a
: assert a
[ key
]== b
[ key
],( name
, key
)
62 assert not b
[ 'c14nPreflight' ][ 'executed' ] and not b
[ 'c14nPreflight' ][ 'probes' ] and not b
[ 'c14nPreflight' ][ 'nativeValues' ]
63 controls
. append ({ 'label' : r
[ 'label' ], 'expectedExitCode' : 1 , 'previousReportFieldsUnchanged' : True , 'newProbesNotClaimedExecuted' : True })
64 assert len ( controls
)== 113
65 prior
= read ( PRIOR
/ 'execution.json' ); statuses
= prior
[ 'statusesAfter' ]
66 assert new
[ 'cases' ]== old
[ 'cases' ] and new
[ 'otherCorpusInputs' ]=={ 'positiveUnexecuted' : 3 , 'negativeUnregisteredUnexecuted' : 0 }
67 assert protected
=={ p
: sha ( ROOT
/ p
) for p
in protected
}
68 save ( out
/ 'comparison-summary.json' ,{ 'kind' : 'CXML-candidate-vs-independent-XTM-C14N-oracles' , 'W3CC14NQualified' : False , 'readerIntegration' : False , 'probes' : comparisons
, 'matchedProbeRows' : sum ( r
[ 'normativeComparison' ]== 'PASS-for-this-probe' for r
in comparisons
), 'differingProbeRows' : sum ( r
[ 'normativeComparison' ]== 'BYTE-DIFFERENCE' for r
in comparisons
), 'nativeValueRows' : len ( x
[ 'nativeValues' ]), 'nativeValuesApplyOnlyToMatchingProbeRows' : True })
69 save ( out
/ 'failure-controls-audit.json' , controls
)
70 save ( out
/ 'retained-reports.json' , retained
)
71 save ( out
/ 'integrity.json' ,{ 'protectedBaseFilesSHA256' : protected
, 'pinnedCorpusFilesVerified' : sum ( p
. startswith ( 'vendor/cxtm/' ) for p
in protected
), 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
), 'allProductionAndOriginalSourcesUnchanged' : True , 'testOnlyChanges' : sorted ( ALLOWED|
{ 'tests/xml-c14n-preflight.lisp' })})
72 save ( out
/ 'execution.json' ,{ 'kind' : 'slice-22q-preflight-characterization-NOT-C14N-qualification' , 'baseCommit' : BASE
, 'branch' : subprocess
. check_output ([ 'git' , 'branch' , '--show-current' ], cwd
= ROOT
, text
= True ). strip (), 'runtime' : 'SBCL 2.5.4' , 'startedAtUTC' : run
[ 'startedAtUTC' ], 'finishedAtUTC' : run
[ 'finishedAtUTC' ], 'auditedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (), 'collector' : 'scripts/validate-slice-22d.py::collect (unchanged)' , 'auditToolSHA256' : sha ( Path ( __file__
)), 'runs' : run
[ 'runs' ], 'standaloneXTMChecks' : len ( labels
), 'retainedStandaloneXTMChecks' : len ( old
[ 'checks' ]), 'newCharacterizationChecks' : len ( additions
), 'retainedLispChecks' : prior
[ 'retainedLispChecks' ], 'stageChecks' : prior
[ 'stageChecks' ], 'equivalenceChecks' : prior
[ 'equivalenceChecks' ], 'equivalenceReaderPairs' : prior
[ 'equivalenceReaderPairs' ], 'faultControls' : len ( controls
), 'all124OriginalDiagnosesUnchanged' : True , 'retainedReportsByteIdentical' : len ( retained
), 'statusesBefore' : statuses
, 'statusesAfter' : statuses
, 'originalRegistrationsAdded' : 0 , 'productionChanges' :[], 'xmlQualificationChecks' : prior
[ 'xmlQualificationChecks' ], 'pythonUnitTests' : prior
[ 'pythonUnitTests' ], 'nfcOracleComparisons' : prior
[ 'nfcOracleComparisons' ], 'W3CC14NQualified' : False , 'XTMReaderIntegration' : False , 'newDependencies' : 0 , 'nativeValueRows' : len ( x
[ 'nativeValues' ]), 'acceptancePassed' : True })
73 print ( json
. dumps ({ 'acceptancePassed' : True , 'W3CC14NQualified' : False , 'XTMChecks' : len ( labels
), 'newCharacterizationChecks' : len ( additions
), 'faultControls' : len ( controls
), 'statusesUnchanged' : statuses
}), flush
= True )
75 ap
= argparse
. ArgumentParser ( description
= __doc__
); ap
. add_argument ( '--output' , type = Path
, required
= True ); ap
. add_argument ( '--audit-only' , action
= 'store_true' ); a
= ap
. parse_args (); out
= a
. output
. resolve (); assert not out
. is_relative_to ( ROOT
); out
. mkdir ( parents
= True , exist_ok
= True )
77 spec
= importlib
. util
. spec_from_file_location ( 'oldcollector' , ROOT
/ 'scripts/validate-slice-22d.py' ); m
= importlib
. util
. module_from_spec ( spec
); spec
. loader
. exec_module ( m
)
78 m
. EXECUTION_FILES
={ str ( p
. relative_to ( ROOT
)) for folder
in [ 'src' , 'tests' ] for p
in ( ROOT
/ folder
). rglob ( '*.lisp' )} |
{ 'topic-maps.asd' , 'scripts/validate-slice-22d.py' , 'reports/slice-22d/failure-controls.lisp' , 'reports/slice-22q/reproduce.py' }
79 m
. collect ( out
, ROOT
/ 'reports/slice-22d/baseline' )
81 if __name__
== '__main__' : main ()