]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/validate-slice-19d2.py
2 """Validate controlled XTM Mergemap and all retained suites, offline and fresh."""
8 from pathlib
import Path
14 ROOT
= Path ( __file__
). resolve (). parent
. parent
15 BASE
= 'f6a86668fce782a2e77d3bd763d58b0489079d75'
16 POLICY
= '(version 1)(allow default)(deny network*)'
20 return subprocess
. check_output ( args
, cwd
= ROOT
, text
= True ). strip ()
24 return hashlib
. sha256 ( path
. read_bytes ()). hexdigest ()
28 return json
. loads ( path
. read_text ())
32 parser
= argparse
. ArgumentParser ( description
= __doc__
)
33 parser
. add_argument ( '--sbcl' , default
= os
. environ
. get ( 'SBCL' , 'sbcl' ))
34 parser
. add_argument ( '--output' , type = Path
, default
= ROOT
/ 'reports/slice-19d2' )
35 args
= parser
. parse_args ()
36 runtime
= shutil
. which ( args
. sbcl
)
37 assert runtime
and command ([ runtime
, '--version' ]) == 'SBCL 2.5.4'
38 output
= args
. output
. resolve ()
39 output
. mkdir ( parents
= True , exist_ok
= True )
40 protected_paths
= [ 'src/tmdm' , 'src/cxtm' , 'src/xtm' , 'src/xml' , 'patches' , 'flake.nix' ,
41 'flake.lock' , 'vendor' , 'references' , 'scripts/prepare-cxml.py' ,
42 'tests/cxml-probe.lisp' , 'tests/cxml-encoding-regressions.lisp' , 'tests/xtm-identity.lisp' , 'scripts/test.sh' ]
43 assert not command ([ 'git' , 'diff' , BASE
, '--' ] + protected_paths
)
44 protected
= { str ( p
. relative_to ( ROOT
)): sha ( p
) for directory
in [ 'vendor/cxtm' , 'references' , 'patches' ]
45 for p
in ( ROOT
/ directory
). rglob ( '*' ) if p
. is_file ()}
46 sources
= { str ( p
. relative_to ( ROOT
)): sha ( p
) for directory
in [ 'src' , 'tests' , 'scripts' , 'docs' ]
47 for p
in ( ROOT
/ directory
). rglob ( '*' ) if p
. is_file () and p
. suffix
in [ '.lisp' , '.py' , '.sh' , '.md' ]}
48 sources
. update ({ p
: sha ( ROOT
/ p
) for p
in [ 'topic-maps.asd' , 'README.md' , 'flake.nix' , 'flake.lock' ]})
49 started
= datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat ()
50 xtm_source
= ' \n ' . join ( p
. read_text () for p
in sorted (( ROOT
/ 'src/xtm' ). glob ( '*.lisp' )))
51 assert xtm_source
. count ( '(topic-maps.xml:parse-octets ' ) == 1
52 assert not re
. search ( r
'\(cxml:parse[-\w]*\s' , xtm_source
, re
. I
)
54 with tempfile
. TemporaryDirectory ( prefix
= 'topic-maps-19d2-' ) as tmp
:
56 preparation
= [ 'python3' , 'scripts/prepare-cxml.py' , '--directory' , str ( tmp
/ 'sources' )]
57 provenance
= json
. loads ( command ( preparation
))
58 environment
= dict ( os
. environ
, CL_SOURCE_REGISTRY
= provenance
[ 'sourceRegistry' ], PYTHONDONTWRITEBYTECODE
= '1' )
59 base_report
= tmp
/ 'core.json'
60 final_prefix
= tmp
/ 'xtm-mergemap'
61 for label
, invocation
in [
62 ( 'xml' , [ runtime
, '--noinform' , '--no-sysinit' , '--no-userinit' , '--script' , 'tests/cxml-probe.lisp' ]),
63 ( 'xtm' , [ runtime
, '--noinform' , '--no-sysinit' , '--no-userinit' , '--script' , 'scripts/fresh-xtm.lisp' ]),
64 ( 'regression' , [ str ( ROOT
/ 'scripts/test.sh' )])]:
65 env
= dict ( environment
, SBCL
= runtime
)
67 env
[ 'TOPIC_MAPS_XTM_REPORT' ] = str ( output
/ 'xtm.json' )
68 if label
== 'regression' :
69 env
[ 'TOPIC_MAPS_REPORT' ] = str ( base_report
)
70 env
[ 'TOPIC_MAPS_XTM_MERGEMAP_REPORT' ] = str ( final_prefix
)
71 invocation
= [ '/usr/bin/sandbox-exec' , '-p' , POLICY
] + invocation
72 context
= { 'label' : label
, 'command' : invocation
, 'freshProcess' : True ,
73 'network' : 'denied by sandbox-exec' , 'startedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat ()}
74 result
= subprocess
. run ( invocation
, cwd
= ROOT
, env
= env
, stdout
= subprocess
. PIPE
,
75 stderr
= subprocess
. STDOUT
, timeout
= 240 )
76 ( output
/ ( label
+ '.log' )). write_bytes ( result
. stdout
)
77 context
. update ( exitCode
= result
. returncode
, finishedAtUTC
= datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat ())
78 assert result
. returncode
== 0 , label
+ ' failed; see saved log'
79 if label
in [ 'xml' , 'xtm' ]:
80 prefix
= 'CXML' if label
== 'xml' else 'XTM'
81 passed
= re
. findall ( r
'^' + prefix
+ r
'-CHECK-PASS (.+)$' , result
. stdout
. decode (), re
. M
)
82 assert len ( passed
) == len ( set ( passed
))
83 assert prefix
+ '-SUMMARY ' + str ( len ( passed
)) + ' PASS 0 FAIL' in result
. stdout
. decode ()
84 context
[ 'passedChecks' ] = len ( passed
)
85 assert len ( passed
) == ( 146 if label
== 'xml' else 327 )
87 base
= read ( base_report
)
88 baseline
= ROOT
/ 'reports/slice-19c'
89 assert base
[ 'statuses' ] == { 'pass' : 204 , 'fail' : 4 , 'unsupported' : 0 , 'not-run' : 65 }
90 assert base
== read ( baseline
/ 'validation.json' )
92 for old
in sorted ( baseline
. glob ( '*.json' )):
93 if old
. name
== 'execution.json' :
95 path
= base_report
if old
. name
== 'validation.json' else Path ( str ( base_report
) + '.' + old
. name
)
96 prior
, current
= read ( old
), read ( path
)
97 if old
. name
in [ 'mergemap.json' , 'cxtm.json' ]:
98 renamed
= { 'diagnostic:known-XTM-unimplemented:specific-condition' : 'diagnostic:XTM-explicit-resolver-required:specific-condition' ,
99 'diagnostic:known-XTM-unimplemented:source-position' : 'diagnostic:XTM-explicit-resolver-required:source-position' ,
100 'policy:XTM-recognized-without-retrieval:specific-condition' : 'policy:XTM-explicit-resolver-before-retrieval:specific-condition' ,
101 'policy:XTM-recognized-without-retrieval:source-position' : 'policy:XTM-explicit-resolver-before-retrieval:source-position' ,
102 'policy:XTM-does-not-call-resolver' : 'policy:XTM-without-resolver-does-not-fetch' }
103 assert current
[ 'checks' ] == [ renamed
. get ( c
, c
) for c
in prior
[ 'checks' ]]
104 assert { k
: v
for k
, v
in prior
. items () if k
!= 'checks' } == { k
: v
for k
, v
in current
. items () if k
!= 'checks' }
106 assert old
. read_bytes () == path
. read_bytes (), old
. name
107 retained
. append ( old
. name
)
108 final
= read ( Path ( str ( final_prefix
) + '.validation.json' ))
109 cxtm
= read ( Path ( str ( final_prefix
) + '.cxtm.json' ))
110 integration
= read ( Path ( str ( final_prefix
) + '.integration.json' ))
111 assert final
[ 'statuses' ] == { 'pass' : 205 , 'fail' : 4 , 'unsupported' : 0 , 'not-run' : 64 }
112 changes
= [( a
[ 'id' ], a
[ 'status' ], b
[ 'status' ]) for a
, b
in zip ( base
[ 'cases' ], final
[ 'cases' ]) if a
!= b
]
113 assert changes
== [( 'ctm/in/mergemap-xtm.ctm' , 'not-run' , 'pass' )]
114 assert len ( final
[ 'cases' ]) == len ( base
[ 'cases' ]) == 273
115 old_cxtm
= read ( Path ( str ( base_report
) + '.cxtm.json' ))
116 assert cxtm
[ 'cases' ][:- 1 ] == old_cxtm
[ 'cases' ]
117 assert cxtm
[ 'contractCounts' ] == { 'pass' : 177 , 'fail' : 4 }
118 original
= cxtm
[ 'cases' ][- 1 ]
119 expected
= ( ROOT
/ 'vendor/cxtm/ctm/baseline/mergemap-xtm.ctm.cxtm' ). read_bytes ()
120 actual
= original
[ 'actualCXTM' ]. encode ()
121 assert len ( actual
) == len ( expected
) == 1519
122 assert actual
== expected
== original
[ 'expectedCXTM' ]. encode ()
123 ( output
/ 'mergemap-xtm.actual.cxtm' ). write_bytes ( actual
)
124 for name
in [ 'validation' , 'cxtm' , 'integration' ]:
125 shutil
. copyfile ( Path ( str ( final_prefix
) + '.' + name
+ '.json' ), output
/ ( name
+ '.json' ))
126 assert integration
[ 'failed' ] == 0 and integration
[ 'passed' ] == len ( integration
[ 'checks' ]) == len ( set ( integration
[ 'checks' ]))
127 log
= ( output
/ 'regression.log' ). read_text ()
128 previous_totals
= read ( ROOT
/ 'reports/slice-19d1/regression-execution.json' )[ 'regressionChecks' ]
129 totals
= { marker
: int ( re
. search ( r
'(?m)^' + marker
+ r
'-PASS: (\d+)' , log
). group ( 1 )) for marker
in previous_totals
}
130 assert totals
== previous_totals
and sum ( totals
. values ()) == 12100
131 assert int ( re
. search ( r
'XTM-MERGEMAP-PASS: (\d+)' , log
). group ( 1 )) == integration
[ 'passed' ]
132 assert 'Ran 2 tests' in log
and 'NFC-ORACLE-PASS: 16294' in log
133 assert 'XTM-ADAPTER-BOUNDARY-PASS:' in log
134 xtm
= read ( output
/ 'xtm.json' )
135 assert xtm
== read ( ROOT
/ 'reports/slice-19d1/xtm-1.json' )
136 assert xtm
[ 'statuses' ] == { 'pass' : 6 , 'fail' : 0 , 'unsupported' : 0 , 'not-run' : 0 }
137 assert xtm
[ 'otherCorpusInputs' ] == { 'positiveUnexecuted' : 107 , 'negativeUnregisteredUnexecuted' : 11 }
138 forms
= ( ROOT
/ 'docs/slice-19d2-xtm-mergemap.md' ). read_text (). split ( '```lisp \n ' )[- 1 ]. split ( ' \n ```' , 1 )[ 0 ]
139 invocation
= [ '/usr/bin/sandbox-exec' , '-p' , POLICY
, runtime
, '--noinform' , '--no-sysinit' , '--no-userinit' , '--script' , '/dev/stdin' ]
140 inspection
= subprocess
. run ( invocation
, cwd
= ROOT
, env
= environment
, input = forms
. encode (),
141 stdout
= subprocess
. PIPE
, stderr
= subprocess
. STDOUT
, timeout
= 120 )
142 ( output
/ 'inspection.log' ). write_bytes ( inspection
. stdout
)
143 assert inspection
. returncode
== 0 , 'Documented inspection forms failed'
144 assert protected
== { p
: sha ( ROOT
/ p
) for p
in protected
}
145 assert sources
== { p
: sha ( ROOT
/ p
) for p
in sources
}
146 assert not command ([ 'git' , 'diff' , BASE
, '--' ] + protected_paths
)
147 evidence
= { 'baseCommit' : BASE
, 'branch' : command ([ 'git' , 'branch' , '--show-current' ]),
148 'startedAtUTC' : started
, 'finishedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (),
149 'runtime' : 'SBCL 2.5.4' , 'newDependencyDownloads' : 0 , 'sourcePreparationCommand' : preparation
,
150 'qualifiedSourceProvenance' : provenance
, 'sourceSHA256' : sources
,
151 'pinnedCorpusXMLPatchesAndTMDMUnchanged' : True , 'runs' : runs
,
152 'retainedLispChecks' : sum ( totals
. values ()), 'retainedChecks' : totals
,
153 'newXTMIntegrationChecks' : integration
[ 'passed' ], 'retainedReportsVerified' : retained
,
154 'retainedReportChange' : 'Five obsolete XTM-unimplemented check labels now test missing explicit resolver, in mergemap.json and its checks embedded in cxtm.json; original cases/observations unchanged' ,
155 'ctmStatuses' : final
[ 'statuses' ], 'ctmStatusChanges' : changes
,
156 'xtmCorpusStatuses' : { 'pass' : 6 , 'fail' : 0 , 'unsupported' : 0 , 'not-run' : 118 },
157 'xtmPositiveAndNegativeRegistration' : 'Six original positives executed; 107 other positives and 11 negative inputs remain unexecuted.' ,
158 'allFourKnownDiscrepanciesUnchanged' : True , 'originalBaselineByteCount' : 1519 ,
159 'documentedInspectionPassed' : True , 'rawUTF16AtXMLBoundaryEQ' : True , 'unchangedChildModelAndEQImportReferences' : True ,
160 'persistentLispImagesTouched' : False ,
161 'fullOfflineNixRebuild' : 'NOT-RUN; existing verified Store sources and unchanged local patches, fresh offline ASDF loads.' }
162 ( output
/ 'execution.json' ). write_text ( json
. dumps ( evidence
, indent
= 2 ) + ' \n ' )
163 print ( json
. dumps ({ k
: evidence
[ k
] for k
in [ 'runtime' , 'retainedLispChecks' , 'newXTMIntegrationChecks' , 'ctmStatuses' , 'xtmCorpusStatuses' ]}, indent
= 2 ))
166 if __name__
== '__main__' :