]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22a/reproduce.py
2 """Reproduce Slice 22A through the existing offline harness and diagnostic APIs.
3 No production/test registration changes. Outputs must be outside the repository.
11 from pathlib
import Path
16 sys
. dont_write_bytecode
= True
17 ROOT
= Path ( __file__
). resolve (). parents
[ 2 ]
18 EVIDENCE
= Path ( __file__
). resolve (). parent
19 BASE
= '1c0b92c68fe1114b926d4405b553dfb096d2f425'
20 RUNTIME
= '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
21 POLICY
= '(version 1)(allow default)(deny network*)(deny file-write* (subpath "' + str ( ROOT
)+ '"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
24 def read ( p
): return json
. loads ( p
. read_text ())
25 def save ( p
, v
): p
. write_text ( json
. dumps ( v
, indent
= 2 , ensure_ascii
= False )+ ' \n ' )
26 def sha ( p
): return hashlib
. sha256 ( p
. read_bytes ()). hexdigest ()
27 def git (* args
): return subprocess
. check_output ([ 'git' ,* args
], cwd
= ROOT
, text
= True ). strip ()
30 def verify_baseline ( baseline
):
31 previous
= ROOT
/ 'reports/slice-21o'
32 names
=[ 'xtm.json' , 'ctm.validation.json' , 'ctm.cxtm.json' , 'ctm.encoding.json' , 'after-diagnostics.json' , 'further-variant-reification.json' ]
33 names
+= [ str ( p
. relative_to ( previous
)) for p
in sorted (( previous
/ 'retained' ). glob ( '*.json' ))]
34 names
+= [ p
. name
for p
in sorted ( previous
. glob ( '*-control-*.json' ))]
37 old
, new
= previous
/ name
, baseline
/ name
38 assert read ( old
)== read ( new
), name
39 assert sha ( old
)== sha ( new
), name
40 rows
. append ({ 'previousReport' : str ( old
. relative_to ( ROOT
)), 'freshReport' : 'baseline/' + name
,
41 'bytes' : new
. stat (). st_size
, 'previousSHA256' : sha ( old
), 'freshSHA256' : sha ( new
), 'byteIdentical' : True })
43 ex
= read ( baseline
/ 'execution.json' )
44 assert ex
[ 'acceptancePassed' ] and ex
[ 'standaloneXTMChecks' ]== 2384 and ex
[ 'faultControls' ]== 103
45 assert ex
[ 'xtmStatusesAfter' ]=={ 'pass' : 70 , 'fail' : 0 , 'unsupported' : 0 , 'not-run' : 54 }
46 assert ex
[ 'ctmStatusesAfter' ]=={ 'pass' : 262 , 'fail' : 4 , 'unsupported' : 0 , 'not-run' : 7 }
47 assert len ( ex
[ 'runs' ])== 109 and all ( r
[ 'exitCode' ]== r
[ 'expectedExitCode' ] for r
in ex
[ 'runs' ])
52 p
= argparse
. ArgumentParser ( description
= __doc__
); p
. add_argument ( '--output' , type = Path
, required
= True ); p
. add_argument ( '--sbcl' , default
= RUNTIME
)
53 p
. add_argument ( '--reuse-baseline' , type = Path
, help = 'Verify an already completed fresh execution; never trust its status without byte comparisons.' )
54 p
. add_argument ( '--reuse-observations' , type = Path
, help = 'Verify retained fresh observations against all originals; no added executions.' )
55 args
= p
. parse_args (); out
= args
. output
. resolve (); assert not out
. is_relative_to ( ROOT
); out
. mkdir ( parents
= True , exist_ok
= True )
56 assert not ( out
/ 'execution.json' ). exists (), 'Choose an unused output directory.'
57 subprocess
. run ([ 'git' , 'merge-base' , '--is-ancestor' , BASE
, 'HEAD' ], cwd
= ROOT
, check
= True )
58 tracked
= git ( 'ls-tree' , '-r' , '--name-only' , BASE
). splitlines ()
59 assert not git ( 'diff' , BASE
, '--' ,* tracked
)
60 protected
={ p
: sha ( ROOT
/ p
) for p
in tracked
}
61 inputs
={ str ( p
. relative_to ( ROOT
)): sha ( p
) for p
in [* EVIDENCE
. glob ( '*.py' ),* EVIDENCE
. glob ( '*.lisp' )]}
62 env
={ k
: v
for k
, v
in os
. environ
. items () if not k
. startswith ( 'TOPIC_MAPS_' )}; env
. update ( PYTHONDONTWRITEBYTECODE
= '1' , GIT_OPTIONAL_LOCKS
= '0' , SBCL
= args
. sbcl
)
63 assert subprocess
. check_output ([ args
. sbcl
, '--version' ], text
= True ). strip ()== 'SBCL 2.5.4'
64 runs
=[]; started
= datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat ()
65 def run ( label
, command
, env
, sandbox
= True ):
66 if sandbox
: command
=[ '/usr/bin/sandbox-exec' , '-p' , POLICY
]+ command
67 before
= datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (); r
= subprocess
. run ( command
, cwd
= ROOT
, env
= env
, stdout
= subprocess
. PIPE
, stderr
= subprocess
. STDOUT
)
68 ( out
/( label
+ '.log' )). write_bytes ( r
. stdout
); runs
. append ( dict ( label
= label
, command
= command
, exitCode
= r
. returncode
,
69 startedAtUTC
= before
, finishedAtUTC
= datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (), network
= 'denied in every test child' , repositoryWrites
= 'denied in every test child' ))
70 assert r
. returncode
== 0 ,( label
, r
. stdout
. decode ()[- 4000 :]); return r
. stdout
. decode ()
71 baseline
= args
. reuse_baseline
. resolve () if args
. reuse_baseline
else out
/ 'baseline'
72 if not args
. reuse_baseline
:
73 run ( 'baseline-orchestration' ,[ 'python3' , str ( ROOT
/ 'scripts/validate-slice-21o.py' ), '--sbcl' , args
. sbcl
, '--output' , str ( baseline
)], env
, sandbox
= False )
74 comparisons
= verify_baseline ( baseline
); save ( out
/ 'baseline-comparison.json' , comparisons
)
75 observations
= args
. reuse_observations
. resolve () if args
. reuse_observations
else out
/ 'investigation'
76 if not args
. reuse_observations
:
77 observations
. mkdir (); prov
= json
. loads ( run ( 'source-preparation' ,[ 'python3' , str ( ROOT
/ 'scripts/validate-slice-21o.py' ), '--prepare' , str ( out
/ 'sources' )], env
))
78 save ( observations
/ 'source-preparation.json' , prov
)
79 env
. update ( CL_SOURCE_REGISTRY
= prov
[ 'sourceRegistry' ], TOPIC_MAPS_22A_ROOT
= str ( ROOT
), TOPIC_MAPS_22A_OUTPUT
= str ( observations
), TOPIC_MAPS_20J_OUTPUT
= str ( observations
))
80 ids
=[ c
[ 'id' ] for c
in read ( baseline
/ 'ctm.validation.json' )[ 'cases' ] if c
[ 'status' ]!= 'pass' ]
81 ( observations
/ 'case-ids.sexp' ). write_text ( '(' + ' ' . join ( json
. dumps ( x
) for x
in ids
)+ ') \n ' ); env
[ 'TOPIC_MAPS_22A_IDS' ]= str ( observations
/ 'case-ids.sexp' )
82 prefix
=[ args
. sbcl
, '--noinform' , '--no-sysinit' , '--no-userinit' , '--script' ]
83 run ( 'observations' , prefix
+[ str ( EVIDENCE
/ 'observations.lisp' )], env
)
84 env
[ 'TOPIC_MAPS_22A_PROFILE_REPORT' ]= str ( observations
/ 'profile-probes.json' )
85 run ( 'profile-probes' , prefix
+[ str ( EVIDENCE
/ 'profile-probes.lisp' )], env
)
86 assert read ( observations
/ 'ctm-observations.json' )[ 'caseStatesUnchanged' ]
87 assert read ( observations
/ 'controls.json' )== read ( ROOT
/ 'reports/slice-20j/observed-controls.json' )
88 assert len ( read ( observations
/ 'profile-probes.json' ))== 8
89 spec
= importlib
. util
. spec_from_file_location ( 'closure_matrix' , EVIDENCE
/ 'build-matrix.py' ); module
= importlib
. util
. module_from_spec ( spec
); spec
. loader
. exec_module ( module
)
90 matrix
= module
. make_matrix ( baseline
, observations
); save ( out
/ 'matrix.json' , matrix
);( out
/ 'matrix.md' ). write_text ( module
. markdown_matrix ( matrix
))
91 lock
= read ( ROOT
/ 'references/corpus-lock.json' )
92 for entry
in lock
[ 'files' ]:
93 f
= ROOT
/ 'vendor/cxtm' / entry
[ 'path' ]; assert f
. stat (). st_size
== entry
[ 'size' ] and sha ( f
)== entry
[ 'sha256' ]
94 for snapshot
in [ protected
, inputs
]: assert snapshot
=={ p
: sha ( ROOT
/ p
) for p
in snapshot
}
95 save ( out
/ 'integrity.json' ,{ 'baseCommit' : BASE
, 'allBaseTrackedFilesSHA256' : protected
, 'historicalReportFilesVerified' : sum ( p
. startswith ( 'reports/' ) for p
in protected
),
96 'pinnedCorpusFilesVerified' : len ( lock
[ 'files' ]), 'investigationToolsSHA256' : inputs
, 'allBaseTrackedFilesUnchanged' : True })
97 save ( out
/ 'execution.json' ,{ 'kind' : 'read-only-conformance-closure-investigation' , 'baseCommit' : BASE
, 'branch' : git ( 'branch' , '--show-current' ),
98 'runtime' : 'SBCL 2.5.4' , 'startedAtUTC' : started
, 'finishedAtUTC' : datetime
. datetime
. now ( datetime
. timezone
. utc
). isoformat (),
99 'baselineExecution' : 'baseline/execution.json' , 'baselineWasFreshExecution' : True , 'reusedFreshExecutionDirectory' : str ( baseline
) if args
. reuse_baseline
else None ,
100 'reusedFreshObservationDirectory' : str ( observations
) if args
. reuse_observations
else None , 'behavioralReportsByteIdenticalToSlice21O' : len ( comparisons
),
101 'baselineFreshTestProcesses' : 109 , 'additionalFreshObservationProcesses' : 2 , 'supplementalProfileProbes' : 8 , 'retainedTemplateContextProbes' : 12 ,
102 'ctmOriginalObservations' : 11 , 'xtmOriginalObservations' : 124 , 'matrixCases' : 65 , 'runs' : runs
, 'statusesBefore' : matrix
[ 'registeredTotals' ], 'statusesAfter' : matrix
[ 'registeredTotals' ],
103 'newOriginalPASS' : 0 , 'caseRegistrationChanged' : False , 'productionCodeChanged' : False , 'corpusChanged' : False , 'historicalReportsChanged' : False , 'hyperdocModifiedOrLoaded' : False ,
104 'newDependencyDownloads' : 0 , 'persistentLispImagesTouched' : False , 'validationPassed' : True })
105 print ( json
. dumps ({ 'validationPassed' : True , 'matrixCases' : 65 , 'reportsExactlyReproduced' : len ( comparisons
), 'output' : str ( out
)}))
107 if __name__
== '__main__' : main ()