]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22a/reproduce.py
Qualify closed XML/C14N 1.0 infrastructure for XTM resourceData
[topic-maps.git] / reports / slice-22a / reproduce.py
1 #!/usr/bin/env python3
2 """Reproduce Slice 22A through the existing offline harness and diagnostic APIs.
3 No production/test registration changes. Outputs must be outside the repository.
4 """
5 import argparse
6 import datetime
7 import hashlib
8 import importlib.util
9 import json
10 import os
11 from pathlib import Path
12 import shutil
13 import subprocess
14 import sys
15
16 sys.dont_write_bytecode = True
17 ROOT = Path(__file__).resolve().parents[2]
18 EVIDENCE = Path(__file__).resolve().parent
19 BASE = '1c0b92c68fe1114b926d4405b553dfb096d2f425'
20 RUNTIME = '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
21 POLICY = '(version 1)(allow default)(deny network*)(deny file-write* (subpath "'+str(ROOT)+'"))(deny file-write* (subpath "/Users/rgb/workspace/hyperdoc-dreyeck-ch"))'
22
23
24 def read(p): return json.loads(p.read_text())
25 def save(p,v): p.write_text(json.dumps(v,indent=2,ensure_ascii=False)+'\n')
26 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
27 def git(*args): return subprocess.check_output(['git',*args],cwd=ROOT,text=True).strip()
28
29
30 def verify_baseline(baseline):
31 previous = ROOT/'reports/slice-21o'
32 names=['xtm.json','ctm.validation.json','ctm.cxtm.json','ctm.encoding.json','after-diagnostics.json','further-variant-reification.json']
33 names += [str(p.relative_to(previous)) for p in sorted((previous/'retained').glob('*.json'))]
34 names += [p.name for p in sorted(previous.glob('*-control-*.json'))]
35 rows=[]
36 for name in names:
37 old,new=previous/name,baseline/name
38 assert read(old)==read(new),name
39 assert sha(old)==sha(new),name
40 rows.append({'previousReport':str(old.relative_to(ROOT)),'freshReport':'baseline/'+name,
41 'bytes':new.stat().st_size,'previousSHA256':sha(old),'freshSHA256':sha(new),'byteIdentical':True})
42 assert len(rows)==154
43 ex=read(baseline/'execution.json')
44 assert ex['acceptancePassed'] and ex['standaloneXTMChecks']==2384 and ex['faultControls']==103
45 assert ex['xtmStatusesAfter']=={'pass':70,'fail':0,'unsupported':0,'not-run':54}
46 assert ex['ctmStatusesAfter']=={'pass':262,'fail':4,'unsupported':0,'not-run':7}
47 assert len(ex['runs'])==109 and all(r['exitCode']==r['expectedExitCode'] for r in ex['runs'])
48 return rows
49
50
51 def main():
52 p=argparse.ArgumentParser(description=__doc__);p.add_argument('--output',type=Path,required=True);p.add_argument('--sbcl',default=RUNTIME)
53 p.add_argument('--reuse-baseline',type=Path,help='Verify an already completed fresh execution; never trust its status without byte comparisons.')
54 p.add_argument('--reuse-observations',type=Path,help='Verify retained fresh observations against all originals; no added executions.')
55 args=p.parse_args();out=args.output.resolve();assert not out.is_relative_to(ROOT);out.mkdir(parents=True,exist_ok=True)
56 assert not (out/'execution.json').exists(),'Choose an unused output directory.'
57 subprocess.run(['git','merge-base','--is-ancestor',BASE,'HEAD'],cwd=ROOT,check=True)
58 tracked=git('ls-tree','-r','--name-only',BASE).splitlines()
59 assert not git('diff',BASE,'--',*tracked)
60 protected={p:sha(ROOT/p) for p in tracked}
61 inputs={str(p.relative_to(ROOT)):sha(p) for p in [*EVIDENCE.glob('*.py'),*EVIDENCE.glob('*.lisp')]}
62 env={k:v for k,v in os.environ.items() if not k.startswith('TOPIC_MAPS_')};env.update(PYTHONDONTWRITEBYTECODE='1',GIT_OPTIONAL_LOCKS='0',SBCL=args.sbcl)
63 assert subprocess.check_output([args.sbcl,'--version'],text=True).strip()=='SBCL 2.5.4'
64 runs=[];started=datetime.datetime.now(datetime.timezone.utc).isoformat()
65 def run(label,command,env,sandbox=True):
66 if sandbox:command=['/usr/bin/sandbox-exec','-p',POLICY]+command
67 before=datetime.datetime.now(datetime.timezone.utc).isoformat();r=subprocess.run(command,cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
68 (out/(label+'.log')).write_bytes(r.stdout);runs.append(dict(label=label,command=command,exitCode=r.returncode,
69 startedAtUTC=before,finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),network='denied in every test child',repositoryWrites='denied in every test child'))
70 assert r.returncode==0,(label,r.stdout.decode()[-4000:]);return r.stdout.decode()
71 baseline=args.reuse_baseline.resolve() if args.reuse_baseline else out/'baseline'
72 if not args.reuse_baseline:
73 run('baseline-orchestration',['python3',str(ROOT/'scripts/validate-slice-21o.py'),'--sbcl',args.sbcl,'--output',str(baseline)],env,sandbox=False)
74 comparisons=verify_baseline(baseline);save(out/'baseline-comparison.json',comparisons)
75 observations=args.reuse_observations.resolve() if args.reuse_observations else out/'investigation'
76 if not args.reuse_observations:
77 observations.mkdir();prov=json.loads(run('source-preparation',['python3',str(ROOT/'scripts/validate-slice-21o.py'),'--prepare',str(out/'sources')],env))
78 save(observations/'source-preparation.json',prov)
79 env.update(CL_SOURCE_REGISTRY=prov['sourceRegistry'],TOPIC_MAPS_22A_ROOT=str(ROOT),TOPIC_MAPS_22A_OUTPUT=str(observations),TOPIC_MAPS_20J_OUTPUT=str(observations))
80 ids=[c['id'] for c in read(baseline/'ctm.validation.json')['cases'] if c['status']!='pass']
81 (observations/'case-ids.sexp').write_text('('+' '.join(json.dumps(x) for x in ids)+')\n');env['TOPIC_MAPS_22A_IDS']=str(observations/'case-ids.sexp')
82 prefix=[args.sbcl,'--noinform','--no-sysinit','--no-userinit','--script']
83 run('observations',prefix+[str(EVIDENCE/'observations.lisp')],env)
84 env['TOPIC_MAPS_22A_PROFILE_REPORT']=str(observations/'profile-probes.json')
85 run('profile-probes',prefix+[str(EVIDENCE/'profile-probes.lisp')],env)
86 assert read(observations/'ctm-observations.json')['caseStatesUnchanged']
87 assert read(observations/'controls.json')==read(ROOT/'reports/slice-20j/observed-controls.json')
88 assert len(read(observations/'profile-probes.json'))==8
89 spec=importlib.util.spec_from_file_location('closure_matrix',EVIDENCE/'build-matrix.py');module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
90 matrix=module.make_matrix(baseline,observations);save(out/'matrix.json',matrix);(out/'matrix.md').write_text(module.markdown_matrix(matrix))
91 lock=read(ROOT/'references/corpus-lock.json')
92 for entry in lock['files']:
93 f=ROOT/'vendor/cxtm'/entry['path'];assert f.stat().st_size==entry['size'] and sha(f)==entry['sha256']
94 for snapshot in [protected,inputs]:assert snapshot=={p:sha(ROOT/p) for p in snapshot}
95 save(out/'integrity.json',{'baseCommit':BASE,'allBaseTrackedFilesSHA256':protected,'historicalReportFilesVerified':sum(p.startswith('reports/') for p in protected),
96 'pinnedCorpusFilesVerified':len(lock['files']),'investigationToolsSHA256':inputs,'allBaseTrackedFilesUnchanged':True})
97 save(out/'execution.json',{'kind':'read-only-conformance-closure-investigation','baseCommit':BASE,'branch':git('branch','--show-current'),
98 'runtime':'SBCL 2.5.4','startedAtUTC':started,'finishedAtUTC':datetime.datetime.now(datetime.timezone.utc).isoformat(),
99 'baselineExecution':'baseline/execution.json','baselineWasFreshExecution':True,'reusedFreshExecutionDirectory':str(baseline) if args.reuse_baseline else None,
100 'reusedFreshObservationDirectory':str(observations) if args.reuse_observations else None,'behavioralReportsByteIdenticalToSlice21O':len(comparisons),
101 'baselineFreshTestProcesses':109,'additionalFreshObservationProcesses':2,'supplementalProfileProbes':8,'retainedTemplateContextProbes':12,
102 'ctmOriginalObservations':11,'xtmOriginalObservations':124,'matrixCases':65,'runs':runs,'statusesBefore':matrix['registeredTotals'],'statusesAfter':matrix['registeredTotals'],
103 'newOriginalPASS':0,'caseRegistrationChanged':False,'productionCodeChanged':False,'corpusChanged':False,'historicalReportsChanged':False,'hyperdocModifiedOrLoaded':False,
104 'newDependencyDownloads':0,'persistentLispImagesTouched':False,'validationPassed':True})
105 print(json.dumps({'validationPassed':True,'matrixCases':65,'reportsExactlyReproduced':len(comparisons),'output':str(out)}))
106
107 if __name__=='__main__':main()