]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22c/reproduce.py
test: prove CTM/XTM anyType equivalence (slice 22T)
[topic-maps.git] / reports / slice-22c / reproduce.py
1 #!/usr/bin/env python3
2 """Slice 22C: reuse the existing offline regression and read-only observers.
3 Default: execute all 114 regression processes, then CTM/auxiliary observations.
4 --reuse-regression accepts only a freshly executed, fully checked report directory.
5 Writes only to --output outside the repositories; no registration or norm download.
6 """
7 import argparse, datetime, hashlib, json, os, shutil, subprocess, sys
8 from pathlib import Path
9 sys.dont_write_bytecode = True
10 ROOT = Path(__file__).resolve().parents[2]
11 HERE = Path(__file__).resolve().parent
12 BASE = 'bc9dcfb02f945422970d65266dc66b991806777d'
13 SBCL = '/nix/store/hlsiblxpxr1hax3dw9x8znd6zy097ddj-sbcl-2.5.4/bin/sbcl'
14 HYPERDOC = Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch')
15 POLICY = f'(version 1)(allow default)(deny network*)(deny file-write* (subpath "{ROOT}"))(deny file-write* (subpath "{HYPERDOC}"))'
16 def read(p): return json.loads(p.read_text())
17 def save(p,v): p.write_text(json.dumps(v,ensure_ascii=False,indent=2)+'\n')
18 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
19 def git(*a): return subprocess.check_output(['git',*a],cwd=ROOT,text=True).strip()
20 def main():
21 ap=argparse.ArgumentParser(description=__doc__)
22 ap.add_argument('--output',type=Path,required=True);ap.add_argument('--sbcl',default=SBCL)
23 ap.add_argument('--reuse-regression',type=Path)
24 args=ap.parse_args();out=args.output.resolve()
25 assert not out.is_relative_to(ROOT) and not out.is_relative_to(HYPERDOC)
26 out.mkdir(parents=True,exist_ok=True)
27 assert not (out/'execution.json').exists(),'Choose a new output directory.'
28 assert subprocess.check_output([args.sbcl,'--version'],text=True).strip()=='SBCL 2.5.4'
29 subprocess.run(['git','merge-base','--is-ancestor',BASE,'HEAD'],cwd=ROOT,check=True)
30 tracked=git('ls-tree','-r','--name-only',BASE).splitlines()
31 assert not set(git('diff','--name-only',BASE).splitlines()) & set(tracked)
32 protected={p:sha(ROOT/p) for p in tracked}
33 env={k:v for k,v in os.environ.items() if not k.startswith('TOPIC_MAPS_')}
34 env.update(PYTHONDONTWRITEBYTECODE='1',GIT_OPTIONAL_LOCKS='0',SBCL=args.sbcl)
35 started=datetime.datetime.now(datetime.timezone.utc).isoformat();runs=[]
36 def run(label,command,child_env=env,sandbox=True):
37 if sandbox:command=['/usr/bin/sandbox-exec','-p',POLICY]+command
38 start=datetime.datetime.now(datetime.timezone.utc).isoformat()
39 p=subprocess.run(command,cwd=ROOT,env=child_env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,timeout=900)
40 (out/(label+'.log')).write_bytes(p.stdout)
41 runs.append(dict(label=label,command=command,exitCode=p.returncode,startedAtUTC=start,
42 finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
43 network='denied in test children',repositoryWrites='denied in test children',freshProcess=True))
44 assert p.returncode==0,(label,p.stdout.decode()[-4000:])
45 return p.stdout.decode()
46 regression=args.reuse_regression.resolve() if args.reuse_regression else out/'regression'
47 if not args.reuse_regression:
48 run('regression-orchestration',['python3',str(ROOT/'scripts/validate-slice-22b.py'),'--sbcl',args.sbcl,'--output',str(regression)],sandbox=False)
49 ex=read(regression/'execution.json')
50 expected={'XTM':{'pass':71,'fail':0,'not-run':53},'CTM':{'pass':262,'fail':4,'not-run':7},'total':{'pass':333,'fail':4,'not-run':60}}
51 assert ex['acceptancePassed'] and ex['statusesAfter']==expected
52 assert len(ex['runs'])==114 and ex['faultControls']==108
53 assert all(r['exitCode']==r['expectedExitCode'] and r['network']=='denied' and r['repositoryWrites']=='denied' for r in ex['runs'])
54 old=ROOT/'reports/slice-22b/after'
55 report_names=['xtm.json','ctm.validation.json','ctm.cxtm.json','ctm.encoding.json','after-diagnostics.json','original-contract.json']
56 report_names += [str(p.relative_to(old)) for p in sorted((old/'retained').glob('*.json'))]
57 report_names += [p.name for p in sorted(old.glob('*-control-*.json'))]
58 comparisons=[]
59 for name in report_names:
60 assert (old/name).read_bytes()==(regression/name).read_bytes(),name
61 comparisons.append(dict(previous='reports/slice-22b/after/'+name,fresh='regression/'+name,sha256=sha(regression/name),byteIdentical=True))
62 assert len(read(regression/'after-diagnostics.json'))==124
63 save(out/'regression-comparison.json',comparisons)
64 obs=out/'investigation';obs.mkdir()
65 prov=json.loads(run('source-preparation',['python3',str(ROOT/'scripts/validate-slice-21o.py'),'--prepare',str(out/'sources')]))
66 save(obs/'source-preparation.json',prov)
67 ids=[c['id'] for c in read(regression/'ctm.validation.json')['cases'] if c['status']!='pass']
68 assert len(ids)==11
69 (obs/'case-ids.sexp').write_text('('+' '.join(json.dumps(x) for x in ids)+')\n')
70 env.update(CL_SOURCE_REGISTRY=prov['sourceRegistry'],TOPIC_MAPS_22A_ROOT=str(ROOT),TOPIC_MAPS_22A_OUTPUT=str(obs),
71 TOPIC_MAPS_22A_IDS=str(obs/'case-ids.sexp'),TOPIC_MAPS_20J_OUTPUT=str(obs),TOPIC_MAPS_22C_OUTPUT=str(obs))
72 prefix=[args.sbcl,'--noinform','--no-sysinit','--no-userinit','--script']
73 run('ctm-observations',prefix+[str(ROOT/'reports/slice-22a/observations.lisp')])
74 run('xtm-auxiliary-observations',prefix+[str(HERE/'auxiliary.lisp')])
75 obs_comp=[]
76 for name in ['ctm-observations.json','controls.json']:
77 prior=ROOT/'reports/slice-22a/investigation'/name
78 assert (obs/name).read_bytes()==prior.read_bytes(),name
79 obs_comp.append(dict(name=name,sha256=sha(obs/name),byteIdenticalTo22A=True))
80 assert read(obs/'ctm-observations.json')['caseStatesUnchanged']
81 save(out/'observation-comparison.json',obs_comp)
82 lock=read(ROOT/'references/corpus-lock.json')
83 for e in lock['files']:
84 p=ROOT/'vendor/cxtm'/e['path'];assert p.stat().st_size==e['size'] and sha(p)==e['sha256']
85 assert protected=={p:sha(ROOT/p) for p in tracked}
86 save(out/'integrity.json',dict(baseCommit=BASE,allBaseTrackedFilesSHA256=protected,allBaseTrackedFilesUnchanged=True,
87 pinnedCorpusFilesVerified=len(lock['files']),historicalReportFilesVerified=sum(p.startswith('reports/') for p in tracked)))
88 save(out/'execution.json',dict(kind='slice-22c-read-only-normative-investigation',baseCommit=BASE,branch=git('branch','--show-current'),
89 runtime='SBCL 2.5.4',startedAtUTC=started,finishedAtUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),runs=runs,
90 regressionDirectory=str(regression),reusedFreshRegression=args.reuse_regression is not None,
91 regressionStartedAtUTC=ex['startedAtUTC'],regressionFinishedAtUTC=ex['finishedAtUTC'],regressionProcesses=114,
92 regressionBehavioralReportsByteIdentical=len(comparisons),diagnosticObservations=124,faultControls=108,
93 additionalObservationProcesses=2,ctmOriginalObservations=11,xtmOriginalObservations=124,auxiliaryReads=2,
94 statusesBefore=expected,statusesAfter=expected,newOriginalPASS=0,productionChanged=False,registrationChanged=False,
95 originalSourcesAndBaselinesUnchanged=True,historicalReportsUnchanged=True,newDownloads=0,persistentImages=False,validationPassed=True))
96 print(json.dumps(dict(validationPassed=True,reportsExactlyReproduced=len(comparisons),output=str(out))))
97 if __name__=='__main__':main()