]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - reports/slice-22c/audit-evidence.py
Implement noncyclic XTM mergeMap content import (Slice 22E)
[topic-maps.git] / reports / slice-22c / audit-evidence.py
1 #!/usr/bin/env python3
2 """Read-only audit of Slice 22C evidence; writes its result only outside both repos."""
3 import argparse, ast, datetime, hashlib, json, re, subprocess
4 from pathlib import Path
5 from urllib.parse import unquote
6 ROOT = Path(__file__).resolve().parents[2]
7 HERE = Path(__file__).resolve().parent
8 HYPERDOC = Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch')
9 BASE = 'bc9dcfb02f945422970d65266dc66b991806777d'
10 def read(p): return json.loads(p.read_text())
11 def sha(p): return hashlib.sha256(p.read_bytes()).hexdigest()
12 def git(*args, cwd=ROOT): return subprocess.check_output(['git', *args], cwd=cwd, text=True).strip()
13 def main():
14 ap=argparse.ArgumentParser(description=__doc__);ap.add_argument('--output',type=Path,required=True)
15 out=ap.parse_args().output.resolve()
16 assert not out.is_relative_to(ROOT) and not out.is_relative_to(HYPERDOC)
17 out.mkdir(parents=True,exist_ok=True)
18 initial=read(HERE/'initial-state.json');protected=initial['trackedFileSHA256']
19 assert len(protected)==6891 and protected=={p:sha(ROOT/p) for p in protected}
20 assert git('merge-base',BASE,'HEAD')==BASE
21 historical=[p for p in protected if p.startswith('reports/')];assert len(historical)==4615
22 lock=read(ROOT/'references/corpus-lock.json')
23 for f in lock['files']:
24 p=ROOT/'vendor/cxtm'/f['path'];assert p.stat().st_size==f['size'] and sha(p)==f['sha256']
25 assert len(lock['files'])==2006
26 hyperdoc=initial['priorHyperdocInspection']
27 hyperdoc_head=git('rev-parse','HEAD',cwd=HYPERDOC)
28 hyperdoc_status=git('status','--porcelain=v1',cwd=HYPERDOC)
29 hyperdoc_hashes={p:sha(HYPERDOC/p) for p in hyperdoc['files']}
30 hyperdoc_changed={p:{'before':h,'after':hyperdoc_hashes[p]} for p,h in hyperdoc['files'].items() if h!=hyperdoc_hashes[p]}
31 # Other work may advance this neighbouring repo while the user pauses/resumes
32 # the task. Record that observation; never reset it or attribute it to 22C.
33 new_paths=set(git('ls-files','--others','--exclude-standard').splitlines())|set(git('diff','--name-only',BASE).splitlines())
34 allowed=lambda p: p.startswith('reports/slice-22c/') or p in ['docs/slice-22c-normative-decisions.md','docs/slice-22c-original-evidence.md']
35 assert all(allowed(p) for p in new_paths)
36 # Also include committed 22C files when verifying the finished commit later.
37 artifacts=[p for p in HERE.rglob('*') if p.is_file()]+[ROOT/'docs/slice-22c-normative-decisions.md',ROOT/'docs/slice-22c-original-evidence.md']
38 assert not any(p.is_symlink() or p.suffix in ['.fasl','.pyc','.pdf','.html'] for p in artifacts)
39 json_count=0
40 for p in artifacts:
41 if p.suffix=='.json':read(p);json_count+=1
42 if p.suffix=='.py':ast.parse(p.read_text(),filename=str(p))
43 execution=read(HERE/'execution.json');reg=read(HERE/'regression/execution.json')
44 expected={'CTM':{'pass':262,'fail':4,'not-run':7},'XTM':{'pass':71,'fail':0,'not-run':53},'total':{'pass':333,'fail':4,'not-run':60}}
45 assert execution['statusesBefore']==execution['statusesAfter']==reg['statusesAfter']==expected
46 assert execution['validationPassed'] and execution['newOriginalPASS']==0 and reg['acceptancePassed']
47 assert len(reg['runs'])==114 and sum(r['expectedExitCode']==1 for r in reg['runs'])==108
48 for r in reg['runs']:
49 assert r['exitCode']==r['expectedExitCode']
50 assert r['network']==r['repositoryWrites']==r['hyperdocWrites']=='denied'
51 assert r['command'][:2]==['/usr/bin/sandbox-exec','-p'] and '(deny network*)' in r['command'][2]
52 if any('/sbcl' in arg for arg in r['command']):
53 assert '--no-sysinit' in r['command'] and '--no-userinit' in r['command']
54 comparisons=read(HERE/'regression-comparison.json');assert len(comparisons)==159
55 for c in comparisons:
56 previous=ROOT/c['previous'];fresh=HERE/c['fresh']
57 assert previous.read_bytes()==fresh.read_bytes() and sha(fresh)==c['sha256']
58 assert len(read(HERE/'regression/after-diagnostics.json'))==124
59 for c in read(HERE/'observation-comparison.json'):
60 p=HERE/'investigation'/c['name']
61 assert sha(p)==c['sha256'] and p.read_bytes()==(ROOT/'reports/slice-22a/investigation'/c['name']).read_bytes()
62 evidence=read(HERE/'case-evidence.json');assert len(evidence['cases'])==13
63 assert evidence['decisionCounts']=={'A':1,'B':0,'C':12} and evidence['statusChanges']==evidence['newPASS']==0
64 original_ids={c['id'] for c in read(ROOT/'reports/slice-22a/matrix.json')['cases'] if c['id'].startswith('ctm/') or c['id'] in ['xtm2/in/mergemap-itemid.xtm','xtm2/in/mergemap-loop.xtm']}
65 assert {c['id'] for c in evidence['cases']}==original_ids
66 count_diff=count_templates=count_expanded=0
67 for c in evidence['cases']:
68 assert c['registeredBefore']==c['registeredAfter']
69 for f in [c['source'],c['baseline'],*c['auxiliarySources']]:
70 if f is None:continue
71 p=ROOT/f['path'];b=p.read_bytes()
72 assert b.hex()==f['hex'] and len(b)==f['bytes'] and sha(p)==f['sha256']
73 if c.get('completeModelComparison'):
74 count_diff+=1
75 assert c['actualCondition'] is None and c['completeModelComparison']['nativePropertiesMatchActualCXTM']
76 assert len(c['completeModelComparison']['differences'])==1 and not c['comparison']['byteExact']
77 actual=HERE/'comparisons'/(Path(c['id']).name+'.actual.cxtm')
78 assert sha(actual)==c['actualCXTM']['sha256'] and actual.read_bytes().hex()==c['actualCXTM']['hex']
79 assert (HERE/'comparisons'/(Path(c['id']).name+'.diff')).read_text()==c['completeCXTMUnifiedDiff']
80 if c.get('templateAnalysis'):
81 count_templates+=1;assert c['variablePositionsReverifiedFromOriginal']
82 assert c['actualCondition']['condition']=='INVALID-TEMPLATE-BODY'
83 for control in c['expandedControls']:
84 count_expanded+=1;assert control['byteExact'] and control['condition'] is None
85 assert control['actualCXTM']==c['baseline']['text']
86 if c.get('intendedNegativeCause'):assert c['intendedNegativeCause']['currentlyReached'] is False
87 assert (count_diff,count_templates,count_expanded)==(4,7,3)
88 links=[]
89 for p in [ROOT/'docs/slice-22c-normative-decisions.md',ROOT/'docs/slice-22c-original-evidence.md']:
90 for target in re.findall(r'\]\(([^)]+)\)',p.read_text()):
91 if '://' in target:continue
92 dest=(p.parent/unquote(target.split('#')[0])).resolve()
93 assert dest.exists() or dest in [HERE/'precommit-audit.json',HERE/'artifacts.json'],(p,target)
94 links.append(dict(document=str(p.relative_to(ROOT)),target=target))
95 sources=['scripts/validate-slice-22b.py','scripts/validate-slice-21o.py','reports/slice-22a/observations.lisp','reports/slice-20j/controls.lisp','reports/slice-22c/reproduce.py','reports/slice-22c/auxiliary.lisp','reports/slice-22c/build-evidence.py','reports/slice-22c/audit-evidence.py']
96 result=dict(kind='slice-22c-final-read-only-evidence-audit',timeUTC=datetime.datetime.now(datetime.timezone.utc).isoformat(),
97 baseCommit=BASE,observedHEAD=git('rev-parse','HEAD'),branch=git('branch','--show-current'),
98 baseTrackedFilesVerified=len(protected),historicalReportsVerified=len(historical),pinnedCorpusFilesVerified=len(lock['files']),
99 allOriginalFilesUnchanged=True,onlyNewSlice22CPaths=True,
100 hyperdocUnchanged=(hyperdoc_head==hyperdoc['head'] and hyperdoc_status==hyperdoc['status'] and not hyperdoc_changed),
101 hyperdocHead=hyperdoc_head,hyperdocHeadAtStart=hyperdoc['head'],hyperdocClean=(hyperdoc_status==''),
102 hyperdocInspectedSourceCount=len(hyperdoc['files']),hyperdocSourceHashesUnchanged=len(hyperdoc['files'])-len(hyperdoc_changed),
103 hyperdocChangedSourceHashes=hyperdoc_changed,
104 hyperdocChangedPathsSinceStart=git('diff','--name-only',hyperdoc['head'],'HEAD',cwd=HYPERDOC).splitlines(),
105 hyperdocTouchedBySlice=False,
106 hyperdocProvenance='Observed neighbouring-repository change during interruption, after the completed offline regression; earlier audit retained in audit-before-resume.json. No writes or loads by Slice 22C.',
107 regressionProcesses=114,expectedFaultExits=108,readerDiagnostics=124,byteIdenticalBehavioralReports=159,
108 completeOriginalComparisons=4,templateObservations=7,expandedControls=3,decisionCounts=evidence['decisionCounts'],
109 statuses=expected,newRegistrations=0,allJSONParsed=json_count,localMarkdownLinks=links,
110 examinedToolHashes={p:sha(ROOT/p) for p in sources},
111 sandboxStartupError='UnsupportedOperation("writable root /Users/rgb/.nix-profile contains symlink component /Users/rgb/.nix-profile; symlinked writable roots are not supported.\nIf this writable root is at or beneath CODEX_HOME and you trust its symlink targets, set `allow_symlinked_codex_home = true` at the top level of `$CODEX_HOME/config.toml` (normally `~/.codex/config.toml`) on the execution host, then restart Codex or its executor. This opt-out trusts targets outside CODEX_HOME and targets changed between commands. It does not apply to other writable roots.")',
112 sandboxResolution='Regular require_escalated approval; canonical SBCL path and explicit child sandbox. No symlink/config edits or approval bypass.',
113 passed=True)
114 (out/'precommit-audit.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
115 print(json.dumps({k:result[k] for k in ['passed','baseTrackedFilesVerified','historicalReportsVerified','pinnedCorpusFilesVerified','decisionCounts']}))
116 if __name__=='__main__':main()