2 """Assemble documentary evidence from fresh observations, never register results.
3 The XML comparison view below is report-only: it does not construct native maps,
4 feed the reader, or participate in the conformance executor.
6 import argparse
, difflib
, hashlib
, json
, re
7 from pathlib
import Path
8 import xml
.etree
.ElementTree
as ET
9 from urllib
.parse
import urljoin
10 ROOT
=Path(__file__
).resolve().parents
[2]
11 HYPERDOC
=Path('/Users/rgb/workspace/hyperdoc-dreyeck-ch')
12 def read(p
):return json
.loads(p
.read_text())
13 def save(p
,v
):p
.write_text(json
.dumps(v
,ensure_ascii
=False,indent
=2)+'\n')
14 def sha(b
):return hashlib
.sha256(b
).hexdigest()
16 b
=p
.read_bytes();return dict(path
=str(p
.relative_to(ROOT
)),bytes
=len(b
),sha256
=sha(b
),hex=b
.hex(),text
=b
.decode('utf-8'))
17 def tree(e
):return dict(element
=e
.tag
,attributes
=dict(e
.attrib
),text
=e
.text
or '',children
=[tree(c
) for c
in e
])
18 def identifiers(e
):return {(k
,x
.text
) for k
in ['itemIdentifiers','subjectIdentifiers','subjectLocators'] for x
in e
.findall(k
+'/locator')}
19 def compare_models(actual
,expected
,native
,doc
):
20 # Match by shared identifiers; canonical topic ordinals are not identities.
21 a
,b
=ET
.fromstring(actual
),ET
.fromstring(expected
)
22 at
,bt
=a
.findall('topic'),b
.findall('topic');assert len(at
)==len(bt
)==native
['topicCount']
23 keys_a
={};keys_b
={};alignment
=[]
25 candidates
=[u
for u
in bt
if identifiers(t
)&identifiers(u
)]
26 assert len(candidates
)==1
27 u
=candidates
[0];anchor
=sorted(identifiers(t
)&identifiers(u
))[0];key
=':'.join(anchor
)
28 keys_a
[t
.get('number')]=key
;keys_b
[u
.get('number')]=key
29 alignment
.append(dict(anchor
=list(anchor
),actualTopicNumber
=t
.get('number'),expectedTopicNumber
=u
.get('number')))
30 assert len(keys_b
)==len(bt
)
31 def normalized(e
,keys
):
32 attrs
={k
:(keys
[v
] if k
in ['topicref','reifier'] else v
) for k
,v
in e
.attrib
.items() if k
!='number'}
33 kids
=[normalized(c
,keys
) for c
in e
]
34 if e
.tag
=='topicMap':kids
=sorted(kids
,key
=lambda x
:json
.dumps(x
.get('identity','')))
35 result
=dict(element
=e
.tag
,attributes
=attrs
,text
=e
.text
if not len(e
) else '',children
=kids
)
36 if e
.tag
=='topic':result
['identity']=keys
[e
.get('number')]
38 aa
,bb
=normalized(a
,keys_a
),normalized(b
,keys_b
)
39 # Independently reconcile every observed native topic/name/occurrence with
40 # its actual CXTM element, resolving only the relative locator forms here.
42 for t
in native
['topics']:
43 ids
={(k
,v
) for k
in ['itemIdentifiers','subjectIdentifiers','subjectLocators'] for v
in t
[k
]}
44 matches
=[e
for e
in at
if ids
=={(k
,urljoin(doc
,v
)) for k
,v
in identifiers(e
)}]
45 assert len(matches
)==1
46 native_keys
[t
['ordinal']]=keys_a
[matches
[0].get('number')]
47 for t
in native
['topics']:
48 e
=next(e
for e
in at
if keys_a
[e
.get('number')]==native_keys
[t
['ordinal']])
49 for kind
,field
in [('name','names'),('occurrence','occurrences')]:
50 xs
=e
.findall(kind
);assert len(xs
)==len(t
[field
])
51 for record
,x
in zip(t
[field
],xs
):
52 assert record
['parent']==t
['ordinal'] and record
['value']==x
.findtext('value')
53 assert keys_a
[x
.find('type').get('topicref')]==native_keys
[record
['type']]
54 assert sorted(native_keys
[n
] for n
in record
['scope'])==sorted(keys_a
[s
.get('topicref')] for s
in x
.findall('scope/scopingTopic'))
55 if kind
=='occurrence':assert record
['datatype']==x
.findtext('datatype')
56 else:assert record
['variantCount']==len(x
.findall('variant'))==0
57 assert len(a
.findall('association'))==native
['associationCount']==0
58 def walk(x
,y
,path
=''):
60 if isinstance(x
,dict) and isinstance(y
,dict):return sum((walk(x
.get(k
),y
.get(k
),path
+'/'+k
) for k
in sorted(x
.keys()|y
.keys())),[])
61 if isinstance(x
,list) and isinstance(y
,list) and len(x
)==len(y
):return sum((walk(v
,w
,path
+'/'+str(i
)) for i
,(v
,w
) in enumerate(zip(x
,y
))),[])
62 return [dict(path
=path
,actual
=x
,expected
=y
)]
63 return dict(kind
='report-only-XML-view-not-a-replacement-TMDM',topicAlignment
=alignment
,
64 nativePropertiesMatchActualCXTM
=True,actual
=aa
,expected
=bb
,differences
=walk(aa
,bb
))
65 def render_appendix(rows
):
66 lines
=['# Slice 22C — Originale, Baselines und tatsächliche Beobachtungen', '',
67 'Generiert aus der erneut geprüften fallbezogenen Evidenz. [Entscheidungen und Editionsgrenzen](slice-22c-normative-decisions.md); [vollständige JSON-Evidenz](../reports/slice-22c/case-evidence.json).', '',
68 'Quellbytes, Hashes und Prozessresultate sind `observed`; grammatische Rollen sind `derived`, historische Argumentabsichten `inferred`. Zeichenpositionen sind nullbasiert, Zeile/Spalte einsbasiert. XTM-Positionen bleiben die ungefähr gemeldeten SAX-Positionen. Die Lesefassung blendet Leerraum am Zeilenende aus; Text und Hexwerte im JSON erhalten sämtliche Originalbytes einschließlich BOM und fehlendem Schluss-LF.', '']
70 lines
.extend([label
+': `'+b
['path']+'`, '+str(b
['bytes'])+' Bytes; SHA-256 `'+b
['sha256']+'`.', '',
71 '```'+('xml' if (b
['path'].endswith(('.xtm','.cxtm')) or '/xtm2/' in b
['path']) else 'text'),b
['text'].rstrip('\n'),'```',''])
72 def structured(label
,obj
):
73 lines
.extend([label
,'','```json',json
.dumps(obj
,ensure_ascii
=False,indent
=2),'```',''])
75 lines
.extend(['## '+c
['id'],'',
76 'Vorher: '+c
['priorClassification']+' / '+c
['priorDecision']+'. Neue Entscheidung: **'+c
['decision']['classification']+'**. Registrierung unverändert: `'+c
['registeredAfter']+'`.', '',
77 'Document IRI: `'+c
['documentIRI']+'`.', ''])
78 blob('Original',c
['source'])
79 lines
.extend(['Quellabschluss: '+('LF vorhanden.' if c
['source']['text'].endswith('\n') else 'kein abschließendes LF.')+' UTF-8-BOM: '+('ja.' if c
['source']['hex'].startswith('efbbbf') else 'nein.'),''])
80 if c
['baseline']:blob('Unveränderte Baseline',c
['baseline'])
81 else:lines
.extend(['Keine positive CXTM-Baseline für diesen negativen Syntaxvertrag.',''])
82 structured('Tatsächliche Condition (einschließlich Code, Position und Cause):',c
['actualCondition'])
84 lines
.extend(['Ist-CXTM: '+str(c
['actualCXTM']['bytes'])+' Bytes, SHA-256 `'+c
['actualCXTM']['sha256']+'`. Vollständiger Bytevergleich: **nicht identisch**. Erste Differenz: '+str(c
['comparison']['firstDifferentByte'])+'.','','```xml',c
['actualCXTM']['text'].rstrip('\n'),'```',''])
85 structured('Verbleibende Modelldifferenzen nach Identitätszuordnung:',c
['completeModelComparison']['differences'])
86 else:lines
.extend(['Kein natives Original-Ergebnismodell und keine Original-Ist-CXTM-Ausgabe; keine Ersatzmap konstruiert.',''])
87 if c
.get('templateAnalysis'):
88 structured('Erneut geprüfte Variablenpositionen, Deklaration, Aufrufe und Bindungen:',c
['templateAnalysis'])
89 for control
in c
['expandedControls']:
90 lines
.extend(['Ausgeschriebene Gegenprobe (kein Original-PASS): '+str(len(control
['actualCXTM'].encode()))+'/'+str(c
['baseline']['bytes'])+' Bytes identisch; Condition null.',''])
91 if c
.get('intendedNegativeCause'):structured('Historisch beabsichtigte, tatsächlich nicht erreichte Argumentprüfung:',c
['intendedNegativeCause'])
92 for a
in c
['auxiliarySources']:blob('Unveränderte Hilfsquelle',a
)
93 if c
.get('sourceReferenceGraph'):structured('Tatsächlicher Ressourcen-Referenzgraph:',c
['sourceReferenceGraph'])
94 if c
.get('auxiliaryObservations'):structured('Separate Hilfsquellenbeobachtung, keine Originalregistrierung:',c
['auxiliaryObservations'])
95 structured('Neue normative Entscheidung und offene Primärquellen:',c
['decision'])
96 return '\n'.join(line
.rstrip() for line
in '\n'.join(lines
).splitlines()).rstrip()+'\n'
98 ap
=argparse
.ArgumentParser(description
=__doc__
);ap
.add_argument('--execution',type=Path
,required
=True);ap
.add_argument('--regression',type=Path
,required
=True);ap
.add_argument('--output',type=Path
,required
=True)
99 args
=ap
.parse_args();out
=args
.output
.resolve();assert not out
.is_relative_to(ROOT
) and not out
.is_relative_to(HYPERDOC
);out
.mkdir(parents
=True,exist_ok
=True)
100 matrix
=read(ROOT
/'reports/slice-22a/matrix.json')
101 old
=[c
for c
in matrix
['cases'] if c
['id'].startswith('ctm/') or c
['id'] in ['xtm2/in/mergemap-itemid.xtm','xtm2/in/mergemap-loop.xtm']]
103 ctm
={c
['id']:c
for c
in read(args
.execution
/'investigation/ctm-observations.json')['cases']}
104 xtm
={c
['id']:c
for c
in read(args
.regression
/'after-diagnostics.json')}
105 states
={c
['id']:c
['status'] for c
in read(args
.regression
/'ctm.validation.json')['cases']}
106 registered
={c
['id'] for c
in read(args
.regression
/'xtm.json')['cases']}
107 controls
=read(args
.execution
/'investigation/controls.json');rows
=[]
108 decisions
=read(Path(__file__
).parent
/'decisions.json')
110 id=prior
['id'];obs
=ctm
.get(id,xtm
.get(id));assert obs
==prior
['observation'],id
111 source
=file_data(ROOT
/prior
['source']['path']);assert source
==prior
['source']
112 baseline
=prior
['expected']['baseline']
113 if baseline
:assert file_data(ROOT
/baseline
['path'])==baseline
114 status
=states
.get(id,'not-run');assert status
==prior
['registeredStatus']
115 if id.startswith('xtm2/'):assert id not in registered
116 actual
=obs
.get('actualCXTM');model
=obs
.get('model')
117 condition
=obs
['condition']
118 row
=dict(id=id,epistemicStatus
='observed',category
=prior
['category'],registeredBefore
=status
,registeredAfter
=status
,
119 source
=source
,documentIRI
=prior
['documentIRI'],baseline
=baseline
,actualCondition
=condition
,nativeModel
=model
,
120 observation
=obs
,priorClassification
=prior
['primaryClassification'],priorDecision
=prior
['normativeDecisionABC'],
121 priorJustification
=prior
['normativeJustificationStatus'],decision
=decisions
[id],
122 actualCXTM
=None if actual
is None else dict(bytes
=len(actual
.encode()),sha256
=sha(actual
.encode()),hex=actual
.encode().hex(),text
=actual
),
123 comparison
=dict(executed
=actual
is not None and baseline
is not None,byteExact
=None if actual
is None or baseline
is None else actual
==baseline
['text'],
124 firstDifferentByte
=next((i
for i
,(a
,b
) in enumerate(zip(actual
.encode(),bytes
.fromhex(baseline
['hex']))) if a
!=b
),min(len(actual
.encode()),baseline
['bytes']) if len(actual
.encode())!=baseline
['bytes'] else None) if actual
is not None and baseline
else None),
125 nativeModelUnavailableReason
=None if model
else 'Reader rejected before returning an original map; no replacement map constructed.')
126 if actual
is not None and baseline
:
127 row
['completeModelComparison']=compare_models(actual
,baseline
['text'],model
,prior
['documentIRI'])
128 diff
=''.join(difflib
.unified_diff(baseline
['text'].splitlines(True),actual
.splitlines(True),fromfile
='unchanged-original-baseline',tofile
='fresh-actual'))
129 (out
/(Path(id).name
+'.diff')).write_text(diff
)
130 (out
/(Path(id).name
+'.actual.cxtm')).write_bytes(actual
.encode())
131 row
['completeCXTMUnifiedDiff']=diff
132 if prior
.get('templateAnalysis') is not None:
133 row
['templateAnalysis']=prior
['templateAnalysis'];row
['intendedNegativeCause']=prior
['intendedNegativeCause']
134 variables
=prior
['templateAnalysis']['variables']
135 matches
=list(re
.finditer(r
'\$[A-Za-z_][A-Za-z_0-9]*',source
['text']))
136 assert [(m
.group(),m
.start()) for m
in matches
]==[(v
['variable'],v
['position']) for v
in variables
]
138 prefix
=source
['text'][:v
['position']]
139 assert v
['line']==prefix
.count('\n')+1 and v
['column']==len(prefix
.rsplit('\n',1)[-1])+1
140 row
['variablePositionsReverifiedFromOriginal']=True
141 row
['templateAnalysisProvenance']={'positions':'observed: reverified from unchanged original text','roles':'derived: manually reviewed against source syntax and cited grammar','historicalArgumentIntent':'inferred from original argument markers and comments; not an executed argument check'}
142 matches
=[c
for c
in controls
if c
.get('originalControlOnly')==id]
143 row
['expandedControls']=matches
144 for c
in matches
:assert c
['byteExact'] and c
['condition'] is None and c
['actualCXTM']==baseline
['text']
147 for request
in obs
['requests']:auxiliaries
.append(file_data(ROOT
/'vendor/cxtm'/request
['sourceKey']))
148 elif id.startswith('xtm2/'):
149 auxiliaries
.append(file_data(ROOT
/prior
['auxiliarySource']['path']))
151 for f
in [source
,*auxiliaries
]:
152 root
=ET
.fromstring(f
['text']);doc
=urljoin(prior
['documentIRI'],Path(f
['path']).name
)
153 for e
in root
.findall('{http://www.topicmaps.org/xtm/}mergeMap'):
154 refs
.append(dict(fromDocumentIRI
=doc
,href
=e
.get('href'),toDocumentIRI
=urljoin(doc
,e
.get('href'))))
155 row
['sourceReferenceGraph']=refs
156 row
['auxiliaryObservations']=[x
for x
in read(args
.execution
/'investigation/xtm-auxiliary.json') if x
['id']==prior
['auxiliarySource']['path'].removeprefix('vendor/cxtm/')]
157 row
['auxiliarySources']=auxiliaries
159 counts
={k
:sum(c
['decision']['classification']==k
for c
in rows
) for k
in 'ABC'}
160 save(out
/'case-evidence.json',dict(kind
='slice-22c-independent-normative-decisions-no-registration',baseCommit
='bc9dcfb02f945422970d65266dc66b991806777d',
161 matrixSHA256
=sha((ROOT
/'reports/slice-22a/matrix.json').read_bytes()),closureDeltaSHA256
=sha((ROOT
/'reports/slice-22b/after/closure-delta.json').read_bytes()),
162 cases
=rows
,decisionCounts
=counts
,statusChanges
=0,newPASS
=0))
163 (out
/'slice-22c-original-evidence.md').write_text(render_appendix(rows
))
164 print(json
.dumps(dict(cases
=len(rows
),decisions
=counts
,output
=str(out
))))
165 if __name__
=='__main__':main()