]>
dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/generate-nfc-data.py
2 """Generate ANSI Lisp NFC tables from Python UCD 15.1.0; offline, no runtime dependency.
3 UAX #15: canonical decomposition, combining class, non-excluded composition pairs.
4 Hangul decomposition/composition is algorithmic. Generated output is committed.
7 import unicodedata
as u
8 assert u
.unidata_version
== '15.1.0', u
.unidata_version
9 out
= [';;;; Generated by scripts/generate-nfc-data.py; Unicode 15.1.0.',
10 '(in-package #:topic-maps.cxtm)']
11 def table(name
, rows
):
12 out
.append(f
'(defparameter {name} (let ((h (make-hash-table :test #\'eql)))')
13 out
.append(" (dolist (entry '(")
14 out
.extend(' (' + ' '.join(map(str, row
)) + ')' for row
in rows
)
15 out
.append(' )) (setf (gethash (first entry) h) (rest entry))) h))')
16 decomp
=[]; ccc
=[]; compose
=[]
17 for n
in range(0x110000):
18 c
=chr(n
); d
=u
.decomposition(c
)
19 if d
and not d
.startswith('<'):
20 seq
=[int(x
,16) for x
in d
.split()]; decomp
.append([n
,*seq
])
21 if len(seq
)==2 and u
.normalize('NFC',''.join(map(chr,seq
)))==c
:
22 compose
.append([seq
[0]*0x110000+seq
[1],n
])
23 if u
.combining(c
): ccc
.append([n
,u
.combining(c
)])
24 table('*decompositions*',decomp
); table('*combining-classes*',ccc
); table('*compositions*',compose
)
26 for n
in range(0x110001):
27 assigned
=n
< 0x110000 and u
.category(chr(n
)) != 'Cn'
28 if assigned
and start
is None: start
=n
29 if not assigned
and start
is not None: ranges
.extend([start
,n
-1]); start
=None
30 out
.append('(defparameter *assigned-ranges* #(' + ' '.join(map(str,ranges
)) + '))')
31 pathlib
.Path(__file__
).resolve().parents
[1].joinpath('src/cxtm/nfc-data.lisp').write_text('\n'.join(out
)+'\n')
32 print('UCD 15.1.0:', len(decomp
),'decompositions,',len(ccc
),'combining classes,',len(compose
),'composition pairs')