]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - scripts/generate-nfc-data.py
test(xtm): register variant item-identity merge original
[topic-maps.git] / scripts / generate-nfc-data.py
1 #!/usr/bin/env python3
2 """Generate ANSI Lisp NFC tables from Python UCD 15.1.0; offline, no runtime dependency.
3 UAX #15: canonical decomposition, combining class, non-excluded composition pairs.
4 Hangul decomposition/composition is algorithmic. Generated output is committed.
5 """
6 import pathlib
7 import unicodedata as u
8 assert u.unidata_version == '15.1.0', u.unidata_version
9 out = [';;;; Generated by scripts/generate-nfc-data.py; Unicode 15.1.0.',
10 '(in-package #:topic-maps.cxtm)']
11 def table(name, rows):
12 out.append(f'(defparameter {name} (let ((h (make-hash-table :test #\'eql)))')
13 out.append(" (dolist (entry '(")
14 out.extend(' (' + ' '.join(map(str, row)) + ')' for row in rows)
15 out.append(' )) (setf (gethash (first entry) h) (rest entry))) h))')
16 decomp=[]; ccc=[]; compose=[]
17 for n in range(0x110000):
18 c=chr(n); d=u.decomposition(c)
19 if d and not d.startswith('<'):
20 seq=[int(x,16) for x in d.split()]; decomp.append([n,*seq])
21 if len(seq)==2 and u.normalize('NFC',''.join(map(chr,seq)))==c:
22 compose.append([seq[0]*0x110000+seq[1],n])
23 if u.combining(c): ccc.append([n,u.combining(c)])
24 table('*decompositions*',decomp); table('*combining-classes*',ccc); table('*compositions*',compose)
25 ranges=[]; start=None
26 for n in range(0x110001):
27 assigned=n < 0x110000 and u.category(chr(n)) != 'Cn'
28 if assigned and start is None: start=n
29 if not assigned and start is not None: ranges.extend([start,n-1]); start=None
30 out.append('(defparameter *assigned-ranges* #(' + ' '.join(map(str,ranges)) + '))')
31 pathlib.Path(__file__).resolve().parents[1].joinpath('src/cxtm/nfc-data.lisp').write_text('\n'.join(out)+'\n')
32 print('UCD 15.1.0:', len(decomp),'decompositions,',len(ccc),'combining classes,',len(compose),'composition pairs')