]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/ctm/unicode.lisp
test: register seven negative XTM originals (slice 21C)
[topic-maps.git] / src / ctm / unicode.lisp
1 (in-package #:topic-maps.ctm)
2
3 (defun scalar-character (code position)
4 (unless (and (<= 0 code #x10ffff) (not (<= #xd800 code #xdfff)))
5 (problem 'ctm-rejection :invalid-unicode position "Not a Unicode scalar value."))
6 (or (code-char code)
7 (problem 'ctm-unsupported :character-repertoire position
8 "The Common Lisp implementation cannot represent this Unicode character.")))
9
10 (defun decode-utf8 (octets &key (position 0) (strip-bom nil))
11 "Strict UTF-8 decoding without implementation-specific external-format flags."
12 (let* ((size (length octets))
13 (i (if (and strip-bom (>= size 3)
14 (= (aref octets 0) #xef) (= (aref octets 1) #xbb)
15 (= (aref octets 2) #xbf)) 3 0)))
16 (with-output-to-string (output)
17 (loop while (< i size) do
18 (let* ((first (aref octets i))
19 (width (cond ((< first #x80) 1) ((<= #xc2 first #xdf) 2)
20 ((<= #xe0 first #xef) 3) ((<= #xf0 first #xf4) 4)
21 (t 0))))
22 (when (or (zerop width) (> (+ i width) size))
23 (problem 'ctm-rejection :invalid-utf8 (+ position i)
24 "Invalid or incomplete UTF-8 sequence." :byte))
25 (let ((code (logand first (ecase width (1 #x7f) (2 #x1f) (3 #x0f) (4 #x07)))))
26 (loop for j from 1 below width for byte = (aref octets (+ i j)) do
27 (unless (<= #x80 byte #xbf)
28 (problem 'ctm-rejection :invalid-utf8 (+ position i j)
29 "Expected UTF-8 continuation byte." :byte))
30 (setf code (+ (ash code 6) (logand byte #x3f))))
31 (unless (and (>= code (ecase width (1 0) (2 #x80) (3 #x800) (4 #x10000)))
32 (<= code #x10ffff) (not (<= #xd800 code #xdfff)))
33 (problem 'ctm-rejection :invalid-utf8 (+ position i)
34 "Overlong or invalid Unicode UTF-8 sequence." :byte))
35 (write-char (scalar-character code (+ position i)) output))
36 (incf i width))))))
37
38 (defun unescape-iri (text position)
39 "CTM 3.3.3: decode consecutive percent octets as UTF-8, preserving other chars."
40 (with-output-to-string (out)
41 (loop for i from 0 below (length text) do
42 (if (char= (char text i) #\%)
43 (let ((bytes nil) (start i))
44 (loop while (and (< i (length text)) (char= (char text i) #\%)) do
45 (unless (< (+ i 2) (length text))
46 (problem 'ctm-rejection :invalid-percent-escape (+ position i) "Incomplete IRI escape."))
47 (let ((a (digit-char-p (char text (1+ i)) 16))
48 (b (digit-char-p (char text (+ i 2)) 16)))
49 (unless (and a b)
50 (problem 'ctm-rejection :invalid-percent-escape (+ position i) "IRI escape needs two hex digits."))
51 (push (+ (* 16 a) b) bytes))
52 (incf i 3))
53 (write-string (decode-utf8 (coerce (nreverse bytes) '(vector (unsigned-byte 8)))
54 :position (+ position start)) out)
55 (decf i))
56 (write-char (char text i) out)))))
57
58
59 ;; Inspect the ASCII header before decoding (CTM 3.2). The strict UTF-8
60 ;; decoder above is unchanged; only the qualified ISO-8859-5 path is added.
61 (defun ctm-unit-code (sequence position)
62 (let ((value (aref sequence position)))
63 (if (characterp value) (char-code value) value)))
64
65 (defun ctm-ascii-prefix-p (sequence position prefix)
66 (and (<= (+ position (length prefix)) (length sequence))
67 (loop for c across prefix for i from position
68 always (= (char-code c) (ctm-unit-code sequence i)))))
69
70 (defun ctm-encoding-name (sequence position unit)
71 "Return the ASCII charset name, end after its quote, and name position.
72 CTM 3.2/3.6 permits only SP/TAB before this initial header's opening quote."
73 (let ((cursor (+ position (length "%encoding"))) (size (length sequence)))
74 (loop while (and (< cursor size) (member (ctm-unit-code sequence cursor) '(9 32))) do (incf cursor))
75 (unless (and (< cursor size) (= 34 (ctm-unit-code sequence cursor)))
76 (problem 'ctm-rejection :invalid-encoding-declaration cursor "Expected an encoding name in double quotes." unit))
77 (let ((start (incf cursor)))
78 (loop while (and (< cursor size) (/= 34 (ctm-unit-code sequence cursor))) do
79 (unless (and (<= 33 (ctm-unit-code sequence cursor) 126)
80 (/= 92 (ctm-unit-code sequence cursor)))
81 (problem 'ctm-rejection :invalid-encoding-declaration cursor "Expected an ASCII charset name." unit))
82 (incf cursor))
83 (unless (and (< cursor size) (> cursor start))
84 (problem 'ctm-rejection :invalid-encoding-declaration cursor "Encoding name must be nonempty and have a closing quote." unit))
85 (values (map 'string (lambda (value) (if (characterp value) value (code-char value)))
86 (subseq sequence start cursor)) (1+ cursor) start))))
87
88 (defun ctm-encoding-kind (name)
89 "Only the IANA identities implemented here; ASCII names are case-insensitive."
90 (cond ((member name '("UTF-8" "csUTF8") :test #'string-equal) :utf-8)
91 ((member name '("ISO-8859-5" "ISO_8859-5:1988" "ISO_8859-5"
92 "iso-ir-144" "cyrillic" "csISOLatinCyrillic")
93 :test #'string-equal) :iso-8859-5)))
94
95 (defun ctm-encoding-identity (name)
96 (or (ctm-encoding-kind name) (string-downcase name)))
97
98 (defun decode-iso-8859-5 (bytes)
99 "SBCL's existing total single-byte codec, qualified against all 256 values."
100 #+sbcl
101 (sb-ext:octets-to-string (map '(vector (unsigned-byte 8)) #'identity bytes)
102 :external-format :iso-8859-5)
103 #-sbcl
104 (problem 'ctm-unsupported :unsupported-encoding 0
105 "ISO-8859-5 decoding is qualified only on SBCL." :byte))
106
107 (defun compatible-ctm-encoding (current declared position unit)
108 (when (and current (not (equal (ctm-encoding-identity current) (ctm-encoding-identity declared))))
109 (problem 'ctm-rejection :encoding-conflict position
110 (format nil "Encoding ~S conflicts with explicitly selected ~S." declared current) unit))
111 declared)
112
113 (defun decode-ctm-octets (bytes &key encoding)
114 "Apply CTM 3.2 ordering: supplied encoding, BOM, declaration, default, decode.
115 UTF-8 and ISO-8859-5 use their small, explicit IANA identity sets."
116 (check-type encoding (or null string))
117 (let* ((bom (and (>= (length bytes) 3) (= (aref bytes 0) #xef)
118 (= (aref bytes 1) #xbb) (= (aref bytes 2) #xbf)))
119 (start (if bom 3 0)) (selected encoding) (name-position 0))
120 (when bom (setf selected (compatible-ctm-encoding selected "UTF-8" 0 :byte)))
121 (when (ctm-ascii-prefix-p bytes start "%encoding")
122 (multiple-value-bind (name end position) (ctm-encoding-name bytes start :byte)
123 (declare (ignore end))
124 (setf selected (compatible-ctm-encoding selected name position :byte)
125 name-position position)))
126 (case (ctm-encoding-kind (or selected "UTF-8"))
127 (:utf-8 (decode-utf8 bytes :strip-bom t))
128 (:iso-8859-5 (decode-iso-8859-5 bytes))
129 (otherwise
130 (problem 'ctm-unsupported :unsupported-encoding name-position
131 (format nil "Charset ~S is not implemented; supported CTM encodings are UTF-8 and ISO-8859-5." selected) :byte)))))