]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/ctm/lexer.lisp
Implement CTM integer occurrences and canonical integer CXTM
[topic-maps.git] / src / ctm / lexer.lisp
1 (in-package #:topic-maps.ctm)
2
3 (defstruct token kind value (position 0))
4 (defstruct (lexer (:constructor make-lexer (text))) text (cursor 0))
5
6 (defun peek-character (lexer &optional (delta 0))
7 (let ((i (+ (lexer-cursor lexer) delta)))
8 (when (< i (length (lexer-text lexer))) (char (lexer-text lexer) i))))
9 (defun take-character (lexer)
10 (prog1 (peek-character lexer) (incf (lexer-cursor lexer))))
11 (defun whitespace-p (c)
12 (and c (find (char-code c) '(#x20 #x0d #x0a #x09))))
13 (defun name-start-p (c)
14 (and c
15 (or (ascii-letter-p c) (char= c #\_)
16 (let ((n (char-code c)))
17 (some (lambda (r) (<= (first r) n (second r)))
18 '((#xc0 #xd6) (#xd8 #xf6) (#xf8 #x2ff) (#x370 #x37d)
19 (#x37f #x1fff) (#x200c #x200d) (#x2070 #x218f) (#x2c00 #x2fef)
20 (#x3001 #xd7ff) (#xf900 #xfdcf) (#xfdf0 #xfffd) (#x10000 #xeffff)))))))
21 (defun name-part-p (c)
22 (and c (or (name-start-p c) (find c "0123456789-") (= (char-code c) #xb7)
23 (<= #x300 (char-code c) #x36f) (<= #x203f (char-code c) #x2040))))
24
25 (defun skip-layout (lexer)
26 (loop
27 (cond ((whitespace-p (peek-character lexer)) (take-character lexer))
28 ((eql #\# (peek-character lexer))
29 (if (eql #\( (peek-character lexer 1))
30 (progn
31 (incf (lexer-cursor lexer) 2)
32 (loop with level = 1 while (plusp level) do
33 (cond ((null (peek-character lexer))
34 (problem 'ctm-rejection :unterminated-comment (lexer-cursor lexer) "Unterminated block comment."))
35 ((and (eql #\# (peek-character lexer)) (eql #\( (peek-character lexer 1)))
36 (incf level) (incf (lexer-cursor lexer) 2))
37 ((and (eql #\) (peek-character lexer)) (eql #\# (peek-character lexer 1)))
38 (decf level) (incf (lexer-cursor lexer) 2))
39 (t (take-character lexer)))))
40 (loop while (and (peek-character lexer)
41 (not (find (peek-character lexer) '(#\Newline #\Return))))
42 do (take-character lexer))))
43 (t (return)))))
44
45 (defun read-ctm-string (lexer start)
46 (when (and (eql #\" (peek-character lexer)) (eql #\" (peek-character lexer 1)))
47 (problem 'ctm-unsupported :triple-quoted-string start "Triple quoted literals are outside Slice 1."))
48 (with-output-to-string (out)
49 (loop for c = (take-character lexer) do
50 (cond ((null c) (problem 'ctm-rejection :unterminated-string start "Unterminated quoted string."))
51 ((char= c #\") (return))
52 ((char= c #\\)
53 (let ((escape (take-character lexer)))
54 (case escape
55 (#\n (write-char #\Newline out)) (#\r (write-char #\Return out))
56 (#\t (write-char #\Tab out)) (#\" (write-char #\" out)) (#\\ (write-char #\\ out))
57 ((#\u #\U)
58 (let ((code 0))
59 (dotimes (i (if (char= escape #\u) 4 6))
60 (let* ((digit (take-character lexer)) (value (and digit (digit-char-p digit 16))))
61 (unless value
62 (problem 'ctm-rejection :invalid-unicode-escape (1- (lexer-cursor lexer)) "Expected hexadecimal escape digit."))
63 (setf code (+ (* code 16) value))))
64 (write-char (scalar-character code start) out)))
65 (otherwise (problem 'ctm-rejection :invalid-string-escape (1- (lexer-cursor lexer)) "Unknown string escape.")))))
66 (t (write-char c out))))))
67
68 (defun scan-name (lexer)
69 (loop
70 (cond ((name-part-p (peek-character lexer)) (take-character lexer))
71 ((eql #\. (peek-character lexer))
72 ;; Dots belong to an identifier only if followed by a name-part.
73 (let ((delta 0))
74 (loop while (eql #\. (peek-character lexer delta)) do (incf delta))
75 (if (name-part-p (peek-character lexer delta))
76 (incf (lexer-cursor lexer) delta)
77 (return))))
78 (t (return)))))
79
80 (defun excluded-literal-p (text)
81 "Recognize unsupported numeric/date literal syntax, without constructing values."
82 (labels ((digits (s) (and (plusp (length s)) (every (lambda (c) (find c "0123456789")) s)))
83 (numeric (s)
84 (let* ((s (if (and (plusp (length s)) (find (char s 0) "+-")) (subseq s 1) s))
85 (dot (position #\. s)))
86 (if dot (and (or (zerop dot) (digits (subseq s 0 dot))) (digits (subseq s (1+ dot)))) (digits s))))
87 (date (s)
88 (let* ((s (if (and (plusp (length s)) (char= (char s 0) #\-)) (subseq s 1) s))
89 (dash (position #\- s)))
90 (and dash (>= dash 4) (digits (subseq s 0 dash))
91 (= (length s) (+ dash 6)) (char= (char s (+ dash 3)) #\-)
92 (digits (subseq s (1+ dash) (+ dash 3))) (digits (subseq s (+ dash 4)))
93 (<= 1 (parse-integer s :start (1+ dash) :end (+ dash 3)) 12)
94 (<= 1 (parse-integer s :start (+ dash 4)) 31))))
95 (time-literal (s)
96 (and (>= (length s) 8) (digits (subseq s 0 2)) (char= (char s 2) #\:)
97 (digits (subseq s 3 5)) (char= (char s 5) #\:) (digits (subseq s 6 8))
98 (let* ((rest (subseq s 8))
99 (end (position-if (lambda (c) (find c "Z+-")) rest))
100 (fraction (subseq rest 0 end)) (zone (if end (subseq rest end) "")))
101 (and (or (zerop (length fraction))
102 (and (char= (char fraction 0) #\.) (digits (subseq fraction 1))))
103 (or (member zone '("" "Z") :test #'string=)
104 (and (= 6 (length zone)) (find (char zone 0) "+-")
105 (digits (subseq zone 1 3)) (char= (char zone 3) #\:)
106 (digits (subseq zone 4 6)))))))))
107 (or (numeric text) (date text) (string= text "*")
108 (let ((tee (position #\T text))) (and tee (date (subseq text 0 tee)) (time-literal (subseq text (1+ tee))))))))
109
110 (defun read-number-token (lexer start)
111 "One lexical path for supported integers and recognized excluded number/date forms.
112 A dot followed by a digit can belong to a decimal; a terminal dot stays punctuation."
113 (setf (lexer-cursor lexer) start)
114 (loop for c = (peek-character lexer)
115 while (and c (not (whitespace-p c)) (not (find c ";@~()#^<>\""))
116 (or (not (char= c #\.))
117 (and (peek-character lexer 1) (find (peek-character lexer 1) "0123456789"))))
118 do (take-character lexer))
119 (let ((value (subseq (lexer-text lexer) start (lexer-cursor lexer))))
120 (cond ((topic-maps.tmdm:integer-lexical-p value)
121 (make-token :kind :integer :value value :position start))
122 ((excluded-literal-p value) (make-token :kind :excluded-literal :value value :position start))
123 (t (problem 'ctm-rejection :invalid-occurrence-literal start "Malformed numeric/date literal.")))))
124
125 (defun next-token (lexer)
126 (skip-layout lexer)
127 (let* ((start (lexer-cursor lexer)) (c (take-character lexer))
128 (text (lexer-text lexer)))
129 (cond
130 ((null c) (make-token :kind :eof :position start))
131 ((or (find c "0123456789+*")
132 (and (find c "-.") (peek-character lexer) (find (peek-character lexer) "0123456789"))
133 (and (char= c #\-) (eql #\. (peek-character lexer))
134 (peek-character lexer 1) (find (peek-character lexer 1) "0123456789")))
135 (read-number-token lexer start))
136 ((char= c #\") (make-token :kind :string :value (read-ctm-string lexer start) :position start))
137 ((char= c #\<)
138 (loop while (and (peek-character lexer) (not (eql #\> (peek-character lexer))))
139 do (take-character lexer))
140 (unless (peek-character lexer)
141 (problem 'ctm-rejection :unterminated-iri start "Expected closing >."))
142 (let ((value (subseq text (1+ start) (lexer-cursor lexer))))
143 (take-character lexer)
144 (when (zerop (length value))
145 (problem 'ctm-rejection :empty-iri start "Delimited IRI must not be empty (CTM 3.3.3)."))
146 (validate-iri-characters value (1+ start))
147 (make-token :kind :iri :value value :position start)))
148 ((name-start-p c)
149 (scan-name lexer)
150 (let ((head (subseq text start (lexer-cursor lexer))))
151 (cond
152 ((and (scheme-p head) (eql #\: (peek-character lexer))
153 (eql #\/ (peek-character lexer 1)) (eql #\/ (peek-character lexer 2)))
154 (incf (lexer-cursor lexer) 3)
155 (loop while (and (peek-character lexer) (not (whitespace-p (peek-character lexer)))
156 (not (find (peek-character lexer) "<>\"{}|^`\\[]()")))
157 do (take-character lexer))
158 (loop while (and (> (lexer-cursor lexer) start)
159 (find (char text (1- (lexer-cursor lexer))) ";.)"))
160 do (decf (lexer-cursor lexer)))
161 (make-token :kind :iri :value (subseq text start (lexer-cursor lexer)) :position start))
162 ((and (eql #\: (peek-character lexer))
163 (or (name-start-p (peek-character lexer 1))
164 (and (peek-character lexer 1)
165 (find (peek-character lexer 1) "0123456789"))))
166 ;; Only an immediate local part forms a QName (CTM 3.3.4).
167 ;; Otherwise ':' remains a separate token, including name types
168 ;; followed by a string with or without intervening layout.
169 (take-character lexer)
170 (if (name-start-p (peek-character lexer))
171 (scan-name lexer)
172 ;; CTM 3.3.4 production [11]: numeric-leading local parts
173 ;; contain name-part characters, which do not include dots.
174 (loop while (name-part-p (peek-character lexer)) do (take-character lexer)))
175 (make-token :kind :qname :value (subseq text start (lexer-cursor lexer)) :position start))
176 (t (make-token :kind :identifier :value head :position start)))))
177 ((find c ".;-=^:(),") (make-token :kind :punctuation :value c :position start))
178 ((char= c #\%)
179 ;; Production [71] has a literal terminal followed by an identifier;
180 ;; unlike Role separators it does not require intervening whitespace.
181 (if (starts-with "%prefix" (subseq text start))
182 (progn (setf (lexer-cursor lexer) (+ start 7))
183 (make-token :kind :directive :value "prefix" :position start))
184 (progn
185 (scan-name lexer)
186 (problem 'ctm-unsupported :unsupported-directive start
187 (format nil "Directive %~A is outside the implemented subset."
188 (subseq text (1+ start) (lexer-cursor lexer)))))))
189 ((find c "?[$@~")
190 (problem 'ctm-unsupported :unsupported-construct start
191 (format nil "Construct starting with ~S is outside the implemented subset." c)))
192 (t (problem 'ctm-rejection :unexpected-character start (format nil "Unexpected character ~S." c))))))