X-Git-Url: https://dreyeck.freedombox.rocks/gitweb/topic-maps.git/blobdiff_plain/2ad41cd1a945a4ce97824464de40646bb899259e..refs/heads/codex/xtm-slice-22r:/src/ctm/lexer.lisp diff --git a/src/ctm/lexer.lisp b/src/ctm/lexer.lisp index 66a5ed8..2cb1fec 100644 --- a/src/ctm/lexer.lisp +++ b/src/ctm/lexer.lisp @@ -1,6 +1,6 @@ (in-package #:topic-maps.ctm) -(defstruct token kind value (position 0)) +(defstruct token kind value (position 0) end) (defstruct (lexer (:constructor make-lexer (text))) text (cursor 0)) (defun peek-character (lexer &optional (delta 0)) @@ -43,27 +43,37 @@ (t (return))))) (defun read-ctm-string (lexer start) - (when (and (eql #\" (peek-character lexer)) (eql #\" (peek-character lexer 1))) - (problem 'ctm-unsupported :triple-quoted-string start "Triple quoted literals are outside Slice 1.")) - (with-output-to-string (out) - (loop for c = (take-character lexer) do - (cond ((null c) (problem 'ctm-rejection :unterminated-string start "Unterminated quoted string.")) - ((char= c #\") (return)) - ((char= c #\\) - (let ((escape (take-character lexer))) - (case escape - (#\n (write-char #\Newline out)) (#\r (write-char #\Return out)) - (#\t (write-char #\Tab out)) (#\" (write-char #\" out)) (#\\ (write-char #\\ out)) - ((#\u #\U) - (let ((code 0)) - (dotimes (i (if (char= escape #\u) 4 6)) - (let* ((digit (take-character lexer)) (value (and digit (digit-char-p digit 16)))) - (unless value - (problem 'ctm-rejection :invalid-unicode-escape (1- (lexer-cursor lexer)) "Expected hexadecimal escape digit.")) - (setf code (+ (* code 16) value)))) - (write-char (scalar-character code start) out))) - (otherwise (problem 'ctm-rejection :invalid-string-escape (1- (lexer-cursor lexer)) "Unknown string escape."))))) - (t (write-char c out)))))) + ;; NEXT-TOKEN has consumed the first quote. Triple strings share the exact + ;; same escape/scalar path; only their opening and closing delimiters differ. + (let ((triple (and (eql #\" (peek-character lexer)) + (eql #\" (peek-character lexer 1))))) + (when triple (incf (lexer-cursor lexer) 2)) + (with-output-to-string (out) + (loop for c = (take-character lexer) do + (cond ((null c) + (problem 'ctm-rejection :unterminated-string start + (if triple "Unterminated triple-quoted string." "Unterminated quoted string."))) + ((char= c #\") + (cond ((not triple) (return)) + ((and (eql #\" (peek-character lexer)) + (eql #\" (peek-character lexer 1))) + (incf (lexer-cursor lexer) 2) (return)) + (t (write-char c out)))) + ((char= c #\\) + (let ((escape (take-character lexer))) + (case escape + (#\n (write-char #\Newline out)) (#\r (write-char #\Return out)) + (#\t (write-char #\Tab out)) (#\" (write-char #\" out)) (#\\ (write-char #\\ out)) + ((#\u #\U) + (let ((code 0)) + (dotimes (i (if (char= escape #\u) 4 6)) + (let* ((digit (take-character lexer)) (value (and digit (digit-char-p digit 16)))) + (unless value + (problem 'ctm-rejection :invalid-unicode-escape (1- (lexer-cursor lexer)) "Expected hexadecimal escape digit.")) + (setf code (+ (* code 16) value)))) + (write-char (scalar-character code start) out))) + (otherwise (problem 'ctm-rejection :invalid-string-escape (1- (lexer-cursor lexer)) "Unknown string escape."))))) + (t (write-char c out))))))) (defun scan-name (lexer) (loop @@ -77,38 +87,15 @@ (return)))) (t (return))))) -(defun excluded-literal-p (text) - "Recognize unsupported dateTime/* literal syntax, without constructing values." - (labels ((digits (s) (and (plusp (length s)) (every (lambda (c) (find c "0123456789")) s))) - (date (s) - (let* ((s (if (and (plusp (length s)) (char= (char s 0) #\-)) (subseq s 1) s)) - (dash (position #\- s))) - (and dash (>= dash 4) (digits (subseq s 0 dash)) - (= (length s) (+ dash 6)) (char= (char s (+ dash 3)) #\-) - (digits (subseq s (1+ dash) (+ dash 3))) (digits (subseq s (+ dash 4))) - (<= 1 (parse-integer s :start (1+ dash) :end (+ dash 3)) 12) - (<= 1 (parse-integer s :start (+ dash 4)) 31)))) - (time-literal (s) - (and (>= (length s) 8) (digits (subseq s 0 2)) (char= (char s 2) #\:) - (digits (subseq s 3 5)) (char= (char s 5) #\:) (digits (subseq s 6 8)) - (let* ((rest (subseq s 8)) - (end (position-if (lambda (c) (find c "Z+-")) rest)) - (fraction (subseq rest 0 end)) (zone (if end (subseq rest end) ""))) - (and (or (zerop (length fraction)) - (and (char= (char fraction 0) #\.) (digits (subseq fraction 1)))) - (or (member zone '("" "Z") :test #'string=) - (and (= 6 (length zone)) (find (char zone 0) "+-") - (digits (subseq zone 1 3)) (char= (char zone 3) #\:) - (digits (subseq zone 4 6))))))))) - (or (string= text "*") - (let ((tee (position #\T text))) (and tee (date (subseq text 0 tee)) (time-literal (subseq text (1+ tee)))))))) +(defun ctm-integer-literal-p (text) + (string= text "*")) (defun read-number-token (lexer start) - "One token path for Integer, Decimal and Date, plus recognized excluded dateTime/* forms. + "One token path for Integer, Decimal, Date, DateTime and the exact * literal. A dot followed by a digit can belong to a decimal; a terminal dot stays punctuation." (setf (lexer-cursor lexer) start) (loop for c = (peek-character lexer) - while (and c (not (whitespace-p c)) (not (find c ";@~()#^<>\"")) + while (and c (not (whitespace-p c)) (not (find c ";@~()#^<>\"[]")) (or (not (char= c #\.)) (and (peek-character lexer 1) (find (peek-character lexer 1) "0123456789")))) do (take-character lexer)) @@ -117,7 +104,11 @@ A dot followed by a digit can belong to a decimal; a terminal dot stays punctuat (make-token :kind :integer :value value :position start)) ((and (find #\. value) (topic-maps.tmdm:decimal-lexical-p value)) (make-token :kind :decimal :value value :position start)) - ((excluded-literal-p value) (make-token :kind :excluded-literal :value value :position start)) + ((position #\T value) + (multiple-value-bind (parts reason) (topic-maps.tmdm::datetime-parts value) + (unless parts (problem 'ctm-rejection reason start "Expected a valid XSD 1.0 dateTime.")) + (make-token :kind :datetime :value value :position start))) + ((ctm-integer-literal-p value) (make-token :kind :ctm-integer :value value :position start)) ((position #\- value :start (if (and (plusp (length value)) (find (char value 0) "+-")) 1 0)) (multiple-value-bind (a date-end end reason) (topic-maps.tmdm::date-lexical-bounds value) (unless (and a (= date-end end)) @@ -138,6 +129,14 @@ A dot followed by a digit can belong to a decimal; a terminal dot stays punctuat (peek-character lexer 1) (find (peek-character lexer 1) "0123456789"))) (read-number-token lexer start)) ((char= c #\") (make-token :kind :string :value (read-ctm-string lexer start) :position start)) + ((char= c #\?) + ;; A named wildcard requires an immediately adjacent identifier. + (let ((name-start (lexer-cursor lexer))) + (when (name-start-p (peek-character lexer)) (take-character lexer) (scan-name lexer)) + (make-token :kind :wildcard + :value (when (> (lexer-cursor lexer) name-start) + (subseq text name-start (lexer-cursor lexer))) + :position start))) ((char= c #\<) (loop while (and (peek-character lexer) (not (eql #\> (peek-character lexer)))) do (take-character lexer)) @@ -178,19 +177,23 @@ A dot followed by a digit can belong to a decimal; a terminal dot stays punctuat (loop while (name-part-p (peek-character lexer)) do (take-character lexer))) (make-token :kind :qname :value (subseq text start (lexer-cursor lexer)) :position start)) (t (make-token :kind :identifier :value head :position start))))) - ((find c ".;-=^:(),") (make-token :kind :punctuation :value c :position start)) + ((find c ".;-=^:(),@[]~") (make-token :kind :punctuation :value c :position start)) ((char= c #\%) ;; Production [71] has a literal terminal followed by an identifier; ;; unlike Role separators it does not require intervening whitespace. - (if (starts-with "%prefix" (subseq text start)) - (progn (setf (lexer-cursor lexer) (+ start 7)) - (make-token :kind :directive :value "prefix" :position start)) - (progn - (scan-name lexer) - (problem 'ctm-unsupported :unsupported-directive start - (format nil "Directive %~A is outside the implemented subset." - (subseq text (1+ start) (lexer-cursor lexer))))))) - ((find c "?[$@~") - (problem 'ctm-unsupported :unsupported-construct start - (format nil "Construct starting with ~S is outside the implemented subset." c))) + (let ((name (find-if (lambda (name) (starts-with (concatenate 'string "%" name) (subseq text start))) + '("prefix" "mergemap" "include" "encoding" "version")))) + (if name + (progn (setf (lexer-cursor lexer) (+ start 1 (length name))) + (make-token :kind :directive :value name :position start)) + (progn + (scan-name lexer) + (problem 'ctm-rejection :unknown-directive start + (format nil "Unknown CTM directive %~A." + (subseq text (1+ start) (lexer-cursor lexer)))))))) + ((char= c #\$) + (unless (name-start-p (peek-character lexer)) + (problem 'ctm-rejection :invalid-variable start "A variable requires an identifier.")) + (take-character lexer) (scan-name lexer) + (make-token :kind :variable :value (subseq text (1+ start) (lexer-cursor lexer)) :position start)) (t (problem 'ctm-rejection :unexpected-character start (format nil "Unexpected character ~S." c))))))