(in-package #:topic-maps.ctm)
-(defstruct token kind value (position 0))
+(defstruct token kind value (position 0) end)
(defstruct (lexer (:constructor make-lexer (text))) text (cursor 0))
(defun peek-character (lexer &optional (delta 0))
(t (return)))))
(defun read-ctm-string (lexer start)
- (when (and (eql #\" (peek-character lexer)) (eql #\" (peek-character lexer 1)))
- (problem 'ctm-unsupported :triple-quoted-string start "Triple quoted literals are outside Slice 1."))
- (with-output-to-string (out)
- (loop for c = (take-character lexer) do
- (cond ((null c) (problem 'ctm-rejection :unterminated-string start "Unterminated quoted string."))
- ((char= c #\") (return))
- ((char= c #\\)
- (let ((escape (take-character lexer)))
- (case escape
- (#\n (write-char #\Newline out)) (#\r (write-char #\Return out))
- (#\t (write-char #\Tab out)) (#\" (write-char #\" out)) (#\\ (write-char #\\ out))
- ((#\u #\U)
- (let ((code 0))
- (dotimes (i (if (char= escape #\u) 4 6))
- (let* ((digit (take-character lexer)) (value (and digit (digit-char-p digit 16))))
- (unless value
- (problem 'ctm-rejection :invalid-unicode-escape (1- (lexer-cursor lexer)) "Expected hexadecimal escape digit."))
- (setf code (+ (* code 16) value))))
- (write-char (scalar-character code start) out)))
- (otherwise (problem 'ctm-rejection :invalid-string-escape (1- (lexer-cursor lexer)) "Unknown string escape.")))))
- (t (write-char c out))))))
+ ;; NEXT-TOKEN has consumed the first quote. Triple strings share the exact
+ ;; same escape/scalar path; only their opening and closing delimiters differ.
+ (let ((triple (and (eql #\" (peek-character lexer))
+ (eql #\" (peek-character lexer 1)))))
+ (when triple (incf (lexer-cursor lexer) 2))
+ (with-output-to-string (out)
+ (loop for c = (take-character lexer) do
+ (cond ((null c)
+ (problem 'ctm-rejection :unterminated-string start
+ (if triple "Unterminated triple-quoted string." "Unterminated quoted string.")))
+ ((char= c #\")
+ (cond ((not triple) (return))
+ ((and (eql #\" (peek-character lexer))
+ (eql #\" (peek-character lexer 1)))
+ (incf (lexer-cursor lexer) 2) (return))
+ (t (write-char c out))))
+ ((char= c #\\)
+ (let ((escape (take-character lexer)))
+ (case escape
+ (#\n (write-char #\Newline out)) (#\r (write-char #\Return out))
+ (#\t (write-char #\Tab out)) (#\" (write-char #\" out)) (#\\ (write-char #\\ out))
+ ((#\u #\U)
+ (let ((code 0))
+ (dotimes (i (if (char= escape #\u) 4 6))
+ (let* ((digit (take-character lexer)) (value (and digit (digit-char-p digit 16))))
+ (unless value
+ (problem 'ctm-rejection :invalid-unicode-escape (1- (lexer-cursor lexer)) "Expected hexadecimal escape digit."))
+ (setf code (+ (* code 16) value))))
+ (write-char (scalar-character code start) out)))
+ (otherwise (problem 'ctm-rejection :invalid-string-escape (1- (lexer-cursor lexer)) "Unknown string escape.")))))
+ (t (write-char c out)))))))
(defun scan-name (lexer)
(loop
(return))))
(t (return)))))
+(defun ctm-integer-literal-p (text)
+ (string= text "*"))
+
+(defun read-number-token (lexer start)
+ "One token path for Integer, Decimal, Date, DateTime and the exact * literal.
+A dot followed by a digit can belong to a decimal; a terminal dot stays punctuation."
+ (setf (lexer-cursor lexer) start)
+ (loop for c = (peek-character lexer)
+ while (and c (not (whitespace-p c)) (not (find c ";@~()#^<>\"[]"))
+ (or (not (char= c #\.))
+ (and (peek-character lexer 1) (find (peek-character lexer 1) "0123456789"))))
+ do (take-character lexer))
+ (let ((value (subseq (lexer-text lexer) start (lexer-cursor lexer))))
+ (cond ((topic-maps.tmdm:integer-lexical-p value)
+ (make-token :kind :integer :value value :position start))
+ ((and (find #\. value) (topic-maps.tmdm:decimal-lexical-p value))
+ (make-token :kind :decimal :value value :position start))
+ ((position #\T value)
+ (multiple-value-bind (parts reason) (topic-maps.tmdm::datetime-parts value)
+ (unless parts (problem 'ctm-rejection reason start "Expected a valid XSD 1.0 dateTime."))
+ (make-token :kind :datetime :value value :position start)))
+ ((ctm-integer-literal-p value) (make-token :kind :ctm-integer :value value :position start))
+ ((position #\- value :start (if (and (plusp (length value)) (find (char value 0) "+-")) 1 0))
+ (multiple-value-bind (a date-end end reason) (topic-maps.tmdm::date-lexical-bounds value)
+ (unless (and a (= date-end end))
+ (problem 'ctm-rejection (or reason :invalid-date-syntax) start
+ "Expected a valid timezone-free XSD 1.0 date: nonzero year, valid month/day and no leading plus or extended-year zeroes."))
+ (make-token :kind :date :value value :position start)))
+ (t (problem 'ctm-rejection :invalid-occurrence-literal start "Malformed numeric/date literal.")))))
+
(defun next-token (lexer)
(skip-layout lexer)
(let* ((start (lexer-cursor lexer)) (c (take-character lexer))
(text (lexer-text lexer)))
(cond
((null c) (make-token :kind :eof :position start))
+ ((or (find c "0123456789+*")
+ (and (find c "-.") (peek-character lexer) (find (peek-character lexer) "0123456789"))
+ (and (char= c #\-) (eql #\. (peek-character lexer))
+ (peek-character lexer 1) (find (peek-character lexer 1) "0123456789")))
+ (read-number-token lexer start))
((char= c #\") (make-token :kind :string :value (read-ctm-string lexer start) :position start))
+ ((char= c #\?)
+ ;; A named wildcard requires an immediately adjacent identifier.
+ (let ((name-start (lexer-cursor lexer)))
+ (when (name-start-p (peek-character lexer)) (take-character lexer) (scan-name lexer))
+ (make-token :kind :wildcard
+ :value (when (> (lexer-cursor lexer) name-start)
+ (subseq text name-start (lexer-cursor lexer)))
+ :position start)))
((char= c #\<)
(loop while (and (peek-character lexer) (not (eql #\> (peek-character lexer))))
do (take-character lexer))
(eql #\/ (peek-character lexer 1)) (eql #\/ (peek-character lexer 2)))
(incf (lexer-cursor lexer) 3)
(loop while (and (peek-character lexer) (not (whitespace-p (peek-character lexer)))
- (not (find (peek-character lexer) "<>\"{}|^`\\[](")))
+ (not (find (peek-character lexer) "<>\"{}|^`\\[]()")))
do (take-character lexer))
(loop while (and (> (lexer-cursor lexer) start)
(find (char text (1- (lexer-cursor lexer))) ";.)"))
;; Otherwise ':' remains a separate token, including name types
;; followed by a string with or without intervening layout.
(take-character lexer)
- (scan-name lexer)
+ (if (name-start-p (peek-character lexer))
+ (scan-name lexer)
+ ;; CTM 3.3.4 production [11]: numeric-leading local parts
+ ;; contain name-part characters, which do not include dots.
+ (loop while (name-part-p (peek-character lexer)) do (take-character lexer)))
(make-token :kind :qname :value (subseq text start (lexer-cursor lexer)) :position start))
(t (make-token :kind :identifier :value head :position start)))))
- ((find c ".;-=^:") (make-token :kind :punctuation :value c :position start))
- ((find c "%?[$@~()")
- (problem 'ctm-unsupported :unsupported-construct start
- (format nil "Construct starting with ~S is outside Slice 1." c)))
+ ((find c ".;-=^:(),@[]~") (make-token :kind :punctuation :value c :position start))
+ ((char= c #\%)
+ ;; Production [71] has a literal terminal followed by an identifier;
+ ;; unlike Role separators it does not require intervening whitespace.
+ (let ((name (find-if (lambda (name) (starts-with (concatenate 'string "%" name) (subseq text start)))
+ '("prefix" "mergemap" "include" "encoding" "version"))))
+ (if name
+ (progn (setf (lexer-cursor lexer) (+ start 1 (length name)))
+ (make-token :kind :directive :value name :position start))
+ (progn
+ (scan-name lexer)
+ (problem 'ctm-rejection :unknown-directive start
+ (format nil "Unknown CTM directive %~A."
+ (subseq text (1+ start) (lexer-cursor lexer))))))))
+ ((char= c #\$)
+ (unless (name-start-p (peek-character lexer))
+ (problem 'ctm-rejection :invalid-variable start "A variable requires an identifier."))
+ (take-character lexer) (scan-name lexer)
+ (make-token :kind :variable :value (subseq text (1+ start) (lexer-cursor lexer)) :position start))
(t (problem 'ctm-rejection :unexpected-character start (format nil "Unexpected character ~S." c))))))