X-Git-Url: https://dreyeck.freedombox.rocks/gitweb/topic-maps.git/blobdiff_plain/fb60afd548d679d83fd8ec76bc5ff384c1e23e5f..2ad41cd1a945a4ce97824464de40646bb899259e:/src/ctm/lexer.lisp diff --git a/src/ctm/lexer.lisp b/src/ctm/lexer.lisp index 6331cde..66a5ed8 100644 --- a/src/ctm/lexer.lisp +++ b/src/ctm/lexer.lisp @@ -77,12 +77,66 @@ (return)))) (t (return))))) +(defun excluded-literal-p (text) + "Recognize unsupported dateTime/* literal syntax, without constructing values." + (labels ((digits (s) (and (plusp (length s)) (every (lambda (c) (find c "0123456789")) s))) + (date (s) + (let* ((s (if (and (plusp (length s)) (char= (char s 0) #\-)) (subseq s 1) s)) + (dash (position #\- s))) + (and dash (>= dash 4) (digits (subseq s 0 dash)) + (= (length s) (+ dash 6)) (char= (char s (+ dash 3)) #\-) + (digits (subseq s (1+ dash) (+ dash 3))) (digits (subseq s (+ dash 4))) + (<= 1 (parse-integer s :start (1+ dash) :end (+ dash 3)) 12) + (<= 1 (parse-integer s :start (+ dash 4)) 31)))) + (time-literal (s) + (and (>= (length s) 8) (digits (subseq s 0 2)) (char= (char s 2) #\:) + (digits (subseq s 3 5)) (char= (char s 5) #\:) (digits (subseq s 6 8)) + (let* ((rest (subseq s 8)) + (end (position-if (lambda (c) (find c "Z+-")) rest)) + (fraction (subseq rest 0 end)) (zone (if end (subseq rest end) ""))) + (and (or (zerop (length fraction)) + (and (char= (char fraction 0) #\.) (digits (subseq fraction 1)))) + (or (member zone '("" "Z") :test #'string=) + (and (= 6 (length zone)) (find (char zone 0) "+-") + (digits (subseq zone 1 3)) (char= (char zone 3) #\:) + (digits (subseq zone 4 6))))))))) + (or (string= text "*") + (let ((tee (position #\T text))) (and tee (date (subseq text 0 tee)) (time-literal (subseq text (1+ tee)))))))) + +(defun read-number-token (lexer start) + "One token path for Integer, Decimal and Date, plus recognized excluded dateTime/* forms. +A dot followed by a digit can belong to a decimal; a terminal dot stays punctuation." + (setf (lexer-cursor lexer) start) + (loop for c = (peek-character lexer) + while (and c (not (whitespace-p c)) (not (find c ";@~()#^<>\"")) + (or (not (char= c #\.)) + (and (peek-character lexer 1) (find (peek-character lexer 1) "0123456789")))) + do (take-character lexer)) + (let ((value (subseq (lexer-text lexer) start (lexer-cursor lexer)))) + (cond ((topic-maps.tmdm:integer-lexical-p value) + (make-token :kind :integer :value value :position start)) + ((and (find #\. value) (topic-maps.tmdm:decimal-lexical-p value)) + (make-token :kind :decimal :value value :position start)) + ((excluded-literal-p value) (make-token :kind :excluded-literal :value value :position start)) + ((position #\- value :start (if (and (plusp (length value)) (find (char value 0) "+-")) 1 0)) + (multiple-value-bind (a date-end end reason) (topic-maps.tmdm::date-lexical-bounds value) + (unless (and a (= date-end end)) + (problem 'ctm-rejection (or reason :invalid-date-syntax) start + "Expected a valid timezone-free XSD 1.0 date: nonzero year, valid month/day and no leading plus or extended-year zeroes.")) + (make-token :kind :date :value value :position start))) + (t (problem 'ctm-rejection :invalid-occurrence-literal start "Malformed numeric/date literal."))))) + (defun next-token (lexer) (skip-layout lexer) (let* ((start (lexer-cursor lexer)) (c (take-character lexer)) (text (lexer-text lexer))) (cond ((null c) (make-token :kind :eof :position start)) + ((or (find c "0123456789+*") + (and (find c "-.") (peek-character lexer) (find (peek-character lexer) "0123456789")) + (and (char= c #\-) (eql #\. (peek-character lexer)) + (peek-character lexer 1) (find (peek-character lexer 1) "0123456789"))) + (read-number-token lexer start)) ((char= c #\") (make-token :kind :string :value (read-ctm-string lexer start) :position start)) ((char= c #\<) (loop while (and (peek-character lexer) (not (eql #\> (peek-character lexer)))) @@ -103,7 +157,7 @@ (eql #\/ (peek-character lexer 1)) (eql #\/ (peek-character lexer 2))) (incf (lexer-cursor lexer) 3) (loop while (and (peek-character lexer) (not (whitespace-p (peek-character lexer))) - (not (find (peek-character lexer) "<>\"{}|^`\\[]("))) + (not (find (peek-character lexer) "<>\"{}|^`\\[]()"))) do (take-character lexer)) (loop while (and (> (lexer-cursor lexer) start) (find (char text (1- (lexer-cursor lexer))) ";.)")) @@ -117,11 +171,26 @@ ;; Otherwise ':' remains a separate token, including name types ;; followed by a string with or without intervening layout. (take-character lexer) - (scan-name lexer) + (if (name-start-p (peek-character lexer)) + (scan-name lexer) + ;; CTM 3.3.4 production [11]: numeric-leading local parts + ;; contain name-part characters, which do not include dots. + (loop while (name-part-p (peek-character lexer)) do (take-character lexer))) (make-token :kind :qname :value (subseq text start (lexer-cursor lexer)) :position start)) (t (make-token :kind :identifier :value head :position start))))) - ((find c ".;-=^:") (make-token :kind :punctuation :value c :position start)) - ((find c "%?[$@~()") + ((find c ".;-=^:(),") (make-token :kind :punctuation :value c :position start)) + ((char= c #\%) + ;; Production [71] has a literal terminal followed by an identifier; + ;; unlike Role separators it does not require intervening whitespace. + (if (starts-with "%prefix" (subseq text start)) + (progn (setf (lexer-cursor lexer) (+ start 7)) + (make-token :kind :directive :value "prefix" :position start)) + (progn + (scan-name lexer) + (problem 'ctm-unsupported :unsupported-directive start + (format nil "Directive %~A is outside the implemented subset." + (subseq text (1+ start) (lexer-cursor lexer))))))) + ((find c "?[$@~") (problem 'ctm-unsupported :unsupported-construct start - (format nil "Construct starting with ~S is outside Slice 1." c))) + (format nil "Construct starting with ~S is outside the implemented subset." c))) (t (problem 'ctm-rejection :unexpected-character start (format nil "Unexpected character ~S." c))))))