]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/ctm/templates.lisp
Document Topic Maps conformance closure plan for Slice 22A
[topic-maps.git] / src / ctm / templates.lisp
1 (in-package #:topic-maps.ctm)
2
3 ;; Resource policy, not a claim that recursive CTM syntax is invalid.
4 (defparameter *template-depth-limit* 256)
5
6 ;;; CTM 3.14-3.15. Stored, grammar-checked concrete syntax is a reader object,
7 ;;; not a second Topic Maps model. Execution reuses the ordinary readers.
8 (defun identifier-p (token name)
9 (and (eq :identifier (token-kind token)) (string= name (token-value token))))
10
11 (defun call-with-token-checkpoint (parser thunk)
12 (let ((cursor (lexer-cursor (parser-lexer parser)))
13 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
14 (recording (parser-recording parser)) (recorded (parser-recorded parser)))
15 (unwind-protect (progn (setf (parser-recording parser) nil) (funcall thunk))
16 (setf (lexer-cursor (parser-lexer parser)) cursor
17 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
18 (parser-recording parser) recording (parser-recorded parser) recorded))))
19
20 (defun next-punctuation-p (parser char)
21 (call-with-token-checkpoint parser (lambda () (consume parser) (punctuation-p parser char))))
22
23 (defun tail-invocation-p (parser)
24 (and (eq :identifier (token-kind (look parser))) (next-punctuation-p parser #\()))
25
26 (defun global-invocation-p (parser)
27 "Decide by syntax, never by whether a template happens to be defined.
28 A standalone colon in the first argument denotes a Role; QName colons are
29 already inside one lexer token. Embedded syntax is skipped with balanced depth."
30 (and (tail-invocation-p parser)
31 (call-with-token-checkpoint parser
32 (lambda ()
33 (consume parser) (consume parser)
34 (let ((depth 0))
35 (loop for token = (consume parser) do
36 (when (eq :eof (token-kind token)) (return t))
37 (when (eq :punctuation (token-kind token))
38 (let ((c (token-value token)))
39 (cond ((and (zerop depth) (char= c #\:)) (return nil))
40 ((and (zerop depth) (find c ",)")) (return t))
41 ((find c "([") (incf depth))
42 ((find c ")]" ) (decf depth)))))))))))
43
44 (defun call-with-tokens (parser tokens thunk)
45 (let ((cursor (lexer-cursor (parser-lexer parser)))
46 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
47 (replay (parser-replay-p parser)))
48 (unwind-protect
49 (progn (setf (parser-replay-p parser) t (parser-remaining parser) tokens
50 (parser-lookahead parser) nil)
51 (funcall thunk))
52 (setf (lexer-cursor (parser-lexer parser)) cursor
53 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
54 (parser-replay-p parser) replay))))
55
56 (defun variable-binding (parser token)
57 (unless (parser-bindings parser)
58 (problem 'unbound-template-variable :variable-outside-template (token-position token)
59 "Variables are only allowed within templates."))
60 (or (gethash (token-value token) (parser-bindings parser))
61 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
62 (format nil "No argument is bound to $~A." (token-value token)))))
63
64 (defun argument-problem (token expected)
65 (problem 'invalid-template-argument :invalid-template-argument (token-position token)
66 (format nil "The bound argument cannot be used as ~A." expected)))
67
68 (defun bound-identity (parser token explicit-kind)
69 (let ((argument (variable-binding parser token)))
70 (case (template-argument-kind argument)
71 (:iri (values (or explicit-kind :subject-identifier) (template-argument-value argument)))
72 (:topic (when explicit-kind (argument-problem token "an IRI after = or ^"))
73 (values :bound-topic (topic-maps.tmdm:canonical-topic (template-argument-topic argument))))
74 (otherwise (argument-problem token "a Topic reference")))))
75
76 (defun bound-literal (parser token)
77 (let ((argument (variable-binding parser token)))
78 (case (template-argument-kind argument)
79 (:iri (values (template-argument-value argument) (topic-maps.tmdm:iri-datatype)))
80 (:literal (values (template-argument-value argument) (template-argument-datatype argument)))
81 (otherwise (argument-problem token "a literal")))))
82
83 (defun bound-name (parser token)
84 (let ((argument (variable-binding parser token)))
85 (unless (template-argument-plain-string-p argument) (argument-problem token "a quoted name string"))
86 (template-argument-value argument)))
87
88 (defun read-call-argument (parser map)
89 (let ((token (look parser)))
90 (case (token-kind token)
91 (:variable (consume parser) (variable-binding parser token))
92 ((:iri :qname)
93 ;; Resolve/validate the locator once in the caller, without interning a
94 ;; Topic. In particular an unused QName still needs a bound prefix, and
95 ;; a resolved prefix must not be percent-decoded again at variable use.
96 (make-template-argument :kind :iri :token token :value (literal-iri-reference parser)))
97 ((:string :integer :decimal :date :datetime :ctm-integer)
98 (let ((plain (and (eq :string (token-kind token)) (not (next-punctuation-p parser #\^)))))
99 (multiple-value-bind (value datatype) (read-literal parser)
100 (make-template-argument :kind :literal :token token :value value :datatype datatype :plain-string-p plain))))
101 (otherwise
102 (unless (topic-reference-start-p token)
103 (problem 'invalid-template-argument :expected-template-argument (token-position token) "Expected a Topic reference or literal argument."))
104 ;; Evaluate Wildcards/Embedded Topics ONCE in the caller's environment.
105 (make-template-argument :kind :topic :token token :topic (topic-reference parser map))))))
106
107 (defun read-template-invocation (parser map &optional owner)
108 (let* ((name (consume parser)) (arguments nil))
109 (consume parser) ; '(' recognized by dispatch
110 (unless (punctuation-p parser #\))
111 (loop (push (read-call-argument parser map) arguments)
112 (unless (punctuation-p parser #\,) (return))
113 (consume parser)))
114 (unless (punctuation-p parser #\))
115 (problem 'template-error :missing-template-close (token-position (look parser)) "Expected ')' after template arguments."))
116 (consume parser)
117 (setf arguments (nreverse arguments))
118 (when owner (push (make-template-argument :kind :topic :token name :topic (topic-maps.tmdm:canonical-topic owner)) arguments))
119 (let* ((key (list (token-value name) (length arguments)))
120 (definition (gethash key (parser-templates parser))))
121 (unless definition
122 (let ((same-name (loop for key being the hash-keys of (parser-templates parser)
123 thereis (string= (first key) (token-value name)))))
124 (problem (if same-name 'template-arity-error 'undefined-template)
125 (if same-name :template-arity :undefined-template) (token-position name)
126 (format nil "No template ~A/~D is defined at this invocation." (first key) (second key)))))
127 (when (>= (length *template-context*) *template-depth-limit*)
128 (problem 'ctm-unsupported :template-depth-limit (token-position name)
129 "The configured Template expansion depth limit was reached."))
130 (let ((bindings (make-hash-table :test #'equal))
131 (old-bindings (parser-bindings parser)) (old-wildcards (parser-wildcards parser))
132 (*template-context* (cons (list :name (template-definition-name definition)
133 :definition-position (template-definition-position definition)
134 :invocation-position (token-position name)
135 :definition-source (template-definition-source-name definition)
136 :definition-document-iri (template-definition-document-iri definition)
137 :invocation-source *source-name*
138 :invocation-document-iri (parser-document-iri parser)) *template-context*)))
139 (loop for parameter in (template-definition-parameters definition) for argument in arguments
140 do (setf (gethash parameter bindings) argument))
141 (unwind-protect
142 (progn
143 (setf (parser-bindings parser) bindings (parser-wildcards parser) (make-hash-table :test #'equal))
144 ;; The parser's Document IRI, Prefixes, registry and Counter stay
145 ;; in the caller. Only lexical source coordinates switch to the
146 ;; definition; nested invocations preserve both source frames.
147 (let ((*source-name* (template-definition-source-name definition))
148 (*source-text* (template-definition-source-text definition)))
149 (dolist (statement (template-definition-body definition))
150 (call-with-tokens parser (ctm-statement-tokens statement)
151 (lambda () (read-statement parser map))))))
152 (setf (parser-bindings parser) old-bindings (parser-wildcards parser) old-wildcards))))))
153
154 (defun read-statement (parser map)
155 (if (global-invocation-p parser)
156 (read-template-invocation parser map)
157 (let* ((embedded (punctuation-p parser #\[)) (topic (topic-reference parser map)))
158 (if (punctuation-p parser #\()
159 (read-association parser map topic)
160 (if embedded
161 (problem 'ctm-rejection :embedded-topic-at-top-level (token-position (look parser))
162 "An Embedded Topic at top level must be an Association Type, not a standalone declaration.")
163 (read-topic-tail parser map topic #\.))))))
164
165 ;;; Grammar-only traversal. It consumes the SAME lexical tokens, storing checked
166 ;;; statement boundaries. No TMDM constructors, locator interning or wildcard
167 ;;; execution occur here. Argument meaning is checked by the ordinary readers
168 ;;; when the stored statement is invoked, not by textual substitution.
169 (defun syntax-problem (parser message &optional (code :invalid-template-body))
170 (problem 'invalid-template-body code (token-position (look parser)) message))
171 (defun syntax-expect (parser char)
172 (unless (punctuation-p parser char)
173 (syntax-problem parser (format nil "Expected '~A' in template syntax." char)))
174 (consume parser))
175 (defun syntax-variable (parser parameters)
176 (let ((token (consume parser)))
177 (unless (member (token-value token) parameters :test #'string=)
178 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
179 "A template variable has no corresponding parameter."))))
180 (defun syntax-reference (parser parameters &optional iri-only)
181 (let ((token (look parser)))
182 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
183 ((member (token-kind token) '(:iri :qname)) (consume parser))
184 (iri-only (syntax-problem parser "Expected an IRI reference."))
185 ((and (eq :identifier (token-kind token))
186 (not (member (token-value token) '("def" "end" "isa" "ako") :test #'string=))) (consume parser))
187 ((eq :wildcard (token-kind token)) (consume parser))
188 ((or (punctuation-p parser #\=) (punctuation-p parser #\^))
189 (consume parser) (syntax-reference parser parameters t))
190 ((punctuation-p parser #\[)
191 (consume parser) (syntax-tail parser parameters #\] t))
192 (t (syntax-problem parser "Expected a Topic reference in template syntax.")))))
193 (defun syntax-literal (parser parameters)
194 (let ((token (look parser)))
195 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
196 ((member (token-kind token) '(:iri :qname :integer :decimal :date :datetime)) (consume parser))
197 ((eq :ctm-integer (token-kind token)) (consume parser))
198 ((eq :string (token-kind token))
199 (consume parser)
200 (when (punctuation-p parser #\^)
201 (let ((caret (consume parser)))
202 (unless (and (punctuation-p parser #\^) (= (token-position (look parser)) (1+ (token-position caret))))
203 (syntax-problem parser "Expected contiguous '^^'." :invalid-datatype-marker))
204 (consume parser)
205 (when (eq :variable (token-kind (look parser)))
206 (syntax-problem parser "A datatype qualifier cannot contain a variable." :variable-datatype))
207 (unless (member (token-kind (look parser)) '(:iri :qname))
208 (syntax-problem parser "A datatype requires an IRI reference."))
209 (consume parser))))
210 (t (syntax-problem parser "Expected a literal in template syntax.")))))
211 (defun syntax-scope (parser parameters)
212 (when (punctuation-p parser #\@)
213 (consume parser)
214 (loop (syntax-reference parser parameters)
215 (unless (punctuation-p parser #\,) (return)) (consume parser))))
216 (defun syntax-reifier (parser parameters)
217 (when (punctuation-p parser #\~) (consume parser) (syntax-reference parser parameters)))
218 (defun syntax-name (parser parameters)
219 (consume parser) ; '-'
220 (unless (or (eq :string (token-kind (look parser)))
221 (and (eq :variable (token-kind (look parser))) (not (next-punctuation-p parser #\:))))
222 (syntax-reference parser parameters) (syntax-expect parser #\:))
223 (case (token-kind (look parser))
224 (:variable (syntax-variable parser parameters))
225 (:string (consume parser))
226 (otherwise (syntax-problem parser "Expected a string Name value.")))
227 (syntax-scope parser parameters) (syntax-reifier parser parameters)
228 (loop while (punctuation-p parser #\() do
229 (consume parser) (syntax-literal parser parameters)
230 (unless (punctuation-p parser #\@) (syntax-problem parser "A Variant requires explicit Scope." :expected-variant-scope))
231 (syntax-scope parser parameters) (syntax-reifier parser parameters) (syntax-expect parser #\))))
232 (defun syntax-invocation (parser parameters)
233 (consume parser) (syntax-expect parser #\()
234 (unless (punctuation-p parser #\))
235 (loop
236 (if (member (token-kind (look parser)) '(:string :integer :decimal :date :datetime :ctm-integer))
237 (syntax-literal parser parameters) (syntax-reference parser parameters))
238 (unless (punctuation-p parser #\,) (return)) (consume parser)))
239 (syntax-expect parser #\)))
240 (defun syntax-tail (parser parameters terminator &optional required)
241 (let ((seen nil) (separator nil))
242 (loop
243 (when (punctuation-p parser terminator)
244 (when (and required (not seen)) (syntax-problem parser "An Embedded Topic requires a Tail."))
245 (consume parser) (return))
246 (when separator (syntax-expect parser #\;)
247 (when (punctuation-p parser terminator) (consume parser) (return)))
248 (cond
249 ((punctuation-p parser #\-) (syntax-name parser parameters))
250 ((relationship-keyword-p (look parser)) (consume parser) (syntax-reference parser parameters))
251 ((tail-invocation-p parser) (syntax-invocation parser parameters))
252 (t
253 (let ((start (look parser)))
254 (syntax-reference parser parameters)
255 (if (punctuation-p parser #\:)
256 (progn (consume parser) (syntax-literal parser parameters)
257 (syntax-scope parser parameters) (syntax-reifier parser parameters))
258 (when (or (member (token-kind start) '(:variable :identifier :wildcard))
259 (and (eq :punctuation (token-kind start)) (eql #\[ (token-value start))))
260 (syntax-problem parser "Unmarked variable identities in Topic Tails remain outside the supported grammar." :template-identity-tail))))))
261 (setf seen t separator t))))
262 (defun syntax-statement (parser parameters)
263 (if (global-invocation-p parser)
264 (progn (syntax-invocation parser parameters) :invocation)
265 (let ((embedded (punctuation-p parser #\[)))
266 (syntax-reference parser parameters)
267 (if (punctuation-p parser #\()
268 (progn
269 (consume parser)
270 (loop
271 (syntax-reference parser parameters)
272 (let* ((end (lexer-cursor (parser-lexer parser))) (colon (look parser)))
273 (syntax-expect parser #\:)
274 (unless (or (> (token-position colon) end)
275 (> (token-position (look parser)) (1+ (token-position colon))))
276 (syntax-problem parser "A Role separator requires layout." :role-separator-whitespace)))
277 (syntax-reference parser parameters) (syntax-reifier parser parameters)
278 (unless (punctuation-p parser #\,) (return)) (consume parser))
279 (syntax-expect parser #\)) (syntax-scope parser parameters) (syntax-reifier parser parameters)
280 :association)
281 (progn
282 (when embedded (syntax-problem parser "Standalone Embedded Topics are not template-body statements."))
283 (syntax-tail parser parameters #\.) :topic)))))
284 (defun read-template-definition (parser)
285 (let* ((start (consume parser)) (name (consume parser)) (parameters nil) (body nil))
286 (unless (and (eq :identifier (token-kind name))
287 (not (member (token-value name) '("def" "end") :test #'string=)))
288 (syntax-problem parser "A template name must be an identifier."))
289 (syntax-expect parser #\()
290 (unless (punctuation-p parser #\))
291 (loop
292 (unless (eq :variable (token-kind (look parser))) (syntax-problem parser "Expected a parameter variable."))
293 (let ((token (consume parser)))
294 (when (member (token-value token) parameters :test #'string=)
295 (syntax-problem parser "Duplicate parameter names cannot be bound positionally." :duplicate-template-parameter))
296 (push (token-value token) parameters))
297 (unless (punctuation-p parser #\,) (return)) (consume parser)))
298 (syntax-expect parser #\))
299 (setf parameters (nreverse parameters))
300 (let ((key (list (token-value name) (length parameters))))
301 (when (gethash key (parser-templates parser))
302 (problem 'duplicate-template :duplicate-template (token-position name) "The template name/arity is already defined."))
303 (loop until (identifier-p (look parser) "end") do
304 (when (or (eq :eof (token-kind (look parser))) (eq :directive (token-kind (look parser)))
305 (identifier-p (look parser) "def") (punctuation-p parser #\~))
306 (syntax-problem parser "Expected a Topic, Association or Invocation, followed by 'end'."))
307 (let ((position (token-position (look parser))))
308 (setf (parser-recording parser) t (parser-recorded parser) nil)
309 (unwind-protect
310 (let ((kind (syntax-statement parser parameters)))
311 (push (make-ctm-statement :kind kind :position position
312 :tokens (nreverse (parser-recorded parser))) body))
313 (setf (parser-recording parser) nil (parser-recorded parser) nil))))
314 (consume parser)
315 (setf (gethash key (parser-templates parser))
316 (make-template-definition :name (token-value name) :parameters parameters
317 :position (token-position start) :body (nreverse body)
318 :source-name *source-name* :source-text *source-text*
319 :document-iri (parser-document-iri parser))))))