]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/ctm/templates.lisp
Implement CTM Templates with contextual argument binding
[topic-maps.git] / src / ctm / templates.lisp
1 (in-package #:topic-maps.ctm)
2
3 ;; Resource policy, not a claim that recursive CTM syntax is invalid.
4 (defparameter *template-depth-limit* 256)
5
6 ;;; CTM 3.14-3.15. Stored, grammar-checked concrete syntax is a reader object,
7 ;;; not a second Topic Maps model. Execution reuses the ordinary readers.
8 (defun identifier-p (token name)
9 (and (eq :identifier (token-kind token)) (string= name (token-value token))))
10
11 (defun call-with-token-checkpoint (parser thunk)
12 (let ((cursor (lexer-cursor (parser-lexer parser)))
13 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
14 (recording (parser-recording parser)) (recorded (parser-recorded parser)))
15 (unwind-protect (progn (setf (parser-recording parser) nil) (funcall thunk))
16 (setf (lexer-cursor (parser-lexer parser)) cursor
17 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
18 (parser-recording parser) recording (parser-recorded parser) recorded))))
19
20 (defun next-punctuation-p (parser char)
21 (call-with-token-checkpoint parser (lambda () (consume parser) (punctuation-p parser char))))
22
23 (defun tail-invocation-p (parser)
24 (and (eq :identifier (token-kind (look parser))) (next-punctuation-p parser #\()))
25
26 (defun global-invocation-p (parser)
27 "Decide by syntax, never by whether a template happens to be defined.
28 A standalone colon in the first argument denotes a Role; QName colons are
29 already inside one lexer token. Embedded syntax is skipped with balanced depth."
30 (and (tail-invocation-p parser)
31 (call-with-token-checkpoint parser
32 (lambda ()
33 (consume parser) (consume parser)
34 (let ((depth 0))
35 (loop for token = (consume parser) do
36 (when (eq :eof (token-kind token)) (return t))
37 (when (eq :punctuation (token-kind token))
38 (let ((c (token-value token)))
39 (cond ((and (zerop depth) (char= c #\:)) (return nil))
40 ((and (zerop depth) (find c ",)")) (return t))
41 ((find c "([") (incf depth))
42 ((find c ")]" ) (decf depth)))))))))))
43
44 (defun call-with-tokens (parser tokens thunk)
45 (let ((cursor (lexer-cursor (parser-lexer parser)))
46 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
47 (replay (parser-replay-p parser)))
48 (unwind-protect
49 (progn (setf (parser-replay-p parser) t (parser-remaining parser) tokens
50 (parser-lookahead parser) nil)
51 (funcall thunk))
52 (setf (lexer-cursor (parser-lexer parser)) cursor
53 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
54 (parser-replay-p parser) replay))))
55
56 (defun variable-binding (parser token)
57 (unless (parser-bindings parser)
58 (problem 'unbound-template-variable :variable-outside-template (token-position token)
59 "Variables are only allowed within templates."))
60 (or (gethash (token-value token) (parser-bindings parser))
61 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
62 (format nil "No argument is bound to $~A." (token-value token)))))
63
64 (defun argument-problem (token expected)
65 (problem 'invalid-template-argument :invalid-template-argument (token-position token)
66 (format nil "The bound argument cannot be used as ~A." expected)))
67
68 (defun bound-identity (parser token explicit-kind)
69 (let ((argument (variable-binding parser token)))
70 (case (template-argument-kind argument)
71 (:iri (values (or explicit-kind :subject-identifier) (template-argument-value argument)))
72 (:topic (when explicit-kind (argument-problem token "an IRI after = or ^"))
73 (values :bound-topic (topic-maps.tmdm:canonical-topic (template-argument-topic argument))))
74 (otherwise (argument-problem token "a Topic reference")))))
75
76 (defun bound-literal (parser token)
77 (let ((argument (variable-binding parser token)))
78 (case (template-argument-kind argument)
79 (:iri (values (template-argument-value argument) (topic-maps.tmdm:iri-datatype)))
80 (:literal (values (template-argument-value argument) (template-argument-datatype argument)))
81 (otherwise (argument-problem token "a literal")))))
82
83 (defun bound-name (parser token)
84 (let ((argument (variable-binding parser token)))
85 (unless (template-argument-plain-string-p argument) (argument-problem token "a quoted name string"))
86 (template-argument-value argument)))
87
88 (defun read-call-argument (parser map)
89 (let ((token (look parser)))
90 (case (token-kind token)
91 (:variable (consume parser) (variable-binding parser token))
92 ((:iri :qname)
93 ;; Resolve/validate the locator once in the caller, without interning a
94 ;; Topic. In particular an unused QName still needs a bound prefix, and
95 ;; a resolved prefix must not be percent-decoded again at variable use.
96 (make-template-argument :kind :iri :token token :value (literal-iri-reference parser)))
97 ((:string :integer :decimal :date :datetime :excluded-literal)
98 (let ((plain (and (eq :string (token-kind token)) (not (next-punctuation-p parser #\^)))))
99 (multiple-value-bind (value datatype) (read-literal parser)
100 (make-template-argument :kind :literal :token token :value value :datatype datatype :plain-string-p plain))))
101 (otherwise
102 (unless (topic-reference-start-p token)
103 (problem 'invalid-template-argument :expected-template-argument (token-position token) "Expected a Topic reference or literal argument."))
104 ;; Evaluate Wildcards/Embedded Topics ONCE in the caller's environment.
105 (make-template-argument :kind :topic :token token :topic (topic-reference parser map))))))
106
107 (defun read-template-invocation (parser map &optional owner)
108 (let* ((name (consume parser)) (arguments nil))
109 (consume parser) ; '(' recognized by dispatch
110 (unless (punctuation-p parser #\))
111 (loop (push (read-call-argument parser map) arguments)
112 (unless (punctuation-p parser #\,) (return))
113 (consume parser)))
114 (unless (punctuation-p parser #\))
115 (problem 'template-error :missing-template-close (token-position (look parser)) "Expected ')' after template arguments."))
116 (consume parser)
117 (setf arguments (nreverse arguments))
118 (when owner (push (make-template-argument :kind :topic :token name :topic (topic-maps.tmdm:canonical-topic owner)) arguments))
119 (let* ((key (list (token-value name) (length arguments)))
120 (definition (gethash key (parser-templates parser))))
121 (unless definition
122 (let ((same-name (loop for key being the hash-keys of (parser-templates parser)
123 thereis (string= (first key) (token-value name)))))
124 (problem (if same-name 'template-arity-error 'undefined-template)
125 (if same-name :template-arity :undefined-template) (token-position name)
126 (format nil "No template ~A/~D is defined at this invocation." (first key) (second key)))))
127 (when (>= (length *template-context*) *template-depth-limit*)
128 (problem 'ctm-unsupported :template-depth-limit (token-position name)
129 "The configured Template expansion depth limit was reached."))
130 (let ((bindings (make-hash-table :test #'equal))
131 (old-bindings (parser-bindings parser)) (old-wildcards (parser-wildcards parser))
132 (*template-context* (cons (list :name (template-definition-name definition)
133 :definition-position (template-definition-position definition)
134 :invocation-position (token-position name)) *template-context*)))
135 (loop for parameter in (template-definition-parameters definition) for argument in arguments
136 do (setf (gethash parameter bindings) argument))
137 (unwind-protect
138 (progn
139 (setf (parser-bindings parser) bindings (parser-wildcards parser) (make-hash-table :test #'equal))
140 (dolist (statement (template-definition-body definition))
141 (call-with-tokens parser (ctm-statement-tokens statement)
142 (lambda () (read-statement parser map)))))
143 (setf (parser-bindings parser) old-bindings (parser-wildcards parser) old-wildcards))))))
144
145 (defun read-statement (parser map)
146 (if (global-invocation-p parser)
147 (read-template-invocation parser map)
148 (let* ((embedded (punctuation-p parser #\[)) (topic (topic-reference parser map)))
149 (if (punctuation-p parser #\()
150 (read-association parser map topic)
151 (if embedded
152 (problem 'ctm-rejection :embedded-topic-at-top-level (token-position (look parser))
153 "An Embedded Topic at top level must be an Association Type, not a standalone declaration.")
154 (read-topic-tail parser map topic #\.))))))
155
156 ;;; Grammar-only traversal. It consumes the SAME lexical tokens, storing checked
157 ;;; statement boundaries. No TMDM constructors, locator interning or wildcard
158 ;;; execution occur here. Argument meaning is checked by the ordinary readers
159 ;;; when the stored statement is invoked, not by textual substitution.
160 (defun syntax-problem (parser message &optional (code :invalid-template-body))
161 (problem 'invalid-template-body code (token-position (look parser)) message))
162 (defun syntax-expect (parser char)
163 (unless (punctuation-p parser char)
164 (syntax-problem parser (format nil "Expected '~A' in template syntax." char)))
165 (consume parser))
166 (defun syntax-variable (parser parameters)
167 (let ((token (consume parser)))
168 (unless (member (token-value token) parameters :test #'string=)
169 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
170 "A template variable has no corresponding parameter."))))
171 (defun syntax-reference (parser parameters &optional iri-only)
172 (let ((token (look parser)))
173 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
174 ((member (token-kind token) '(:iri :qname)) (consume parser))
175 (iri-only (syntax-problem parser "Expected an IRI reference."))
176 ((and (eq :identifier (token-kind token))
177 (not (member (token-value token) '("def" "end" "isa" "ako") :test #'string=))) (consume parser))
178 ((eq :wildcard (token-kind token)) (consume parser))
179 ((or (punctuation-p parser #\=) (punctuation-p parser #\^))
180 (consume parser) (syntax-reference parser parameters t))
181 ((punctuation-p parser #\[)
182 (consume parser) (syntax-tail parser parameters #\] t))
183 (t (syntax-problem parser "Expected a Topic reference in template syntax.")))))
184 (defun syntax-literal (parser parameters)
185 (let ((token (look parser)))
186 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
187 ((member (token-kind token) '(:iri :qname :integer :decimal :date :datetime)) (consume parser))
188 ((eq :excluded-literal (token-kind token))
189 (problem 'ctm-unsupported :occurrence-literal-datatype (token-position token) "The separate '*' literal remains unsupported."))
190 ((eq :string (token-kind token))
191 (consume parser)
192 (when (punctuation-p parser #\^)
193 (let ((caret (consume parser)))
194 (unless (and (punctuation-p parser #\^) (= (token-position (look parser)) (1+ (token-position caret))))
195 (syntax-problem parser "Expected contiguous '^^'." :invalid-datatype-marker))
196 (consume parser)
197 (when (eq :variable (token-kind (look parser)))
198 (syntax-problem parser "A datatype qualifier cannot contain a variable." :variable-datatype))
199 (unless (member (token-kind (look parser)) '(:iri :qname))
200 (syntax-problem parser "A datatype requires an IRI reference."))
201 (consume parser))))
202 (t (syntax-problem parser "Expected a literal in template syntax.")))))
203 (defun syntax-scope (parser parameters)
204 (when (punctuation-p parser #\@)
205 (consume parser)
206 (loop (syntax-reference parser parameters)
207 (unless (punctuation-p parser #\,) (return)) (consume parser))))
208 (defun syntax-reifier (parser parameters)
209 (when (punctuation-p parser #\~) (consume parser) (syntax-reference parser parameters)))
210 (defun syntax-name (parser parameters)
211 (consume parser) ; '-'
212 (unless (or (eq :string (token-kind (look parser)))
213 (and (eq :variable (token-kind (look parser))) (not (next-punctuation-p parser #\:))))
214 (syntax-reference parser parameters) (syntax-expect parser #\:))
215 (case (token-kind (look parser))
216 (:variable (syntax-variable parser parameters))
217 (:string (consume parser))
218 (otherwise (syntax-problem parser "Expected a string Name value.")))
219 (syntax-scope parser parameters) (syntax-reifier parser parameters)
220 (loop while (punctuation-p parser #\() do
221 (consume parser) (syntax-literal parser parameters)
222 (unless (punctuation-p parser #\@) (syntax-problem parser "A Variant requires explicit Scope." :expected-variant-scope))
223 (syntax-scope parser parameters) (syntax-reifier parser parameters) (syntax-expect parser #\))))
224 (defun syntax-invocation (parser parameters)
225 (consume parser) (syntax-expect parser #\()
226 (unless (punctuation-p parser #\))
227 (loop
228 (if (member (token-kind (look parser)) '(:string :integer :decimal :date :datetime :excluded-literal))
229 (syntax-literal parser parameters) (syntax-reference parser parameters))
230 (unless (punctuation-p parser #\,) (return)) (consume parser)))
231 (syntax-expect parser #\)))
232 (defun syntax-tail (parser parameters terminator &optional required)
233 (let ((seen nil) (separator nil))
234 (loop
235 (when (punctuation-p parser terminator)
236 (when (and required (not seen)) (syntax-problem parser "An Embedded Topic requires a Tail."))
237 (consume parser) (return))
238 (when separator (syntax-expect parser #\;)
239 (when (punctuation-p parser terminator) (consume parser) (return)))
240 (cond
241 ((punctuation-p parser #\-) (syntax-name parser parameters))
242 ((relationship-keyword-p (look parser)) (consume parser) (syntax-reference parser parameters))
243 ((tail-invocation-p parser) (syntax-invocation parser parameters))
244 (t
245 (let ((start (look parser)))
246 (syntax-reference parser parameters)
247 (if (punctuation-p parser #\:)
248 (progn (consume parser) (syntax-literal parser parameters)
249 (syntax-scope parser parameters) (syntax-reifier parser parameters))
250 (when (or (member (token-kind start) '(:variable :identifier :wildcard))
251 (and (eq :punctuation (token-kind start)) (eql #\[ (token-value start))))
252 (syntax-problem parser "Unmarked variable identities in Topic Tails remain outside the supported grammar." :template-identity-tail))))))
253 (setf seen t separator t))))
254 (defun syntax-statement (parser parameters)
255 (if (global-invocation-p parser)
256 (progn (syntax-invocation parser parameters) :invocation)
257 (let ((embedded (punctuation-p parser #\[)))
258 (syntax-reference parser parameters)
259 (if (punctuation-p parser #\()
260 (progn
261 (consume parser)
262 (loop
263 (syntax-reference parser parameters)
264 (let* ((end (lexer-cursor (parser-lexer parser))) (colon (look parser)))
265 (syntax-expect parser #\:)
266 (unless (or (> (token-position colon) end)
267 (> (token-position (look parser)) (1+ (token-position colon))))
268 (syntax-problem parser "A Role separator requires layout." :role-separator-whitespace)))
269 (syntax-reference parser parameters) (syntax-reifier parser parameters)
270 (unless (punctuation-p parser #\,) (return)) (consume parser))
271 (syntax-expect parser #\)) (syntax-scope parser parameters) (syntax-reifier parser parameters)
272 :association)
273 (progn
274 (when embedded (syntax-problem parser "Standalone Embedded Topics are not template-body statements."))
275 (syntax-tail parser parameters #\.) :topic)))))
276 (defun read-template-definition (parser)
277 (let* ((start (consume parser)) (name (consume parser)) (parameters nil) (body nil))
278 (unless (and (eq :identifier (token-kind name))
279 (not (member (token-value name) '("def" "end") :test #'string=)))
280 (syntax-problem parser "A template name must be an identifier."))
281 (syntax-expect parser #\()
282 (unless (punctuation-p parser #\))
283 (loop
284 (unless (eq :variable (token-kind (look parser))) (syntax-problem parser "Expected a parameter variable."))
285 (let ((token (consume parser)))
286 (when (member (token-value token) parameters :test #'string=)
287 (syntax-problem parser "Duplicate parameter names cannot be bound positionally." :duplicate-template-parameter))
288 (push (token-value token) parameters))
289 (unless (punctuation-p parser #\,) (return)) (consume parser)))
290 (syntax-expect parser #\))
291 (setf parameters (nreverse parameters))
292 (let ((key (list (token-value name) (length parameters))))
293 (when (gethash key (parser-templates parser))
294 (problem 'duplicate-template :duplicate-template (token-position name) "The template name/arity is already defined."))
295 (loop until (identifier-p (look parser) "end") do
296 (when (or (eq :eof (token-kind (look parser))) (eq :directive (token-kind (look parser)))
297 (identifier-p (look parser) "def") (punctuation-p parser #\~))
298 (syntax-problem parser "Expected a Topic, Association or Invocation, followed by 'end'."))
299 (let ((position (token-position (look parser))))
300 (setf (parser-recording parser) t (parser-recorded parser) nil)
301 (unwind-protect
302 (let ((kind (syntax-statement parser parameters)))
303 (push (make-ctm-statement :kind kind :position position
304 :tokens (nreverse (parser-recorded parser))) body))
305 (setf (parser-recording parser) nil (parser-recorded parser) nil))))
306 (consume parser)
307 (setf (gethash key (parser-templates parser))
308 (make-template-definition :name (token-value name) :parameters parameters
309 :position (token-position start) :body (nreverse body))))))