]> dreyeck.freedombox.rocks Git - topic-maps.git/blob - src/ctm/templates.lisp
Provide pinned CXML sources and record failed XML qualification
[topic-maps.git] / src / ctm / templates.lisp
1 (in-package #:topic-maps.ctm)
2
3 ;; Resource policy, not a claim that recursive CTM syntax is invalid.
4 (defparameter *template-depth-limit* 256)
5
6 ;;; CTM 3.14-3.15. Stored, grammar-checked concrete syntax is a reader object,
7 ;;; not a second Topic Maps model. Execution reuses the ordinary readers.
8 (defun identifier-p (token name)
9 (and (eq :identifier (token-kind token)) (string= name (token-value token))))
10
11 (defun call-with-token-checkpoint (parser thunk)
12 (let ((cursor (lexer-cursor (parser-lexer parser)))
13 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
14 (recording (parser-recording parser)) (recorded (parser-recorded parser)))
15 (unwind-protect (progn (setf (parser-recording parser) nil) (funcall thunk))
16 (setf (lexer-cursor (parser-lexer parser)) cursor
17 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
18 (parser-recording parser) recording (parser-recorded parser) recorded))))
19
20 (defun next-punctuation-p (parser char)
21 (call-with-token-checkpoint parser (lambda () (consume parser) (punctuation-p parser char))))
22
23 (defun tail-invocation-p (parser)
24 (and (eq :identifier (token-kind (look parser))) (next-punctuation-p parser #\()))
25
26 (defun global-invocation-p (parser)
27 "Decide by syntax, never by whether a template happens to be defined.
28 A standalone colon in the first argument denotes a Role; QName colons are
29 already inside one lexer token. Embedded syntax is skipped with balanced depth."
30 (and (tail-invocation-p parser)
31 (call-with-token-checkpoint parser
32 (lambda ()
33 (consume parser) (consume parser)
34 (let ((depth 0))
35 (loop for token = (consume parser) do
36 (when (eq :eof (token-kind token)) (return t))
37 (when (eq :punctuation (token-kind token))
38 (let ((c (token-value token)))
39 (cond ((and (zerop depth) (char= c #\:)) (return nil))
40 ((and (zerop depth) (find c ",)")) (return t))
41 ((find c "([") (incf depth))
42 ((find c ")]" ) (decf depth)))))))))))
43
44 (defun call-with-tokens (parser tokens thunk)
45 (let ((cursor (lexer-cursor (parser-lexer parser)))
46 (lookahead (parser-lookahead parser)) (remaining (parser-remaining parser))
47 (replay (parser-replay-p parser)))
48 (unwind-protect
49 (progn (setf (parser-replay-p parser) t (parser-remaining parser) tokens
50 (parser-lookahead parser) nil)
51 (funcall thunk))
52 (setf (lexer-cursor (parser-lexer parser)) cursor
53 (parser-lookahead parser) lookahead (parser-remaining parser) remaining
54 (parser-replay-p parser) replay))))
55
56 (defun variable-binding (parser token)
57 (unless (parser-bindings parser)
58 (problem 'unbound-template-variable :variable-outside-template (token-position token)
59 "Variables are only allowed within templates."))
60 (or (gethash (token-value token) (parser-bindings parser))
61 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
62 (format nil "No argument is bound to $~A." (token-value token)))))
63
64 (defun argument-problem (token expected)
65 (problem 'invalid-template-argument :invalid-template-argument (token-position token)
66 (format nil "The bound argument cannot be used as ~A." expected)))
67
68 (defun bound-identity (parser token explicit-kind)
69 (let ((argument (variable-binding parser token)))
70 (case (template-argument-kind argument)
71 (:iri (values (or explicit-kind :subject-identifier) (template-argument-value argument)))
72 (:topic (when explicit-kind (argument-problem token "an IRI after = or ^"))
73 (values :bound-topic (topic-maps.tmdm:canonical-topic (template-argument-topic argument))))
74 (otherwise (argument-problem token "a Topic reference")))))
75
76 (defun bound-literal (parser token)
77 (let ((argument (variable-binding parser token)))
78 (case (template-argument-kind argument)
79 (:iri (values (template-argument-value argument) (topic-maps.tmdm:iri-datatype)))
80 (:literal (values (template-argument-value argument) (template-argument-datatype argument)))
81 (otherwise (argument-problem token "a literal")))))
82
83 (defun bound-name (parser token)
84 (let ((argument (variable-binding parser token)))
85 (unless (template-argument-plain-string-p argument) (argument-problem token "a quoted name string"))
86 (template-argument-value argument)))
87
88 (defun read-call-argument (parser map)
89 (let ((token (look parser)))
90 (case (token-kind token)
91 (:variable (consume parser) (variable-binding parser token))
92 ((:iri :qname)
93 ;; Resolve/validate the locator once in the caller, without interning a
94 ;; Topic. In particular an unused QName still needs a bound prefix, and
95 ;; a resolved prefix must not be percent-decoded again at variable use.
96 (make-template-argument :kind :iri :token token :value (literal-iri-reference parser)))
97 ((:string :integer :decimal :date :datetime :excluded-literal)
98 (let ((plain (and (eq :string (token-kind token)) (not (next-punctuation-p parser #\^)))))
99 (multiple-value-bind (value datatype) (read-literal parser)
100 (make-template-argument :kind :literal :token token :value value :datatype datatype :plain-string-p plain))))
101 (otherwise
102 (unless (topic-reference-start-p token)
103 (problem 'invalid-template-argument :expected-template-argument (token-position token) "Expected a Topic reference or literal argument."))
104 ;; Evaluate Wildcards/Embedded Topics ONCE in the caller's environment.
105 (make-template-argument :kind :topic :token token :topic (topic-reference parser map))))))
106
107 (defun read-template-invocation (parser map &optional owner)
108 (let* ((name (consume parser)) (arguments nil))
109 (consume parser) ; '(' recognized by dispatch
110 (unless (punctuation-p parser #\))
111 (loop (push (read-call-argument parser map) arguments)
112 (unless (punctuation-p parser #\,) (return))
113 (consume parser)))
114 (unless (punctuation-p parser #\))
115 (problem 'template-error :missing-template-close (token-position (look parser)) "Expected ')' after template arguments."))
116 (consume parser)
117 (setf arguments (nreverse arguments))
118 (when owner (push (make-template-argument :kind :topic :token name :topic (topic-maps.tmdm:canonical-topic owner)) arguments))
119 (let* ((key (list (token-value name) (length arguments)))
120 (definition (gethash key (parser-templates parser))))
121 (unless definition
122 (let ((same-name (loop for key being the hash-keys of (parser-templates parser)
123 thereis (string= (first key) (token-value name)))))
124 (problem (if same-name 'template-arity-error 'undefined-template)
125 (if same-name :template-arity :undefined-template) (token-position name)
126 (format nil "No template ~A/~D is defined at this invocation." (first key) (second key)))))
127 (when (>= (length *template-context*) *template-depth-limit*)
128 (problem 'ctm-unsupported :template-depth-limit (token-position name)
129 "The configured Template expansion depth limit was reached."))
130 (let ((bindings (make-hash-table :test #'equal))
131 (old-bindings (parser-bindings parser)) (old-wildcards (parser-wildcards parser))
132 (*template-context* (cons (list :name (template-definition-name definition)
133 :definition-position (template-definition-position definition)
134 :invocation-position (token-position name)
135 :definition-source (template-definition-source-name definition)
136 :definition-document-iri (template-definition-document-iri definition)
137 :invocation-source *source-name*
138 :invocation-document-iri (parser-document-iri parser)) *template-context*)))
139 (loop for parameter in (template-definition-parameters definition) for argument in arguments
140 do (setf (gethash parameter bindings) argument))
141 (unwind-protect
142 (progn
143 (setf (parser-bindings parser) bindings (parser-wildcards parser) (make-hash-table :test #'equal))
144 ;; The parser's Document IRI, Prefixes, registry and Counter stay
145 ;; in the caller. Only lexical source coordinates switch to the
146 ;; definition; nested invocations preserve both source frames.
147 (let ((*source-name* (template-definition-source-name definition))
148 (*source-text* (template-definition-source-text definition)))
149 (dolist (statement (template-definition-body definition))
150 (call-with-tokens parser (ctm-statement-tokens statement)
151 (lambda () (read-statement parser map))))))
152 (setf (parser-bindings parser) old-bindings (parser-wildcards parser) old-wildcards))))))
153
154 (defun read-statement (parser map)
155 (if (global-invocation-p parser)
156 (read-template-invocation parser map)
157 (let* ((embedded (punctuation-p parser #\[)) (topic (topic-reference parser map)))
158 (if (punctuation-p parser #\()
159 (read-association parser map topic)
160 (if embedded
161 (problem 'ctm-rejection :embedded-topic-at-top-level (token-position (look parser))
162 "An Embedded Topic at top level must be an Association Type, not a standalone declaration.")
163 (read-topic-tail parser map topic #\.))))))
164
165 ;;; Grammar-only traversal. It consumes the SAME lexical tokens, storing checked
166 ;;; statement boundaries. No TMDM constructors, locator interning or wildcard
167 ;;; execution occur here. Argument meaning is checked by the ordinary readers
168 ;;; when the stored statement is invoked, not by textual substitution.
169 (defun syntax-problem (parser message &optional (code :invalid-template-body))
170 (problem 'invalid-template-body code (token-position (look parser)) message))
171 (defun syntax-expect (parser char)
172 (unless (punctuation-p parser char)
173 (syntax-problem parser (format nil "Expected '~A' in template syntax." char)))
174 (consume parser))
175 (defun syntax-variable (parser parameters)
176 (let ((token (consume parser)))
177 (unless (member (token-value token) parameters :test #'string=)
178 (problem 'unbound-template-variable :unbound-template-variable (token-position token)
179 "A template variable has no corresponding parameter."))))
180 (defun syntax-reference (parser parameters &optional iri-only)
181 (let ((token (look parser)))
182 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
183 ((member (token-kind token) '(:iri :qname)) (consume parser))
184 (iri-only (syntax-problem parser "Expected an IRI reference."))
185 ((and (eq :identifier (token-kind token))
186 (not (member (token-value token) '("def" "end" "isa" "ako") :test #'string=))) (consume parser))
187 ((eq :wildcard (token-kind token)) (consume parser))
188 ((or (punctuation-p parser #\=) (punctuation-p parser #\^))
189 (consume parser) (syntax-reference parser parameters t))
190 ((punctuation-p parser #\[)
191 (consume parser) (syntax-tail parser parameters #\] t))
192 (t (syntax-problem parser "Expected a Topic reference in template syntax.")))))
193 (defun syntax-literal (parser parameters)
194 (let ((token (look parser)))
195 (cond ((eq :variable (token-kind token)) (syntax-variable parser parameters))
196 ((member (token-kind token) '(:iri :qname :integer :decimal :date :datetime)) (consume parser))
197 ((eq :excluded-literal (token-kind token))
198 (problem 'ctm-unsupported :occurrence-literal-datatype (token-position token) "The separate '*' literal remains unsupported."))
199 ((eq :string (token-kind token))
200 (consume parser)
201 (when (punctuation-p parser #\^)
202 (let ((caret (consume parser)))
203 (unless (and (punctuation-p parser #\^) (= (token-position (look parser)) (1+ (token-position caret))))
204 (syntax-problem parser "Expected contiguous '^^'." :invalid-datatype-marker))
205 (consume parser)
206 (when (eq :variable (token-kind (look parser)))
207 (syntax-problem parser "A datatype qualifier cannot contain a variable." :variable-datatype))
208 (unless (member (token-kind (look parser)) '(:iri :qname))
209 (syntax-problem parser "A datatype requires an IRI reference."))
210 (consume parser))))
211 (t (syntax-problem parser "Expected a literal in template syntax.")))))
212 (defun syntax-scope (parser parameters)
213 (when (punctuation-p parser #\@)
214 (consume parser)
215 (loop (syntax-reference parser parameters)
216 (unless (punctuation-p parser #\,) (return)) (consume parser))))
217 (defun syntax-reifier (parser parameters)
218 (when (punctuation-p parser #\~) (consume parser) (syntax-reference parser parameters)))
219 (defun syntax-name (parser parameters)
220 (consume parser) ; '-'
221 (unless (or (eq :string (token-kind (look parser)))
222 (and (eq :variable (token-kind (look parser))) (not (next-punctuation-p parser #\:))))
223 (syntax-reference parser parameters) (syntax-expect parser #\:))
224 (case (token-kind (look parser))
225 (:variable (syntax-variable parser parameters))
226 (:string (consume parser))
227 (otherwise (syntax-problem parser "Expected a string Name value.")))
228 (syntax-scope parser parameters) (syntax-reifier parser parameters)
229 (loop while (punctuation-p parser #\() do
230 (consume parser) (syntax-literal parser parameters)
231 (unless (punctuation-p parser #\@) (syntax-problem parser "A Variant requires explicit Scope." :expected-variant-scope))
232 (syntax-scope parser parameters) (syntax-reifier parser parameters) (syntax-expect parser #\))))
233 (defun syntax-invocation (parser parameters)
234 (consume parser) (syntax-expect parser #\()
235 (unless (punctuation-p parser #\))
236 (loop
237 (if (member (token-kind (look parser)) '(:string :integer :decimal :date :datetime :excluded-literal))
238 (syntax-literal parser parameters) (syntax-reference parser parameters))
239 (unless (punctuation-p parser #\,) (return)) (consume parser)))
240 (syntax-expect parser #\)))
241 (defun syntax-tail (parser parameters terminator &optional required)
242 (let ((seen nil) (separator nil))
243 (loop
244 (when (punctuation-p parser terminator)
245 (when (and required (not seen)) (syntax-problem parser "An Embedded Topic requires a Tail."))
246 (consume parser) (return))
247 (when separator (syntax-expect parser #\;)
248 (when (punctuation-p parser terminator) (consume parser) (return)))
249 (cond
250 ((punctuation-p parser #\-) (syntax-name parser parameters))
251 ((relationship-keyword-p (look parser)) (consume parser) (syntax-reference parser parameters))
252 ((tail-invocation-p parser) (syntax-invocation parser parameters))
253 (t
254 (let ((start (look parser)))
255 (syntax-reference parser parameters)
256 (if (punctuation-p parser #\:)
257 (progn (consume parser) (syntax-literal parser parameters)
258 (syntax-scope parser parameters) (syntax-reifier parser parameters))
259 (when (or (member (token-kind start) '(:variable :identifier :wildcard))
260 (and (eq :punctuation (token-kind start)) (eql #\[ (token-value start))))
261 (syntax-problem parser "Unmarked variable identities in Topic Tails remain outside the supported grammar." :template-identity-tail))))))
262 (setf seen t separator t))))
263 (defun syntax-statement (parser parameters)
264 (if (global-invocation-p parser)
265 (progn (syntax-invocation parser parameters) :invocation)
266 (let ((embedded (punctuation-p parser #\[)))
267 (syntax-reference parser parameters)
268 (if (punctuation-p parser #\()
269 (progn
270 (consume parser)
271 (loop
272 (syntax-reference parser parameters)
273 (let* ((end (lexer-cursor (parser-lexer parser))) (colon (look parser)))
274 (syntax-expect parser #\:)
275 (unless (or (> (token-position colon) end)
276 (> (token-position (look parser)) (1+ (token-position colon))))
277 (syntax-problem parser "A Role separator requires layout." :role-separator-whitespace)))
278 (syntax-reference parser parameters) (syntax-reifier parser parameters)
279 (unless (punctuation-p parser #\,) (return)) (consume parser))
280 (syntax-expect parser #\)) (syntax-scope parser parameters) (syntax-reifier parser parameters)
281 :association)
282 (progn
283 (when embedded (syntax-problem parser "Standalone Embedded Topics are not template-body statements."))
284 (syntax-tail parser parameters #\.) :topic)))))
285 (defun read-template-definition (parser)
286 (let* ((start (consume parser)) (name (consume parser)) (parameters nil) (body nil))
287 (unless (and (eq :identifier (token-kind name))
288 (not (member (token-value name) '("def" "end") :test #'string=)))
289 (syntax-problem parser "A template name must be an identifier."))
290 (syntax-expect parser #\()
291 (unless (punctuation-p parser #\))
292 (loop
293 (unless (eq :variable (token-kind (look parser))) (syntax-problem parser "Expected a parameter variable."))
294 (let ((token (consume parser)))
295 (when (member (token-value token) parameters :test #'string=)
296 (syntax-problem parser "Duplicate parameter names cannot be bound positionally." :duplicate-template-parameter))
297 (push (token-value token) parameters))
298 (unless (punctuation-p parser #\,) (return)) (consume parser)))
299 (syntax-expect parser #\))
300 (setf parameters (nreverse parameters))
301 (let ((key (list (token-value name) (length parameters))))
302 (when (gethash key (parser-templates parser))
303 (problem 'duplicate-template :duplicate-template (token-position name) "The template name/arity is already defined."))
304 (loop until (identifier-p (look parser) "end") do
305 (when (or (eq :eof (token-kind (look parser))) (eq :directive (token-kind (look parser)))
306 (identifier-p (look parser) "def") (punctuation-p parser #\~))
307 (syntax-problem parser "Expected a Topic, Association or Invocation, followed by 'end'."))
308 (let ((position (token-position (look parser))))
309 (setf (parser-recording parser) t (parser-recorded parser) nil)
310 (unwind-protect
311 (let ((kind (syntax-statement parser parameters)))
312 (push (make-ctm-statement :kind kind :position position
313 :tokens (nreverse (parser-recorded parser))) body))
314 (setf (parser-recording parser) nil (parser-recorded parser) nil))))
315 (consume parser)
316 (setf (gethash key (parser-templates parser))
317 (make-template-definition :name (token-value name) :parameters parameters
318 :position (token-position start) :body (nreverse body)
319 :source-name *source-name* :source-text *source-text*
320 :document-iri (parser-document-iri parser))))))