Coverage for tdom/parser.py: 98%
213 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-09 00:48 +0000
1from collections.abc import Callable, Sequence
2from dataclasses import dataclass, field
3from html.parser import HTMLParser
4from string.templatelib import Template
6from .htmlspec import VOID_ELEMENTS
7from .placeholders import (
8 PlaceholderConfig,
9 PlaceholderState,
10)
11from .placeholders import (
12 make_placeholder_config as default_make_placeholder_config,
13)
14from .template_utils import TemplateRef, combine_template_refs
15from .tnodes import (
16 TAttribute,
17 TComment,
18 TComponent,
19 TDocumentType,
20 TElement,
21 TFragment,
22 TInterpolatedAttribute,
23 TLiteralAttribute,
24 TNode,
25 TSpreadAttribute,
26 TTemplatedAttribute,
27 TText,
28)
30type HTMLAttribute = tuple[str, str | None]
31type HTMLAttributesDict = dict[str, str | None]
34@dataclass
35class OpenTElement:
36 tag: str
37 attrs: tuple[TAttribute, ...]
38 children: list[TNode] = field(default_factory=list)
41@dataclass
42class OpenTFragment:
43 children: list[TNode] = field(default_factory=list)
46@dataclass
47class OpenTComponent:
48 start_i_index: int
49 children_start_s_index: int
50 """The strings index where the component's children template starts."""
51 offset_into_children_start_s: int
52 """The offset INTO the starting string where the component's children template starts."""
53 attrs: tuple[TAttribute, ...]
54 # @NOTE: The `children` are discarded after parsing and are just used to
55 # track template consistency. If the component is processed and
56 # returns its children template then that template will be
57 # re-parsed (or pulled from the cache).
58 children: list[TNode] = field(default_factory=list)
61type OpenTag = OpenTElement | OpenTFragment | OpenTComponent
64def configure_source_tracker(
65 template: Template,
66 make_placeholder_config: Callable[
67 [], PlaceholderConfig
68 ] = default_make_placeholder_config,
69) -> SourceTracker:
70 config = make_placeholder_config()
71 return SourceTracker(
72 template=template, placeholders=PlaceholderState(config=config)
73 )
76@dataclass
77class SourceTracker:
78 """
79 Iterator of template parts that adds placeholders to interpolations.
80 """
82 template: Template
84 placeholders: PlaceholderState
86 index: int = -1
87 " Unified template index that moves over interpolations and strings. "
89 def __iter__(self):
90 #
91 # @NOTE: This iterator is only meant to be used once since we track
92 # placeholders both by adding them and letting the user remove them
93 # with calls to `remove_placeholders()`.
94 return self
96 def __next__(self):
97 if self.index < 2 * len(self.template.strings) - 2:
98 self.index += 1
99 if self.index % 2 == 0:
100 return self.template.strings[self.index // 2]
101 else:
102 return self.placeholders.add_placeholder((self.index - 1) // 2)
103 else:
104 raise StopIteration
106 def remove_placeholders(self, text: str) -> TemplateRef:
107 """
108 Find tracked placeholders in text and mark them as found.
110 @NOTE: Raises if any untracked placeholders are found.
112 If you want to make a TemplateRef without changing state use
113 `self.find_placeholders()`.
114 """
115 return self.placeholders.remove_placeholders(text)
117 def find_placeholders(self, text: str) -> TemplateRef:
118 """
119 Find all placeholders without affecting tracking.
120 """
121 return self.placeholders.config.find_placeholders(text)
123 def has_placeholders(self) -> bool:
124 """
125 Determine if known placeholders still remain.
126 """
127 return not self.placeholders.is_empty
129 def get_expression(
130 self, i_index: int, fallback_prefix: str = "interpolation"
131 ) -> str:
132 """
133 Resolve an interpolation index to its original expression for error messages.
134 Falls back to a synthetic expression if the original is empty.
135 """
136 ip = self.template.interpolations[i_index]
137 return ip.expression if ip.expression else f"{{{fallback_prefix}-{i_index}}}"
139 def format_starttag(self, i_index: int) -> str:
140 """Format a component start tag for error messages."""
141 return self.get_expression(i_index, fallback_prefix="component-starttag")
144class TemplateParser(HTMLParser):
145 root: OpenTFragment
146 stack: list[OpenTag]
147 source: SourceTracker | None
149 def __init__(self, *, convert_charrefs: bool = True):
150 # This calls HTMLParser.reset() which we override to set up our state.
151 super().__init__(convert_charrefs=convert_charrefs)
153 # ------------------------------------------
154 # Parse state helpers
155 # ------------------------------------------
157 def get_parent(self) -> OpenTag:
158 """Return the current parent node to which new children should be added."""
159 return self.stack[-1] if self.stack else self.root
161 def append_child(self, child: TNode) -> None:
162 parent = self.get_parent()
163 parent.children.append(child)
165 # ------------------------------------------
166 # Attribute Helpers
167 # ------------------------------------------
169 def make_tattr(self, attr: HTMLAttribute) -> TAttribute:
170 """Build a TAttribute from a raw attribute tuple."""
171 source = self.get_source()
173 name, value = attr
175 name_ref = source.remove_placeholders(name)
176 value_ref = source.remove_placeholders(value) if value is not None else None
178 if name_ref.is_literal:
179 if value_ref is None or value_ref.is_literal:
180 return TLiteralAttribute(name=name, value=value)
181 elif value_ref.is_singleton:
182 return TInterpolatedAttribute(
183 name=name, value_i_index=value_ref.i_indexes[0]
184 )
185 else:
186 return TTemplatedAttribute(name=name, value_ref=value_ref)
187 if value_ref is not None:
188 raise ValueError(
189 "Attribute names cannot contain interpolations if the value is also interpolated."
190 )
191 if not name_ref.is_singleton:
192 raise ValueError(
193 "Spread attributes must have exactly one interpolation in the name."
194 )
195 return TSpreadAttribute(i_index=name_ref.i_indexes[0])
197 def make_tattrs(self, attrs: Sequence[HTMLAttribute]) -> tuple[TAttribute, ...]:
198 """Build TAttributes from raw attribute tuples."""
199 return tuple(self.make_tattr(attr) for attr in attrs)
201 # ------------------------------------------
202 # Tag Helpers
203 # ------------------------------------------
205 def make_open_tag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> OpenTag:
206 """Build an OpenTag from a raw tag and attribute tuples."""
207 source = self.get_source()
209 tag_ref = source.remove_placeholders(tag)
211 if tag_ref.is_literal:
212 return OpenTElement(tag=tag, attrs=self.make_tattrs(attrs))
214 if not tag_ref.is_singleton:
215 raise ValueError(
216 "Component element tags must have exactly one interpolation."
217 )
219 # HERE BE DRAGONS: the interpolation at i_index should be a
220 # component callable. We do not check this in the parser, instead
221 # relying on higher layers to validate types and render correctly.
222 i_index = tag_ref.i_indexes[0]
224 # @NOTE: This must be called when the tag is handled since it is
225 # populated based on the most recently finished start tag. Otherwise
226 # the value will be out of sync.
227 starttag_ref = self.get_starttag_ref()
229 # The starting s_index of the component's children template. Note that
230 # this string either contains ">" or " />". It might not be
231 # i_index + 1 because attributes WITHIN the component's tag might
232 # contain interpolations causing the i_index (and s_index) to advance
233 # arbitrarily. So we start with the "last" `i_index` in the starttag
234 # and advance to the next string.
235 children_start_s_index = starttag_ref.i_indexes[-1] + 1
237 # @NOTE: The last string should terminate the starttag and end with ">"
238 # So this length is the offset from the last interpolation to the start
239 # of the children's leading string.
240 offset_into_children_start_s = len(starttag_ref.strings[-1])
242 return OpenTComponent(
243 start_i_index=i_index,
244 children_start_s_index=children_start_s_index,
245 offset_into_children_start_s=offset_into_children_start_s,
246 attrs=self.make_tattrs(attrs),
247 )
249 def finalize_tag(
250 self, open_tag: OpenTag, endtag_i_index: int | None = None
251 ) -> TNode:
252 """Finalize an OpenTag into a TNode."""
253 match open_tag:
254 case OpenTElement(tag=tag, attrs=attrs, children=children):
255 return TElement(tag=tag, attrs=attrs, children=tuple(children))
256 case OpenTFragment(children=children):
257 return TFragment(children=tuple(children))
258 case OpenTComponent(
259 start_i_index=start_i_index,
260 children_start_s_index=children_start_s_index,
261 offset_into_children_start_s=offset_into_children_start_s,
262 attrs=attrs,
263 ):
264 children_ref = self.extract_component_children_ref(
265 start_i_index=start_i_index,
266 endtag_i_index=endtag_i_index,
267 children_start_s_index=children_start_s_index,
268 offset_into_children_start_s=offset_into_children_start_s,
269 template=self.get_source().template,
270 )
271 return TComponent(
272 start_i_index=start_i_index,
273 end_i_index=endtag_i_index,
274 children_ref=children_ref,
275 attrs=attrs,
276 )
278 def extract_component_children_ref(
279 self,
280 start_i_index: int,
281 endtag_i_index: int | None,
282 children_start_s_index: int,
283 offset_into_children_start_s: int,
284 template: Template,
285 ) -> TemplateRef:
286 """
287 Extract the component children template from the entire template.
289 We use this template as a "key" into the cache to get the TNode tree.
290 """
291 if start_i_index != endtag_i_index and endtag_i_index is not None:
292 # CASE: <{Comp}>...</{Comp}> or <{Comp}></{Comp}>
294 # Use the interpolation index of the callable in the closing tag
295 # preceding "string" index is always the same as an interpolation index
296 # The "string" should look like this: "...</"
297 children_end_s_index = endtag_i_index
298 # Offset past the trailing part of the component's start tag to get to
299 # where the first "string" of the children's template starts.
300 leading = template.strings[children_start_s_index][
301 offset_into_children_start_s:
302 ]
303 if children_start_s_index == children_end_s_index:
304 # CASE: Entire children template is a string, leading == trailing.
305 leading = leading[: leading.rfind("</")]
306 children_ref = TemplateRef(strings=(leading,), i_indexes=())
307 else:
308 # CASE: Children template contains interpolations so the trailing
309 # "string" will not be the same as the leading "string".
310 trailing = template.strings[children_end_s_index]
311 trailing = trailing[: trailing.rfind("</")]
312 children_ref = TemplateRef(
313 strings=(
314 leading,
315 *template.strings[
316 children_start_s_index + 1 : children_end_s_index
317 ],
318 trailing,
319 ),
320 i_indexes=tuple(
321 range(children_start_s_index, children_end_s_index)
322 ),
323 )
324 else:
325 # CASE: <{Comp} /> -- no children template
326 children_ref = TemplateRef(strings=("",), i_indexes=())
327 return children_ref
329 def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None:
330 """Validate that closing tag matches open tag. Return component end index if applicable."""
331 source = self.get_source()
332 tag_ref = source.remove_placeholders(tag)
334 match open_tag:
335 case OpenTElement():
336 if not tag_ref.is_literal:
337 raise ValueError(
338 f"Component closing tag found for element <{open_tag.tag}>."
339 )
340 if tag != open_tag.tag:
341 raise ValueError(
342 f"Mismatched closing tag </{tag}> for element <{open_tag.tag}>."
343 )
344 return None
346 case OpenTFragment():
347 raise NotImplementedError("We do not support anonymous fragments.")
349 case OpenTComponent(start_i_index=start_i_index):
350 if tag_ref.is_literal:
351 raise ValueError(
352 f"Mismatched closing tag </{tag}> for component starting at {source.format_starttag(start_i_index)}."
353 )
354 if not tag_ref.is_singleton:
355 raise ValueError(
356 "Component end tags must have exactly one interpolation."
357 )
358 # HERE BE DRAGONS: the interpolation at end_i_index shuld be a
359 # component callable that matches the start tag. We do not check
360 # any of this in the parser, instead relying on higher layers.
361 return tag_ref.i_indexes[0]
363 def get_starttag_ref(self) -> TemplateRef:
364 """
365 Wrap get_starttag_text and just raise if None is returned.
366 Do this so we don't guard for `None` everywhere.
367 """
368 source = self.get_source()
369 starttag_text = self.get_starttag_text()
370 if starttag_text is None:
371 raise AssertionError("Expected the parser to have starttag_text set.")
372 return source.find_placeholders(starttag_text)
374 # ------------------------------------------
375 # HTMLParser tag callbacks
376 # ------------------------------------------
378 def handle_starttag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None:
379 open_tag = self.make_open_tag(tag, attrs)
380 if isinstance(open_tag, OpenTElement) and open_tag.tag in VOID_ELEMENTS:
381 final_tag = self.finalize_tag(open_tag)
382 self.append_child(final_tag)
383 else:
384 self.stack.append(open_tag)
386 def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None:
387 """Dispatch a self-closing tag, `<tag />` to specialized handlers."""
388 open_tag = self.make_open_tag(tag, attrs)
389 final_tag = self.finalize_tag(open_tag)
390 self.append_child(final_tag)
392 def handle_endtag(self, tag: str) -> None:
393 if not self.stack:
394 raise ValueError(f"Unexpected closing tag </{tag}> with no open tag.")
396 open_tag = self.stack.pop()
397 endtag_i_index = self.validate_end_tag(tag, open_tag)
398 final_tag = self.finalize_tag(open_tag, endtag_i_index)
399 self.append_child(final_tag)
401 # ------------------------------------------
402 # HTMLParser other callbacks
403 # ------------------------------------------
405 def handle_data(self, data: str) -> None:
406 source = self.get_source()
407 ref = source.remove_placeholders(data)
408 parent = self.get_parent()
409 if parent.children and isinstance(parent.children[-1], TText):
410 parent.children[-1] = TText(
411 ref=combine_template_refs(parent.children[-1].ref, ref)
412 )
413 else:
414 self.append_child(TText(ref=ref))
416 def handle_comment(self, data: str) -> None:
417 source = self.get_source()
418 ref = source.remove_placeholders(data)
419 comment = TComment(ref)
420 self.append_child(comment)
422 def handle_decl(self, decl: str) -> None:
423 source = self.get_source()
424 ref = source.remove_placeholders(decl)
425 if not ref.is_literal:
426 raise ValueError("Interpolations are not allowed in declarations.")
427 elif decl.upper().startswith("DOCTYPE "):
428 doctype_content = decl[7:].strip()
429 doctype = TDocumentType(doctype_content)
430 self.append_child(doctype)
431 else:
432 raise NotImplementedError(
433 "Only well formed DOCTYPE declarations are currently supported."
434 )
436 def reset(self):
437 super().reset()
438 self.root = OpenTFragment()
439 self.stack = []
440 self.source = None
442 def close(self) -> None:
443 if self.waiting_for_data():
444 # We apply heuristics here to try to guess why the parser didn't finish.
445 if self.rawdata.count('"') % 2 == 1 or self.rawdata.count("'") % 2 == 1:
446 raise ValueError(
447 "Parser expects more data, maybe you left an attribute quote unclosed?"
448 )
449 else:
450 raise ValueError(
451 "Parser expects more data, is the template valid html?"
452 )
453 if self.stack:
454 raise ValueError("Invalid HTML structure: unclosed tags remain.")
455 if self.source and self.source.has_placeholders():
456 raise ValueError("Some placeholders were never resolved.")
457 super().close()
459 def waiting_for_data(self):
460 return len(self.rawdata) > 0
462 # ------------------------------------------
463 # Getting the parsed node tree
464 # ------------------------------------------
466 def get_tnode(self) -> TNode:
467 """Get the Node tree parsed from the input HTML."""
468 # TODO: consider always returning a TTag?
469 if len(self.root.children) > 1:
470 # The parse structure results in multiple root elements, so we
471 # return a Fragment to hold them all.
472 return self.finalize_tag(self.root)
473 elif len(self.root.children) == 1:
474 # The parse structure results in a single root element, so we
475 # return that element directly. This will be a non-Fragment Node.
476 return self.root.children[0]
477 else:
478 # Special case: the parse structure is empty; we treat
479 # this as an empty document fragment.
480 # CONSIDER: or as an empty text node?
481 return self.finalize_tag(self.root)
483 # ------------------------------------------
484 # Feeding and parsing
485 # ------------------------------------------
487 def get_source(self) -> SourceTracker:
488 if self.source is None:
489 raise AssertionError("Source has not been initialized.")
490 return self.source
492 def track_source(self, template: Template) -> SourceTracker:
493 if self.source:
494 raise AssertionError("Did you forget to call reset?")
495 source = self.source = configure_source_tracker(template)
496 return source
498 def feed_template(self, template: Template) -> None:
499 """Feed a Template's content to the parser."""
500 for content in self.track_source(template):
501 self.feed(content)
503 @staticmethod
504 def parse(t: Template) -> TNode:
505 """
506 Parse a Template containing valid HTML and substitutions and return
507 a TNode tree representing its structure. This cachable structure can later
508 be resolved against actual interpolation values to produce a Node tree.
509 """
510 parser = TemplateParser()
511 parser.feed_template(t)
512 parser.close()
513 return parser.get_tnode()