Coverage for tdom/parser.py: 98%
248 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-19 21:37 +0000
1from collections.abc import Callable, Sequence
2from dataclasses import dataclass, field
3from html.parser import HTMLParser
4from string.templatelib import Template
6from .exc import TemplatingError
7from .htmlspec import VOID_ELEMENTS
8from .parser_utils import (
9 HTMLAttribute,
10 ParserPositionTranslator,
11 make_parser_pos_translator,
12)
13from .placeholders import (
14 PlaceholderConfig,
15 PlaceholderState,
16)
17from .placeholders import (
18 make_placeholder_config as default_make_placeholder_config,
19)
20from .source import LinePosition, SourceReader
21from .template_utils import PartPosition, TemplateRef, TemplateSpan
22from .tnodes import (
23 TagSourceInfo,
24 TAttribute,
25 TComment,
26 TComponent,
27 TDocumentType,
28 TElement,
29 TFragment,
30 TInterpolatedAttribute,
31 TLiteralAttribute,
32 TNode,
33 TSpreadAttribute,
34 TTemplatedAttribute,
35 TText,
36 TTree,
37)
40class ParsingError(TemplatingError):
41 pass
44class AttributeParsingError(ParsingError):
45 pass
48@dataclass(frozen=True, slots=True)
49class OpenTagSourceInfo:
50 """
51 Retained tag information from the parsed source meant for error reporting.
53 @NOTE: This is an temporary structure that will be finalized when the
54 tag is closed.
55 """
57 starttag_span: TemplateSpan
58 """Source span occupied by the start tag."""
59 startend: bool
60 """Was parsed as startend tag, ie. <tag />."""
62 @property
63 def starttag_pos(self) -> PartPosition:
64 """Template part position where the start tag begins."""
65 return self.starttag_span.start
67 def close(self, endtag_pos: PartPosition | None = None) -> TagSourceInfo:
68 return TagSourceInfo(
69 starttag_span=self.starttag_span,
70 startend=self.startend,
71 endtag_pos=endtag_pos,
72 )
75@dataclass
76class OpenTElement:
77 tag: str
78 attrs: tuple[TAttribute, ...]
79 source_pos: PartPosition
80 sinfo: OpenTagSourceInfo
81 children: list[TNode] = field(default_factory=list)
84@dataclass
85class OpenTFragment:
86 children: list[TNode] = field(default_factory=list)
89@dataclass
90class OpenTComponent:
91 start_i_index: int
92 children_start: PartPosition
93 """Source position where the component's children start."""
94 attrs: tuple[TAttribute, ...]
95 source_pos: PartPosition
96 sinfo: OpenTagSourceInfo
97 # @NOTE: The `children` are discarded after parsing and are just used to
98 # track template consistency. If the component is processed and
99 # returns its children template then that template will be
100 # re-parsed (or pulled from the cache).
101 children: list[TNode] = field(default_factory=list)
104type OpenTag = OpenTElement | OpenTComponent
107def configure_source_tracker(
108 template: Template,
109 make_placeholder_config: Callable[
110 [], PlaceholderConfig
111 ] = default_make_placeholder_config,
112) -> SourceTracker:
113 """
114 Configure and return source tracker with its subcomponents.
115 """
116 config = make_placeholder_config()
117 return SourceTracker(
118 template=template,
119 placeholders=PlaceholderState(config=config),
120 parser_pos_translator=make_parser_pos_translator(template, config),
121 )
124@dataclass
125class SourceTracker:
126 """
127 Iterator of template parts that adds placeholders to interpolations.
128 """
130 template: Template
132 placeholders: PlaceholderState
134 parser_pos_translator: ParserPositionTranslator
135 """Translator from parser position to template part position."""
137 index: int = -1
138 """Unified template index that moves over interpolations and strings."""
140 def __iter__(self):
141 #
142 # @NOTE: This iterator is only meant to be used once since we track
143 # placeholders both by adding them and letting the user remove them
144 # with calls to `remove_placeholders()`.
145 return self
147 def __next__(self):
148 if self.index < 2 * len(self.template.strings) - 2:
149 self.index += 1
150 if self.index % 2 == 0:
151 return self.template.strings[self.index // 2]
152 else:
153 return self.placeholders.add_placeholder((self.index - 1) // 2)
154 else:
155 raise StopIteration
157 def remove_placeholders(self, text: str) -> TemplateRef:
158 """
159 Find tracked placeholders in text and mark them as found.
161 @NOTE: Raises if any untracked placeholders are found.
163 If you want to make a TemplateRef without changing state use
164 `self.find_placeholders()`.
165 """
166 return self.placeholders.remove_placeholders(text)
168 def find_placeholders(self, text: str) -> TemplateRef:
169 """
170 Find all placeholders without affecting tracking.
171 """
172 return self.placeholders.config.find_placeholders(text)
174 def has_placeholders(self) -> bool:
175 """
176 Determine if known placeholders still remain.
177 """
178 return not self.placeholders.is_empty
180 def translate_parser_pos(self, raw_parser_pos: LinePosition) -> PartPosition:
181 """
182 Translate a parser position to a part position within the template.
183 """
184 return self.parser_pos_translator.translate(raw_parser_pos)
186 def translate_parser_span(
187 self,
188 raw_parser_start: LinePosition,
189 raw_parser_length: int,
190 ) -> TemplateSpan:
191 """Translate a span in parser input into a span in the source template."""
192 return self.parser_pos_translator.translate_span(
193 raw_parser_start, raw_parser_length
194 )
197def make_error_helper(parser: TemplateParser) -> ParsingErrorHelper:
198 """
199 Factory that creates an error helper for the parser.
201 @NOTE: This should only be used when an exception is already being
202 generated.
203 """
204 return ParsingErrorHelper(
205 reader=SourceReader(template=parser.get_source().template),
206 sinfo_table=parser.sinfo_table.copy(),
207 )
210@dataclass()
211class ParsingErrorHelper:
212 """
213 Helps the parser include extra info in parsing errors.
215 This helper *should not* decide what error to raise.
217 This helper *should* help to:
218 - include possible reasons (did you forget to close a quote?)
219 - include original template source (the start tag, etc.)
220 - include position info (line #, etc.)
221 """
223 reader: SourceReader
224 """ Used to read original template fragments and/or interpolation values. """
226 sinfo_table: dict[PartPosition, TagSourceInfo]
227 """ Source info mapping, copied from parser. """
229 def make_mismatch_error(
230 self,
231 starttag_sinfo: OpenTagSourceInfo,
232 endtag_ref: TemplateRef,
233 endtag_pos: PartPosition,
234 ) -> ParsingError:
235 starttag_repr = self.reader.span_to_repr(starttag_sinfo.starttag_span)
236 starttag_pos_msg = self.reader.make_template_pos_msg(
237 starttag_sinfo.starttag_pos
238 )
239 endtag_repr = self.reader.ref_to_repr(endtag_ref)
240 endtag_pos_msg = self.reader.make_template_pos_msg(endtag_pos)
241 return ParsingError(
242 f"Mismatched closing tag </{endtag_repr}> at {endtag_pos_msg} for {starttag_repr} at {starttag_pos_msg}."
243 )
245 def make_malformed_endtag_error(
246 self, endtag_ref: TemplateRef, endtag_pos: PartPosition
247 ) -> ParsingError:
248 endtag_repr = self.reader.ref_to_repr(endtag_ref)
249 endtag_pos_msg = self.reader.make_template_pos_msg(endtag_pos)
250 return ParsingError(
251 f"Component end tags must have exactly one interpolation, {endtag_repr} at {endtag_pos_msg}."
252 )
254 def make_unexpected_endtag_error(
255 self, endtag_ref: TemplateRef, endtag_pos: PartPosition
256 ) -> ParsingError:
257 endtag_repr = self.reader.ref_to_repr(endtag_ref)
258 endtag_pos_msg = self.reader.make_template_pos_msg(endtag_pos)
259 return ParsingError(
260 f"Unexpected closing tag </{endtag_repr}> with no open tag, {endtag_pos_msg}."
261 )
263 def make_unclosed_starttag_error(self, parent: OpenTag):
264 starttag_repr = self.reader.span_to_repr(parent.sinfo.starttag_span)
265 pos_msg = self.reader.make_template_pos_msg(parent.source_pos)
266 return ParsingError(
267 f"Invalid HTML structure: unclosed tag {starttag_repr} at {pos_msg}."
268 )
271class TemplateParser(HTMLParser):
272 root: OpenTFragment
273 stack: list[OpenTag]
274 source: SourceTracker | None
276 sinfo_table: dict[PartPosition, TagSourceInfo]
277 """Tags with more source info than just a position are tracked in this mapping."""
279 def __init__(self, *, convert_charrefs: bool = True):
280 # This calls HTMLParser.reset() which we override to set up our state.
281 super().__init__(convert_charrefs=convert_charrefs)
283 # ------------------------------------------
284 # Parse state helpers
285 # ------------------------------------------
287 def get_parent(self) -> OpenTag | OpenTFragment:
288 """Return the current parent node to which new children should be added."""
289 return self.stack[-1] if self.stack else self.root
291 def append_child(self, child: TNode) -> None:
292 parent = self.get_parent()
293 parent.children.append(child)
295 def get_parser_pos(self) -> LinePosition:
296 """
297 Get the current position of the parser.
299 @NOTE: This position is relative to text embedded with placeholders but
300 can be translated back to the position within the original template.
301 Since it *IS* relative to placeholders, ie. "SLOTS", this position is
302 unique across a "family" of templates with the same structure.
303 """
304 line, offset = self.getpos()
305 return LinePosition(line=line, offset=offset)
307 def get_source_pos(self, parser_pos: LinePosition | None = None) -> PartPosition:
308 """Translate the parser position into a part position in the source template."""
309 source = self.get_source()
310 return source.translate_parser_pos(
311 self.get_parser_pos() if parser_pos is None else parser_pos
312 )
314 # ------------------------------------------
315 # Attribute Helpers
316 # ------------------------------------------
318 def make_tattr(self, attr: HTMLAttribute) -> TAttribute:
319 """Build a TAttribute from a raw attribute tuple."""
320 source = self.get_source()
322 name, value = attr
324 name_ref = source.remove_placeholders(name)
325 value_ref = source.remove_placeholders(value) if value is not None else None
327 if name_ref.is_literal:
328 if value_ref is None or value_ref.is_literal:
329 return TLiteralAttribute(name=name, value=value)
330 elif value_ref.is_singleton:
331 return TInterpolatedAttribute(
332 name=name, value_i_index=value_ref.i_start
333 )
334 else:
335 return TTemplatedAttribute(name=name, value_ref=value_ref)
336 if value_ref is not None:
337 raise AttributeParsingError(
338 "Attribute names cannot contain interpolations if the value is also interpolated."
339 )
340 if not name_ref.is_singleton:
341 raise AttributeParsingError(
342 "Spread attributes must have exactly one interpolation in the name."
343 )
344 return TSpreadAttribute(i_index=name_ref.i_start)
346 def make_tattrs(self, attrs: Sequence[HTMLAttribute]) -> tuple[TAttribute, ...]:
347 """Build TAttributes from raw attribute tuples."""
348 return tuple(self.make_tattr(attr) for attr in attrs)
350 # ------------------------------------------
351 # Tag Helpers
352 # ------------------------------------------
354 def make_open_tag(
355 self,
356 tag: str,
357 attrs: Sequence[HTMLAttribute],
358 startend: bool = False,
359 ) -> OpenTag:
360 """Build an OpenTag from a raw tag and attribute tuples."""
361 source = self.get_source()
363 tag_ref = source.remove_placeholders(tag)
365 if tag_ref.is_literal:
366 source_pos = self.get_source_pos()
367 return OpenTElement(
368 tag=tag,
369 attrs=self.make_tattrs(attrs),
370 sinfo=OpenTagSourceInfo(
371 starttag_span=self.get_starttag_span(),
372 startend=startend,
373 ),
374 source_pos=source_pos,
375 )
377 if not tag_ref.is_singleton:
378 raise ParsingError(
379 "Component element tags must have exactly one interpolation."
380 )
382 # HERE BE DRAGONS: the interpolation at i_index should be a
383 # component callable. We do not check this in the parser, instead
384 # relying on higher layers to validate types and render correctly.
385 i_index = tag_ref.i_start
387 # This must be called while handling the tag because HTMLParser retains
388 # only the most recently parsed start tag text.
389 starttag_span = self.get_starttag_span()
390 source_pos = starttag_span.start
392 return OpenTComponent(
393 start_i_index=i_index,
394 children_start=starttag_span.stop,
395 attrs=self.make_tattrs(attrs),
396 source_pos=source_pos,
397 sinfo=OpenTagSourceInfo(
398 starttag_span=starttag_span,
399 startend=startend,
400 ),
401 )
403 def finalize_tag(
404 self,
405 open_tag: OpenTag,
406 endtag_i_index: int | None = None,
407 endtag_pos: PartPosition | None = None,
408 ) -> TNode:
409 """Finalize an OpenTag into a TNode."""
410 match open_tag:
411 case OpenTElement(
412 tag=tag,
413 attrs=attrs,
414 children=children,
415 source_pos=source_pos,
416 sinfo=sinfo,
417 ):
418 source_pos = (
419 open_tag.source_pos
420 ) # Re-assignment for ty regression in 0.0.59
421 self.sinfo_table[source_pos] = sinfo.close(endtag_pos=endtag_pos)
422 return TElement(
423 tag=tag,
424 attrs=attrs,
425 children=tuple(children),
426 source_pos=source_pos,
427 )
428 case OpenTComponent(
429 start_i_index=start_i_index,
430 children_start=children_start,
431 attrs=attrs,
432 source_pos=source_pos,
433 sinfo=sinfo,
434 ):
435 children_span = (
436 TemplateSpan(start=children_start, stop=endtag_pos)
437 if endtag_pos is not None
438 else None
439 )
440 self.sinfo_table[source_pos] = sinfo.close(endtag_pos=endtag_pos)
441 return TComponent(
442 start_i_index=start_i_index,
443 end_i_index=endtag_i_index,
444 children_span=children_span,
445 attrs=attrs,
446 source_pos=source_pos,
447 )
449 def validate_end_tag(self, tag: str, open_tag: OpenTag) -> int | None:
450 """Validate that closing tag matches open tag. Return component end index if applicable."""
451 source = self.get_source()
452 tag_ref = source.placeholders.remove_placeholders(tag)
454 match open_tag:
455 case OpenTElement():
456 if tag_ref.is_singleton or (tag_ref.is_literal and tag != open_tag.tag):
457 raise make_error_helper(self).make_mismatch_error(
458 open_tag.sinfo, tag_ref, self.get_source_pos()
459 )
460 elif not tag_ref.is_singleton and not tag_ref.is_literal:
461 raise make_error_helper(self).make_malformed_endtag_error(
462 tag_ref, self.get_source_pos()
463 )
464 return None
465 case OpenTComponent():
466 if tag_ref.is_literal:
467 raise make_error_helper(self).make_mismatch_error(
468 open_tag.sinfo, tag_ref, self.get_source_pos()
469 )
470 if not tag_ref.is_singleton:
471 raise make_error_helper(self).make_malformed_endtag_error(
472 tag_ref, self.get_source_pos()
473 )
474 return tag_ref.i_start
476 def get_starttag_span(self) -> TemplateSpan:
477 """Return the source span occupied by the current start tag."""
478 starttag_text = self.get_starttag_text()
479 if starttag_text is None:
480 raise AssertionError("Expected the parser to have starttag_text set.")
482 source = self.get_source()
483 line_pos = self.get_parser_pos()
484 return source.translate_parser_span(line_pos, len(starttag_text))
486 # ------------------------------------------
487 # HTMLParser tag callbacks
488 # ------------------------------------------
490 def handle_starttag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None:
491 open_tag = self.make_open_tag(tag, attrs)
492 if isinstance(open_tag, OpenTElement) and open_tag.tag in VOID_ELEMENTS:
493 final_tag = self.finalize_tag(open_tag)
494 self.append_child(final_tag)
495 else:
496 self.stack.append(open_tag)
498 def handle_startendtag(self, tag: str, attrs: Sequence[HTMLAttribute]) -> None:
499 """Dispatch a self-closing tag, `<tag />` to specialized handlers."""
500 open_tag = self.make_open_tag(tag, attrs, startend=True)
501 final_tag = self.finalize_tag(open_tag)
502 self.append_child(final_tag)
504 def handle_endtag(self, tag: str) -> None:
505 endtag_pos = self.get_source_pos()
506 if not self.stack:
507 source = self.get_source()
508 endtag_ref = source.find_placeholders(tag)
509 if endtag_ref.is_literal or endtag_ref.is_singleton:
510 raise make_error_helper(self).make_unexpected_endtag_error(
511 endtag_ref, endtag_pos
512 )
513 else:
514 raise make_error_helper(self).make_malformed_endtag_error(
515 endtag_ref, endtag_pos
516 )
517 open_tag = self.stack.pop()
518 endtag_i_index = self.validate_end_tag(tag, open_tag)
519 final_tag = self.finalize_tag(
520 open_tag,
521 endtag_i_index=endtag_i_index,
522 endtag_pos=endtag_pos,
523 )
524 self.append_child(final_tag)
526 # ------------------------------------------
527 # HTMLParser other callbacks
528 # ------------------------------------------
530 def handle_data(self, data: str) -> None:
531 source = self.get_source()
532 ref = source.remove_placeholders(data)
533 parent = self.get_parent()
534 if parent.children and isinstance(parent.children[-1], TText):
535 prior_text = parent.children[-1]
536 parent.children[-1] = TText(
537 ref=prior_text.ref.concat(ref),
538 # Keep starting position of the prior text
539 source_pos=prior_text.source_pos,
540 )
541 else:
542 self.append_child(TText(ref=ref, source_pos=self.get_source_pos()))
544 def handle_comment(self, data: str) -> None:
545 source = self.get_source()
546 ref = source.remove_placeholders(data)
547 comment = TComment(ref=ref, source_pos=self.get_source_pos())
548 self.append_child(comment)
550 def handle_decl(self, decl: str) -> None:
551 source = self.get_source()
552 ref = source.remove_placeholders(decl)
553 if not ref.is_literal:
554 raise ParsingError("Interpolations are not allowed in declarations.")
555 elif decl.upper().startswith("DOCTYPE "):
556 doctype_content = decl[7:].strip()
557 doctype = TDocumentType(doctype_content, source_pos=self.get_source_pos())
558 self.append_child(doctype)
559 else:
560 raise ParsingError(
561 "Only well formed DOCTYPE declarations are currently supported."
562 )
564 def reset(self):
565 super().reset()
566 self.root = OpenTFragment()
567 self.stack = []
568 self.source = None
569 self.sinfo_table = {}
571 def close(self) -> None:
572 super().close()
573 if self.stack:
574 raise make_error_helper(self).make_unclosed_starttag_error(self.stack[-1])
575 if self.source and self.source.has_placeholders():
576 raise ParsingError("Some placeholders were never resolved.")
578 # ------------------------------------------
579 # Getting the parsed node tree
580 # ------------------------------------------
582 def get_tnode(self) -> TNode:
583 """Get the Node tree parsed from the input HTML."""
584 if len(self.root.children) > 1:
585 # The parse structure results in multiple root elements, so we
586 # return a Fragment to hold them all.
587 return TFragment(children=tuple(self.root.children))
588 elif len(self.root.children) == 1:
589 # The parse structure results in a single root element, so we
590 # return that element directly. This will be a non-Fragment Node.
591 return self.root.children[0]
592 else:
593 # Special case: the parse structure is empty; we treat
594 # this as an empty document fragment.
595 # CONSIDER: or as an empty text node?
596 return TFragment(children=())
598 def get_ttree(self) -> TTree:
599 return TTree(
600 self.get_tnode(),
601 sinfos=tuple(self.sinfo_table.values()),
602 )
604 # ------------------------------------------
605 # Feeding and parsing
606 # ------------------------------------------
608 def get_source(self) -> SourceTracker:
609 if self.source is None:
610 raise AssertionError("Source has not been initialized.")
611 return self.source
613 def track_source(self, template: Template) -> SourceTracker:
614 if self.source:
615 raise AssertionError("Did you forget to call reset?")
616 source = self.source = configure_source_tracker(template)
617 return source
619 def feed_template(self, template: Template) -> None:
620 """Feed a Template's content to the parser."""
621 for content in self.track_source(template):
622 self.feed(content)
624 @staticmethod
625 def parse(t: Template) -> TTree:
626 """
627 Parse a Template containing valid HTML and substitutions and return
628 a TTree representing its structure. This cachable structure can later
629 be resolved against actual interpolation values to produce HTML.
630 """
631 parser = TemplateParser()
632 parser.feed_template(t)
633 parser.close()
634 return parser.get_ttree()