👌 Fix quadratic inline tokenization on runs of special characters

markdown_it/common/html_re.py

@@ -21,7 +21,9 @@
 cdata = "<!\\[CDATA\\[[\\s\\S]*?\\]\\]>"
 
 HTML_TAG_RE = re.compile(
-    "^(?:"
+    # No leading `^`: applied via `.match(src, pos)`, which anchors at `pos`
+    # without slicing `src[pos:]` (an O(len) copy per call, quadratic over a run).
+    "(?:"
     + open_tag
     + "|"
     + close_tag

markdown_it/parser_inline.py

@@ -197,7 +197,7 @@ def tokenize(self, state: StateInline) -> None:
                     break
                 continue
 
-            state.pending += state.src[state.pos]
+            state.append_pending(state.src[state.pos])
             state.pos += 1
 
         if state.pending:

markdown_it/rules_inline/backticks.py

@@ -25,7 +25,7 @@ def backtick(state: StateInline, silent: bool) -> bool:
 
     if state.backticksScanned and state.backticks.get(openerLength, 0) <= start:
         if not silent:
-            state.pending += marker
+            state.append_pending(marker)
         state.pos += openerLength
         return True
 
@@ -67,6 +67,6 @@ def backtick(state: StateInline, silent: bool) -> bool:
     state.backticksScanned = True
 
     if not silent:
-        state.pending += marker
+        state.append_pending(marker)
     state.pos += openerLength
     return True

markdown_it/rules_inline/entity.py

@@ -5,8 +5,11 @@
 from ..common.utils import fromCodePoint, isValidEntityCode
 from .state_inline import StateInline
 
-DIGITAL_RE = re.compile(r"^&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));", re.IGNORECASE)
-NAMED_RE = re.compile(r"^&([a-z][a-z0-9]{1,31});", re.IGNORECASE)
+# NB: no leading `^` -- these are applied via `.match(src, pos)`, which already
+# anchors at `pos`.  Anchoring this way avoids building `src[pos:]` on every
+# `&`, which is an O(len) copy per call and makes long runs quadratic.
+DIGITAL_RE = re.compile(r"&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));", re.IGNORECASE)
+NAMED_RE = re.compile(r"&([a-z][a-z0-9]{1,31});", re.IGNORECASE)
 
 
 def entity(state: StateInline, silent: bool) -> bool:
@@ -20,7 +23,7 @@ def entity(state: StateInline, silent: bool) -> bool:
         return False
 
     if state.src[pos + 1] == "#":
-        if match := DIGITAL_RE.search(state.src[pos:]):
+        if match := DIGITAL_RE.match(state.src, pos):
             if not silent:
                 match1 = match.group(1)
                 code = (
@@ -40,7 +43,7 @@ def entity(state: StateInline, silent: bool) -> bool:
             return True
 
     else:
-        if (match := NAMED_RE.search(state.src[pos:])) and match.group(1) in entities:
+        if (match := NAMED_RE.match(state.src, pos)) and match.group(1) in entities:
             if not silent:
                 token = state.push("text_special", "", 0)
                 token.content = entities[match.group(1)]

markdown_it/rules_inline/html_inline.py

@@ -26,7 +26,7 @@ def html_inline(state: StateInline, silent: bool) -> bool:
     if ch not in ("!", "?", "/") and not isLetter(ord(ch)):  # /* / */
         return False
 
-    match = HTML_TAG_RE.search(state.src[pos:])
+    match = HTML_TAG_RE.match(state.src, pos)
     if not match:
         return False
 

markdown_it/rules_inline/state_inline.py

@@ -54,7 +54,16 @@ def __init__(
         self.pos = 0
         self.posMax = len(self.src)
         self.level = 0
-        self.pending = ""
+        # `pending` holds literal text not yet flushed to a token.  It is
+        # exposed as a plain `str` (see the property below), but is accumulated
+        # through a list buffer so that appending one character at a time -- the
+        # inline tokenizer's fallback path -- stays amortised O(1).  Appending
+        # to a `str` *attribute* cannot use CPython's in-place concatenation
+        # optimisation (the attribute holds a second reference), so each `+=`
+        # copies the whole string, making long runs of non-markup characters
+        # quadratic.
+        self._pending = ""
+        self._pending_buffer: list[str] = []
         self.pendingLevel = 0
 
         # Stores { start: end } pairs. Useful for backtrack
@@ -81,6 +90,49 @@ def __repr__(self) -> str:
             f"(pos=[{self.pos} of {self.posMax}], token={len(self.tokens)})"
         )
 
+    @property
+    def pending(self) -> str:
+        """Literal text accumulated so far, but not yet flushed to a token."""
+        buffer = self._pending_buffer
+        if buffer:
+            # Move the string into a local and drop the instance's reference
+            # before concatenating.  With a single reference left, CPython
+            # resizes the string in place (amortised O(new chars)) rather
+            # than copying it, so a rule that reads `pending` on every
+            # character (e.g. an attribute-syntax plugin) stays linear.
+            text = self._pending
+            self._pending = ""
+            text += "".join(buffer)
+            buffer.clear()
+            self._pending = text
+        return self._pending
+
+    @pending.setter
+    def pending(self, value: str) -> None:
+        self._pending = value
+        # Assign rather than `.clear()` so the setter also works on an
+        # instance whose `__init__` has not run yet (subclasses that set
+        # `pending` before calling `super().__init__()`), and so a copied
+        # state never shares a buffer with its original.
+        self._pending_buffer = []
+
+    def __copy__(self) -> StateInline:
+        """Shallow copy that does not share the pending text buffer."""
+        text = self.pending  # materialise (and clear) our own buffer first
+        new = self.__class__.__new__(self.__class__)
+        new.__dict__.update(self.__dict__)
+        new._pending = text
+        new._pending_buffer = []
+        return new
+
+    def append_pending(self, text: str) -> None:
+        """Append literal text to `pending`, in amortised O(1) time.
+
+        Prefer this to ``state.pending += text`` on hot paths: it buffers the
+        fragment rather than rebuilding the whole ``pending`` string per call.
+        """
+        self._pending_buffer.append(text)
+
     def pushPending(self) -> Token:
         token = Token("text", "", 0)
         token.content = self.pending

markdown_it/rules_inline/text.py

@@ -16,7 +16,7 @@ def text(state: StateInline, silent: bool) -> bool:
         return False
 
     if not silent:
-        state.pending += state.src[state.pos : pos]
+        state.append_pending(state.src[state.pos : pos])
 
     state.pos = pos
 

tests/test_api/test_main.py

@@ -1,3 +1,5 @@
+import pytest
+
 from markdown_it import MarkdownIt
 from markdown_it.token import Token
 
@@ -365,3 +367,175 @@ def test_text_join_merges_adjacent_text_special_tokens():
     assert len(children_on) == 1
     assert children_on[0].type == "text"
     assert children_on[0].content == "***"
+
+
+def test_long_special_char_runs_render_correctly():
+    """Runs of characters that begin an inline rule but do not form a construct
+    render as literal text, however long the run.
+
+    These inputs were previously tokenised in O(n^2) time (see
+    `test_inline_rules_do_not_slice_remaining_source` and
+    `test_pending_appends_do_not_materialise` for the invariants that keep them
+    linear); this test pins the *output*.
+    """
+    md = MarkdownIt()
+
+    for src, expected in [
+        ("&" * 32, "&" * 32),  # entity rule rejects, falls back to pending
+        ("&" * 8, "&" * 8),  # NAMED_RE still matches
+        ("#" * 8, "#" * 8),  # DIGITAL_RE still matches
+        ("#" * 8, "#" * 8),  # DIGITAL_RE, hex form
+        ("&#" * 8, "&#" * 8),  # `#` branch, no match
+        ("&am" * 8, "&am" * 8),  # named prefix, unterminated
+        ("&nope;" * 4, "&nope;" * 4),  # well-formed but unknown name
+        ("<" * 16, "&lt;" * 16),  # html_inline rejects (html=False anyway)
+        ("{" * 32, "{" * 32),  # no rule at all -> pure pending fallback
+        ("~" * 16, "~" * 16),  # sub-length strikethrough delimiters
+    ]:
+        assert md.renderInline(src) == expected, src
+
+    # Backtick markers also feed `pending`; unclosed runs stay literal.
+    assert md.renderInline("`" * 3 + "a") == "```a"
+    assert md.renderInline("`a``b") == "`a``b"
+    # ...while a matched pair still becomes code.
+    assert md.renderInline("`a`") == "<code>a</code>"
+
+    # Longer runs, including the `html_inline` path (only reachable with
+    # `html=True`; `<a<a...` is never a valid tag, so it renders escaped).
+    assert md.renderInline("&" * 20_000) == "&amp;" * 20_000
+    md_html = MarkdownIt("commonmark", {"html": True})
+    assert md_html.renderInline("<a" * 10_000) == "&lt;a" * 10_000
+
+
+class _SliceCountingStr(str):
+    """A ``str`` that records the length of every slice taken from it."""
+
+    slice_lengths: list[int]
+
+    def __new__(cls, value: str) -> "_SliceCountingStr":
+        self = super().__new__(cls, value)
+        self.slice_lengths = []
+        return self
+
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            start, stop, _ = key.indices(len(self))
+            self.slice_lengths.append(max(0, stop - start))
+        return str.__getitem__(self, key)
+
+
+@pytest.mark.parametrize(
+    "preset,options,src",
+    [
+        ("commonmark", {}, "&" * 5_000),  # entity rule
+        ("commonmark", {}, "&#" * 5_000),  # entity rule, numeric branch
+        ("commonmark", {"html": True}, "<a" * 5_000),  # html_inline rule
+    ],
+    ids=["entity-named", "entity-numeric", "html_inline"],
+)
+def test_inline_rules_do_not_slice_remaining_source(preset, options, src):
+    """Inline rules must not copy the rest of the source on every attempt.
+
+    `entity` and `html_inline` used to match ``^``-anchored regexes against
+    ``state.src[pos:]``, an O(len) copy per ``&`` / ``<`` and therefore O(n^2)
+    over a run.  They now anchor with ``.match(src, pos)``.  A run of 5 000
+    openers previously produced ~5 000 slices of up to 10 000 characters; the
+    only slices left are the ``text`` rule's single-character chunks.
+    """
+    md = MarkdownIt(preset, options)
+    counting = _SliceCountingStr(src)
+    md.inline.parse(counting, md, {}, [])
+    assert max(counting.slice_lengths, default=0) <= 1
+
+
+def test_pending_appends_do_not_materialise():
+    """`append_pending` must be amortised O(1): it buffers fragments and only
+    builds the string when `pending` is read.
+
+    The old ``state.pending += ch`` copied the whole accumulated string on
+    every character (``str += x`` on an attribute cannot resize in place).
+    """
+    from markdown_it.rules_inline.state_inline import StateInline
+
+    state = StateInline("", MarkdownIt(), {}, [])
+    for _ in range(10_000):
+        state.append_pending("x")
+    assert len(state._pending_buffer) == 10_000
+    assert state._pending == ""
+
+    assert state.pending == "x" * 10_000
+    assert state._pending_buffer == []
+
+    state.pending = ""
+    assert state.pending == ""
+
+
+def test_state_inline_pending_buffer_semantics():
+    """`StateInline.pending` is buffered internally but must behave exactly
+    like the plain ``str`` attribute it replaced."""
+    import copy
+
+    from markdown_it.rules_inline.state_inline import StateInline
+
+    md = MarkdownIt()
+    state = StateInline("", md, {}, [])
+
+    # append / read / append preserve order and the read materialises lazily
+    state.append_pending("a")
+    assert state.pending == "a"
+    state.append_pending("b")
+    state.append_pending("c")
+    assert state.pending == "abc"
+
+    # assignment replaces everything buffered so far
+    state.pending = "xy"
+    state.append_pending("z")
+    assert state.pending == "xyz"
+    state.pending = state.pending[:-1]
+    assert state.pending == "xy"
+
+    # flushing to a token empties both the string and the buffer
+    token = state.pushPending()
+    assert token.content == "xy"
+    assert state.pending == ""
+    assert not state._pending_buffer
+
+    # a str handed out earlier is never mutated by later appends
+    state.append_pending("hello")
+    held = state.pending
+    state.append_pending(" world")
+    assert state.pending == "hello world"
+    assert held == "hello"
+
+    # a shallow copy must not share the pending buffer with its original
+    state.pending = ""
+    state.append_pending("q")
+    clone = copy.copy(state)
+    clone.append_pending("r")
+    state.append_pending("s")
+    assert (state.pending, clone.pending) == ("qs", "qr")
+
+    # the setter works before __init__ has run (subclasses that set
+    # `pending` before calling super().__init__(), or __new__ + assignment)
+    bare = StateInline.__new__(StateInline)
+    bare.pending = "pre"
+    assert bare.pending == "pre"
+
+
+def test_rule_reading_pending_each_char_renders_correctly():
+    """A rule that reads ``state.pending`` on every character (as some
+    attribute-syntax plugins do) interleaves reads with appends; the output
+    must be unaffected."""
+    seen: list[int] = []
+
+    def peek_rule(state, silent):
+        seen.append(len(state.pending))
+        return False
+
+    md = MarkdownIt()
+    md.inline.ruler.before("text", "peek", peek_rule)
+    src = "{" * 5_000 + "&" * 5_000
+    assert md.renderInline(src) == "{" * 5_000 + "&amp;" * 5_000
+    # the rule saw the text accumulate one character at a time
+    assert seen[:4] == [0, 1, 2, 3]
+    assert max(seen) == 9_999