From 8cc528b3304a4599955efd6560c2996f4446d920 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 10:07:21 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Fix=20quadratic?= =?UTF-8?q?=20inline=20tokenization=20on=20runs=20of=20special=20character?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long runs of characters that begin an inline rule but never complete a construct -- bare `&`, `<`, `~`, `{`, incomplete entities -- were tokenised in O(n^2) time, so a few hundred KB of such input stalls `render()` for many seconds. Output was always correct; the cost is pure CPU. Two independent quadratic factors, each hit once per character in a run: 1. `state.pending` accumulation. The inline tokenizer's fallback path did `state.pending += state.src[state.pos]` one character at a time. Because `pending` is an *attribute*, `str += ch` cannot use CPython's in-place concatenation optimisation (the attribute holds a second reference), so every append copies the whole accumulated string. 2. `src[pos:]` slicing in `entity` / `html_inline`. Both matched `^`-anchored regexes against `state.src[pos:]`, copying the remainder of the source on every `&` / `<`. On Python 3.10 this carried a second quadratic factor inside the regex engine itself: before 3.11, `search()` on a `^`-anchored pattern retries at every offset. Neither is quadratic in the JavaScript markdown-it (rope-backed string concatenation, and sticky/lastIndex regex matching), so this is specific to the Python port. Fix: - `StateInline.pending` now accumulates through a lazily-materialised list buffer via `append_pending()`, giving amortised O(1) appends. It is still exposed as a plain `str` property, so existing readers and third-party plugins doing `state.pending += x` keep working unchanged. - `entity` and `html_inline` anchor with `.match(state.src, pos)` instead of slicing; the leading `^` is dropped since `.match` already anchors at `pos`. Neither pattern uses `\b` or lookbehind, so this is exactly equivalent. `HTML_TAG_RE` has a single consumer; the separate `HTML_OPEN_CLOSE_TAG_RE` is untouched. All affected inputs now grow ~2.0x per doubling (linear) out to 640k characters. `"&" * 400_000` drops from 6.8s to 1.3s on CPython 3.11, from 653s to 1.8s on CPython 3.10, and from 145s to 0.2s on PyPy. Runs of `[` were also reported as superlinear, but once the `pending` quadratic is removed they measure a flat 2.0x per doubling: that path is linear already (bounded by the existing `skipToken` cache), just with a large constant. Output is unchanged: a 47,936-case differential run over the repo fixtures, the CommonMark spec, targeted constructs and fuzzed inputs across seven presets compares rendered HTML, `renderInline`, and full token streams byte-for-byte against the previous behaviour with zero differences. The test suites of mdit-py-plugins, myst-parser, mdformat and rich are unaffected. Adds `test_long_special_char_runs_are_linear`, which asserts correct output for the affected constructs and renders inputs that took ~16s before this change, so a regression trips the global 10s test timeout. Same class as the previously fixed #367 and #389. The root causes and this fix were independently identified by Haim Dimer in #411, which this supersedes. Co-authored-by: Haim Dimer --- markdown_it/common/html_re.py | 4 +- markdown_it/parser_inline.py | 2 +- markdown_it/rules_inline/backticks.py | 4 +- markdown_it/rules_inline/entity.py | 11 ++++-- markdown_it/rules_inline/html_inline.py | 2 +- markdown_it/rules_inline/state_inline.py | 32 +++++++++++++++- markdown_it/rules_inline/text.py | 2 +- tests/test_api/test_main.py | 49 ++++++++++++++++++++++++ 8 files changed, 95 insertions(+), 11 deletions(-) diff --git a/markdown_it/common/html_re.py b/markdown_it/common/html_re.py index ab822c5f..88cddc44 100644 --- a/markdown_it/common/html_re.py +++ b/markdown_it/common/html_re.py @@ -21,7 +21,9 @@ cdata = "" HTML_TAG_RE = re.compile( - "^(?:" + # No leading `^`: applied via `.match(src, pos)`, which anchors at `pos` + # without slicing `src[pos:]` (an O(len) copy per call, quadratic over a run). + "(?:" + open_tag + "|" + close_tag diff --git a/markdown_it/parser_inline.py b/markdown_it/parser_inline.py index 8fabb988..af66a7fa 100644 --- a/markdown_it/parser_inline.py +++ b/markdown_it/parser_inline.py @@ -197,7 +197,7 @@ def tokenize(self, state: StateInline) -> None: break continue - state.pending += state.src[state.pos] + state.append_pending(state.src[state.pos]) state.pos += 1 if state.pending: diff --git a/markdown_it/rules_inline/backticks.py b/markdown_it/rules_inline/backticks.py index fc60d6b1..67b5b5af 100644 --- a/markdown_it/rules_inline/backticks.py +++ b/markdown_it/rules_inline/backticks.py @@ -25,7 +25,7 @@ def backtick(state: StateInline, silent: bool) -> bool: if state.backticksScanned and state.backticks.get(openerLength, 0) <= start: if not silent: - state.pending += marker + state.append_pending(marker) state.pos += openerLength return True @@ -67,6 +67,6 @@ def backtick(state: StateInline, silent: bool) -> bool: state.backticksScanned = True if not silent: - state.pending += marker + state.append_pending(marker) state.pos += openerLength return True diff --git a/markdown_it/rules_inline/entity.py b/markdown_it/rules_inline/entity.py index ec9d3965..313b22b5 100644 --- a/markdown_it/rules_inline/entity.py +++ b/markdown_it/rules_inline/entity.py @@ -5,8 +5,11 @@ from ..common.utils import fromCodePoint, isValidEntityCode from .state_inline import StateInline -DIGITAL_RE = re.compile(r"^&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));", re.IGNORECASE) -NAMED_RE = re.compile(r"^&([a-z][a-z0-9]{1,31});", re.IGNORECASE) +# NB: no leading `^` -- these are applied via `.match(src, pos)`, which already +# anchors at `pos`. Anchoring this way avoids building `src[pos:]` on every +# `&`, which is an O(len) copy per call and makes long runs quadratic. +DIGITAL_RE = re.compile(r"&#((?:x[a-f0-9]{1,6}|[0-9]{1,7}));", re.IGNORECASE) +NAMED_RE = re.compile(r"&([a-z][a-z0-9]{1,31});", re.IGNORECASE) def entity(state: StateInline, silent: bool) -> bool: @@ -20,7 +23,7 @@ def entity(state: StateInline, silent: bool) -> bool: return False if state.src[pos + 1] == "#": - if match := DIGITAL_RE.search(state.src[pos:]): + if match := DIGITAL_RE.match(state.src, pos): if not silent: match1 = match.group(1) code = ( @@ -40,7 +43,7 @@ def entity(state: StateInline, silent: bool) -> bool: return True else: - if (match := NAMED_RE.search(state.src[pos:])) and match.group(1) in entities: + if (match := NAMED_RE.match(state.src, pos)) and match.group(1) in entities: if not silent: token = state.push("text_special", "", 0) token.content = entities[match.group(1)] diff --git a/markdown_it/rules_inline/html_inline.py b/markdown_it/rules_inline/html_inline.py index 9065e1d0..5eed14a8 100644 --- a/markdown_it/rules_inline/html_inline.py +++ b/markdown_it/rules_inline/html_inline.py @@ -26,7 +26,7 @@ def html_inline(state: StateInline, silent: bool) -> bool: if ch not in ("!", "?", "/") and not isLetter(ord(ch)): # /* / */ return False - match = HTML_TAG_RE.search(state.src[pos:]) + match = HTML_TAG_RE.match(state.src, pos) if not match: return False diff --git a/markdown_it/rules_inline/state_inline.py b/markdown_it/rules_inline/state_inline.py index de35287d..ccbbe1ea 100644 --- a/markdown_it/rules_inline/state_inline.py +++ b/markdown_it/rules_inline/state_inline.py @@ -54,7 +54,16 @@ def __init__( self.pos = 0 self.posMax = len(self.src) self.level = 0 - self.pending = "" + # `pending` holds literal text not yet flushed to a token. It is + # exposed as a plain `str` (see the property below), but is accumulated + # through a list buffer so that appending one character at a time -- the + # inline tokenizer's fallback path -- stays amortised O(1). Appending + # to a `str` *attribute* cannot use CPython's in-place concatenation + # optimisation (the attribute holds a second reference), so each `+=` + # copies the whole string, making long runs of non-markup characters + # quadratic. + self._pending = "" + self._pending_buffer: list[str] = [] self.pendingLevel = 0 # Stores { start: end } pairs. Useful for backtrack @@ -81,6 +90,27 @@ def __repr__(self) -> str: f"(pos=[{self.pos} of {self.posMax}], token={len(self.tokens)})" ) + @property + def pending(self) -> str: + """Literal text accumulated so far, but not yet flushed to a token.""" + if self._pending_buffer: + self._pending += "".join(self._pending_buffer) + self._pending_buffer.clear() + return self._pending + + @pending.setter + def pending(self, value: str) -> None: + self._pending = value + self._pending_buffer.clear() + + def append_pending(self, text: str) -> None: + """Append literal text to `pending`, in amortised O(1) time. + + Prefer this to ``state.pending += text`` on hot paths: it buffers the + fragment rather than rebuilding the whole ``pending`` string per call. + """ + self._pending_buffer.append(text) + def pushPending(self) -> Token: token = Token("text", "", 0) token.content = self.pending diff --git a/markdown_it/rules_inline/text.py b/markdown_it/rules_inline/text.py index ef0cc9ce..45b41437 100644 --- a/markdown_it/rules_inline/text.py +++ b/markdown_it/rules_inline/text.py @@ -16,7 +16,7 @@ def text(state: StateInline, silent: bool) -> bool: return False if not silent: - state.pending += state.src[state.pos : pos] + state.append_pending(state.src[state.pos : pos]) state.pos = pos diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 297c20d8..32e12a0e 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -365,3 +365,52 @@ def test_text_join_merges_adjacent_text_special_tokens(): assert len(children_on) == 1 assert children_on[0].type == "text" assert children_on[0].content == "***" + + +def test_long_special_char_runs_are_linear(): + """Long runs of characters that start an inline rule but form no construct + must tokenise in linear time. + + Two independent O(n^2) factors used to be hit once per character in such + runs: + + 1. the inline tokenizer's fallback appended to ``state.pending`` one + character at a time, and ``str += ch`` on an *attribute* cannot reuse the + buffer in place, so every append copied the whole accumulated string; + 2. the ``entity`` and ``html_inline`` rules matched ``^``-anchored regexes + against ``state.src[pos:]``, copying the rest of the source per ``&``/``<``. + + The large inputs below took ~16s combined before the fix, so a regression + trips the global 10s test timeout. + """ + md = MarkdownIt() + + # Correctness of the affected constructs (small, exact). + for src, expected in [ + ("&" * 32, "&" * 32), # entity rule rejects, falls back to pending + ("&" * 8, "&" * 8), # NAMED_RE still matches + ("#" * 8, "#" * 8), # DIGITAL_RE still matches + ("#" * 8, "#" * 8), # DIGITAL_RE, hex form + ("&#" * 8, "&#" * 8), # `#` branch, no match + ("&am" * 8, "&am" * 8), # named prefix, unterminated + ("&nope;" * 4, "&nope;" * 4), # well-formed but unknown name + ("<" * 16, "<" * 16), # html_inline rejects (html=False anyway) + ("{" * 32, "{" * 32), # no rule at all -> pure pending fallback + ("~" * 16, "~" * 16), # sub-length strikethrough delimiters + ]: + assert md.renderInline(src) == expected, src + + # Backtick markers also feed `pending`; unclosed runs stay literal. + assert md.renderInline("`" * 3 + "a") == "```a" + assert md.renderInline("`a``b") == "`a``b" + # ...while a matched pair still becomes code. + assert md.renderInline("`a`") == "a" + + # Headline: half a million bare ampersands exercises both the `pending` + # fallback and the `entity` rule's per-character regex match. + assert md.renderInline("&" * 500_000) == "&" * 500_000 + + # `html_inline`'s slice was only reachable with `html=True`; ` Date: Tue, 8 Sep 2026 11:24:15 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Keep=20`pending`?= =?UTF-8?q?=20linear=20under=20per-character=20readers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The buffered `StateInline.pending` from the previous commit materialised its buffer with `self._pending += joined`. That is itself an attribute concatenation, so every read copied the whole accumulated string. Any inline rule that reads `state.pending` before its cheap character check therefore re-introduced the quadratic on every character: the mdit-py-plugins `attrs` rule does exactly that, and myst-parser's `attrs_inline` extension enables it. With that plugin, `"~a~" * 400_000` still took 34s (5.7x per doubling) after the previous commit. The getter now moves the string into a local and drops the instance's reference before concatenating, so CPython can resize it in place. The same input takes 3.0s and grows 2.0x per doubling; verified on CPython 3.10 through 3.13. A str handed out to a caller is never mutated by this path, since a second reference disables the in-place resize. Two latent hazards of the list buffer are closed at the same time: - The setter used `.clear()`, so assigning `pending` on an instance whose `__init__` had not run (a subclass setting it before `super().__init__()`, or `__new__` plus assignment) raised `AttributeError`. It now assigns a fresh list. - `copy.copy(state)` shared the mutable buffer, so appends on the copy leaked into the original. A `__copy__` gives the copy its own buffer. Tests cover the buffer's ordering, flush, copy and pre-init semantics, and a rule that reads `pending` on every character over a 800k-character run, which exceeded the global 10s test timeout before this change. --- markdown_it/rules_inline/state_inline.py | 30 ++++++++-- tests/test_api/test_main.py | 71 ++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/markdown_it/rules_inline/state_inline.py b/markdown_it/rules_inline/state_inline.py index ccbbe1ea..8f0ca1e8 100644 --- a/markdown_it/rules_inline/state_inline.py +++ b/markdown_it/rules_inline/state_inline.py @@ -93,15 +93,37 @@ def __repr__(self) -> str: @property def pending(self) -> str: """Literal text accumulated so far, but not yet flushed to a token.""" - if self._pending_buffer: - self._pending += "".join(self._pending_buffer) - self._pending_buffer.clear() + buffer = self._pending_buffer + if buffer: + # Move the string into a local and drop the instance's reference + # before concatenating. With a single reference left, CPython + # resizes the string in place (amortised O(new chars)) rather + # than copying it, so a rule that reads `pending` on every + # character (e.g. an attribute-syntax plugin) stays linear. + text = self._pending + self._pending = "" + text += "".join(buffer) + buffer.clear() + self._pending = text return self._pending @pending.setter def pending(self, value: str) -> None: self._pending = value - self._pending_buffer.clear() + # Assign rather than `.clear()` so the setter also works on an + # instance whose `__init__` has not run yet (subclasses that set + # `pending` before calling `super().__init__()`), and so a copied + # state never shares a buffer with its original. + self._pending_buffer = [] + + def __copy__(self) -> StateInline: + """Shallow copy that does not share the pending text buffer.""" + text = self.pending # materialise (and clear) our own buffer first + new = self.__class__.__new__(self.__class__) + new.__dict__.update(self.__dict__) + new._pending = text + new._pending_buffer = [] + return new def append_pending(self, text: str) -> None: """Append literal text to `pending`, in amortised O(1) time. diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 32e12a0e..61c24688 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -414,3 +414,74 @@ def test_long_special_char_runs_are_linear(): # never a valid tag, so it renders escaped. md_html = MarkdownIt("commonmark", {"html": True}) assert md_html.renderInline(" Date: Tue, 8 Sep 2026 11:32:21 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20Replace=20timing=20g?= =?UTF-8?q?uards=20with=20deterministic=20linearity=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two regression tests added with the fix relied on the global 10s pytest timeout to catch a return to quadratic behaviour. CI runs every matrix job under `--cov`, whose tracer slows these pure-Python loops by more than 4x, so the tests timed out on the fixed code. Following the precedent of #367 and #389, the tests now pin the behaviour deterministically rather than by wall clock: - `test_inline_rules_do_not_slice_remaining_source` drives the inline parser with a `str` subclass that records every slice taken from it. The old `entity` / `html_inline` rules produced ~5000 slices of up to 10000 characters on a 5000-opener run; the only slices left are the `text` rule's single-character chunks. - `test_pending_appends_do_not_materialise` checks that `append_pending` buffers fragments without building the string until `pending` is read. - `test_long_special_char_runs_render_correctly` keeps the output assertions for every affected construct at modest sizes. - `test_rule_reading_pending_each_char_renders_correctly` interleaves per-character reads of `pending` with appends and checks the output. Each of the structural tests fails on the pre-fix code and passes in well under a second with or without coverage. --- tests/test_api/test_main.py | 118 ++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 32 deletions(-) diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 61c24688..dca3e8b2 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -1,3 +1,5 @@ +import pytest + from markdown_it import MarkdownIt from markdown_it.token import Token @@ -367,25 +369,17 @@ def test_text_join_merges_adjacent_text_special_tokens(): assert children_on[0].content == "***" -def test_long_special_char_runs_are_linear(): - """Long runs of characters that start an inline rule but form no construct - must tokenise in linear time. - - Two independent O(n^2) factors used to be hit once per character in such - runs: - - 1. the inline tokenizer's fallback appended to ``state.pending`` one - character at a time, and ``str += ch`` on an *attribute* cannot reuse the - buffer in place, so every append copied the whole accumulated string; - 2. the ``entity`` and ``html_inline`` rules matched ``^``-anchored regexes - against ``state.src[pos:]``, copying the rest of the source per ``&``/``<``. +def test_long_special_char_runs_render_correctly(): + """Runs of characters that begin an inline rule but do not form a construct + render as literal text, however long the run. - The large inputs below took ~16s combined before the fix, so a regression - trips the global 10s test timeout. + These inputs were previously tokenised in O(n^2) time (see + `test_inline_rules_do_not_slice_remaining_source` and + `test_pending_appends_do_not_materialise` for the invariants that keep them + linear); this test pins the *output*. """ md = MarkdownIt() - # Correctness of the affected constructs (small, exact). for src, expected in [ ("&" * 32, "&" * 32), # entity rule rejects, falls back to pending ("&" * 8, "&" * 8), # NAMED_RE still matches @@ -406,14 +400,74 @@ def test_long_special_char_runs_are_linear(): # ...while a matched pair still becomes code. assert md.renderInline("`a`") == "a" - # Headline: half a million bare ampersands exercises both the `pending` - # fallback and the `entity` rule's per-character regex match. - assert md.renderInline("&" * 500_000) == "&" * 500_000 - - # `html_inline`'s slice was only reachable with `html=True`; ` "_SliceCountingStr": + self = super().__new__(cls, value) + self.slice_lengths = [] + return self + + def __getitem__(self, key): + if isinstance(key, slice): + start, stop, _ = key.indices(len(self)) + self.slice_lengths.append(max(0, stop - start)) + return str.__getitem__(self, key) + + +@pytest.mark.parametrize( + "preset,options,src", + [ + ("commonmark", {}, "&" * 5_000), # entity rule + ("commonmark", {}, "&#" * 5_000), # entity rule, numeric branch + ("commonmark", {"html": True}, "