From cd2bb8e163f515df557be1a8815024d40b5c41fd Mon Sep 17 00:00:00 2001 From: facelessuser Date: Mon, 7 Sep 2026 13:32:47 -0600 Subject: [PATCH] Rewrite emphasis handling - This is a complete rewrite of how emphasis handling is done. - Drop use of multiple regex patterns in run in multiple passes and instead evaluate delimiters, nested or otherwise, and build up HTML elements. - Try to consume tokens as much as possible until a full element is constructed (with children if any). - If an outer set of tokens cannot be resolved, but one or more sub tokens can, render the first sub token span and cache the remaining ones for subsequent reentry and render those until the cache is exhausted. - Two tests results were updated to match new behavior. --- markdown/extensions/legacy_em.py | 28 +- markdown/inlinepatterns.py | 531 +++++++++++++++------- tests/misc/underscores.html | 6 - tests/misc/underscores.txt | 11 - tests/test_syntax/inline/test_emphasis.py | 33 +- 5 files changed, 400 insertions(+), 209 deletions(-) delete mode 100644 tests/misc/underscores.html delete mode 100644 tests/misc/underscores.txt diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 39efe9a73..a78ebb402 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,29 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE -import re - -# _emphasis_ -EMPHASIS_RE = r'(_)([^_]+)\1' - -# __strong__ -STRONG_RE = r'(_{2})(.+?)\1' - -# __strong_em___ -STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' - - -class LegacyUnderscoreProcessor(UnderscoreProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -50,8 +28,8 @@ def extendMarkdown(self, md): | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ - # flake8: noqa: E501 48-50 - md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + # flake8: noqa: E501 27-29 + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 0f3533b2e..02a05aecf 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -41,7 +41,8 @@ from __future__ import annotations from . import util -from typing import TYPE_CHECKING, Any, Collection, NamedTuple +from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast +from collections import deque import re import xml.etree.ElementTree as etree from html import entities @@ -89,9 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70) - inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60) - inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) return inlinePatterns @@ -107,36 +107,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro ESCAPE_RE = r'\\(.)' """ Match a backslash escaped character (`\\<` or `\\*`). """ -EMPHASIS_RE = r'(\*)([^\*]+)\1' -""" Match emphasis with an asterisk (`*emphasis*`). """ - -STRONG_RE = r'(\*{2})(.+?)\1' -""" Match strong with an asterisk (`**strong**`). """ - -SMART_STRONG_RE = r'(?)` or `[text](url "title")`). """ @@ -149,9 +119,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro IMAGE_REFERENCE_RE = IMAGE_LINK_RE """ Match start of image reference (`![alt text][2]`). """ -NOT_STRONG_RE = r'((^|(?<=\s))(\*{1,3}|_{1,3})(?=\s|$))' -""" Match a stand-alone `*` or `_`. """ - AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>' """ Match an automatic link (``). """ @@ -592,151 +559,383 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class AsteriskProcessor(InlineProcessor): - """Emphasis processor for handling strong and em matches inside asterisks.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" - def build_single(self, m: re.Match[str], tag: str, idx: int) -> etree.Element: - """Return single tag.""" - el1 = etree.Element(tag) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - return el1 - - def build_double(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tag.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el2, None, idx) - el1.append(el2) - if len(m.groups()) == 3: - text = m.group(3) - self.parse_sub_patterns(text, el1, el2, idx) - return el1 - - def build_double2(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tags (variant 2): `text text`.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - text = m.group(3) - el1.append(el2) - self.parse_sub_patterns(text, el2, None, idx) - return el1 - - def parse_sub_patterns( - self, data: str, parent: etree.Element, last: etree.Element | None, idx: int + def __init__( + self, + token: str, + tags: str, + md: Markdown | None = None, + smart: bool = False, + double: bool = False ) -> None: """ - Parses sub patterns. - - `data`: text to evaluate. + Initialize. - `parent`: Parent to attach text and sub elements to. - - `last`: Last appended child to parent. Can also be None if parent has no children. + Arguments: + token: A single character token. + tags: A tag or two tags seprated by comma. When two are specified, the first will be the + one that takes double tokens. + md: the Markdown object + smart: Enable intelligent word logic. + double: If only one tag is specified, indicate whether it requires double tokens. - `idx`: Current pattern index that was used to evaluate the parent. """ - offset = 0 - pos = 0 - - length = len(data) - while pos < length: - # Find the start of potential emphasis or strong tokens - if self.compiled_re.match(data, pos): - matched = False - # See if the we can match an emphasis/strong pattern - for index, item in enumerate(self.PATTERNS): - # Only evaluate patterns that are after what was used on the parent - if index <= idx: - continue - m = item.pattern.match(data, pos) - if m: - # Append child nodes to parent - # Text nodes should be appended to the last - # child if present, and if not, it should - # be added as the parent's text node. - text = data[offset:m.start(0)] - if text: - if last is not None: - last.tail = text - else: - parent.text = text - el = self.build_element(m, item.builder, item.tags, index) - parent.append(el) - last = el - # Move our position past the matched hunk - offset = pos = m.end(0) - matched = True - if not matched: - # We matched nothing, move on to the next character - pos += 1 + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + + self.smart = smart + self.tags = tags.split(',') + self.double = len(tags) != 2 and double + super().__init__(self._build_patterns(token), md) + + def _build_patterns(self, token: str) -> str: + """Build regular expression patterns.""" + + # Build up patterns + self.token = token + etoken = re.escape(token) + avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - # Increment position as no potential emphasis start was found. - pos += 1 - - # Append any leftover text as a text node. - text = data[offset:] - if text: - if last is not None: - last.tail = text + # This case is not currently used + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + # Patterns for "dumb" cases. + else: + if len(self.tags) == 2: + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - parent.text = text - - def build_element(self, m: re.Match[str], builder: str, tags: str, index: int) -> etree.Element: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + + return fr'{etoken}' + + def _build_element( + self, + data: str, + start: int = 0, + offset: int = 0 + ) -> tuple[etree.Element, int]: """Element builder.""" - if builder == 'double2': - return self.build_double2(m, tags, index) - elif builder == 'double': - return self.build_double(m, tags, index) + regions = self.regions + el: etree.Element | None = None + last: Any = None + previous: Any = None + greater: Any = None + lesser: Any = None + + triple = set() + outer: list[etree.Element] = [] + outer_r: list[tuple[int, int, int, int, int]] = [] + + if len(self.tags) == 2: + greater, lesser = self.tags + elif self.double: + greater = self.tags[0] + lesser = None else: - return self.build_single(m, tags, index) - - def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]: - """Parse patterns.""" - - el = None - start = None - end = None - - for index, item in enumerate(self.PATTERNS): - m1 = item.pattern.match(data, m.start(0)) - if m1: - start = m1.start(0) - end = m1.end(0) - el = self.build_element(m1, item.builder, item.tags, index) + lesser = self.tags[0] + greater = None + + # Iterate regions creating the elements they represent + end = len(regions) + idx = 0 + for idx, i in enumerate(range(start, end), 1): + r = regions[i] + # Not contained within region + if idx and r[0] > regions[start][3]: + idx -= 1 break - return el, start, end + # Get the appropriate element(s) + if r[4] == 3: + el1 = etree.Element(greater) + el2 = etree.Element(lesser) + elif r[4] == 2: + el1 = etree.Element(greater) + el2 = None + else: + el1 = etree.Element(lesser) + el2 = None + + # Populate the elements with their text + if idx > 1: + if last.text is None: + if previous[2] < r[0]: + last.text = data[previous[1]+offset:previous[2]+offset] + else: + last.text = data[previous[1]+offset:r[0]+offset] + if last is not outer[-1] and last.tail is None: + if r[0] < outer_r[-1][3]: + last.tail = data[previous[3]+offset:r[0]+offset] + else: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset] + + # First element + if el is None: + el = el1 + last = el + outer.append(el) + outer_r.append(r) + + # Subsequent elements + else: + # Is the current outer element no longer wrapping this one? + while len(outer_r) > 1 and r[3] > outer_r[-1][3]: + outer.pop() + outer_r.pop() + # Double nested element (triple token) + if outer[-1] in triple: + outer[-1][-1].append(el1) + + # Non-nested + else: + outer[-1].append(el1) + + # Is this element wrapping the next? + if i + 1 < end: + if r[3] > regions[i + 1][3]: + outer.append(el1) + outer_r.append(r) + + # Track the last element we parsed. + last = el1 + + # Nest secondary element if there is one. + # Track triple tokens (double elements) + # so we can identify quickly and properly nest. + if el2 is not None: + el1.append(el2) + last = el2 + triple.add(el1) + + # Track the previous region. + previous = r + + # Populate remaining elements with their text + while outer: + if last.text is None: + last.text = data[previous[1]+offset:previous[2]+offset] + if last.tail is None and last is not outer[-1]: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + last = outer.pop() + previous = outer_r.pop() + + return cast('etree.Element', el), idx + + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + stack = self.stack + regions = self.regions + + # Process the next region(s) in the cache + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + + # Determine next offset + self.cache_index += count + if self.cache_index < len(regions): + self.cache_pos = regions[self.cache_index][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + + # Nothing left to process + else: + regions.clear() + stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + + # Whether element is valid or not, we'll advance past the end + return el, start + offset, end + offset + + def handleMatch( # type: ignore[override] + self, + m: re.Match[str], + data: str + ) -> tuple[etree.Element | None, int | None, int | None]: + """Parse delimiter pattern.""" + + # Do we have entries we haven't returned yet? + if self.regions: + return self.get_cached_result(m.start(0), data) + + # If token is not an opening, quit + m2 = self.boundary.match(data, m.start(0)) + if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] + if m2 is not None: + m = m2 + # Advance past the full length of the delimiter found + return None, m.start(0), m.end(0) + + # Get the stack and regions + stack = self.stack + regions = self.regions + + # Data offset + offset = m2.end(0) + # Stack of opening delimiters + stack.append((m2.start(0), offset, len(m2.group(0)))) + + # Pair tokens until the stack is empty or we can no longer find tokens. + while stack: + m2 = self.boundary.search(data, offset) + if m2 is None: + break + offset = m2.end(0) + + # Get current and last delimiter size + current = len(m2.group(0)) + last = stack[-1][-1] + + # Some delimiters may be ambiguous and look like both a start or an end + is_start = m2.lastgroup[0] != 'e' # type: ignore[index] + is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] + ambiguous = is_start and is_end + + # Find closing tokens + # Looking for: + # - `*em*` + # - `**strong**` + # - `***strong,em***` + # - `*em**` + # - `*em***` + # - `**strong***` + # + # Avoid ambiguous tokens that could be a start or an end. + # Consume starts until the end token is fully consumed. + # If we don't consume the entire end, see if next rule consumes it. + if is_end and ((not ambiguous and current > last) or (current == last)): + is_start = False + + # Consume previous tokens until the delimiter is consumed + s = m2.start(0) + while current and last <= current: + delimiter = stack.pop() + + # Build up region for pair and adjust accounting. + regions.append((delimiter[0], delimiter[1], s, s + delimiter[-1], delimiter[-1])) + s += delimiter[-1] + current -= delimiter[-1] + if not stack: + is_end = False + break + last = stack[-1][-1] + + # Do we still have more to consume? + is_end = current and stack and last > current + + # Looking for: + # - `***em*` + # - `***strong**` + # - `**em*` + if is_end and (last == 3 or not ambiguous) and last > current: + is_start = False + delimiter = stack.pop() + new = last - current + regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) + stack.append((delimiter[0], delimiter[0] + new, new)) + + # Find opening tokens + if is_start: + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + stack.append((m2.start(0), m2.end(0), current)) + + # Build the HTML elements + if regions: + # Regions may be out of order. + regions.sort(key=lambda x: x[0]) + start, end = regions[0][0], regions[0][3] + el, count = self._build_element(data) + + # Cache unprocessed regions to avoid repeated searches + if count < len(regions): + self.cache_index = count + self.cache_pos = self.regions[count][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + else: + # Cleanup + stack.clear() + regions.clear() -class UnderscoreProcessor(AsteriskProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" + return el, start, end - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ + # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. + start = m.start(0) + end = stack[-1][1] if stack else m.end(0) + stack.clear() + return None, start, end class LinkInlineProcessor(InlineProcessor): diff --git a/tests/misc/underscores.html b/tests/misc/underscores.html deleted file mode 100644 index 72d51b8b5..000000000 --- a/tests/misc/underscores.html +++ /dev/null @@ -1,6 +0,0 @@ -

THIS_SHOULD_STAY_AS_IS

-

Here is some emphasis, ok?

-

Ok, at least this should work.

-

THIS__SHOULD__STAY

-

Here is some strong stuff.

-

THISSHOULDSTAY?

\ No newline at end of file diff --git a/tests/misc/underscores.txt b/tests/misc/underscores.txt deleted file mode 100644 index 3c7f4bdd9..000000000 --- a/tests/misc/underscores.txt +++ /dev/null @@ -1,11 +0,0 @@ -THIS_SHOULD_STAY_AS_IS - -Here is some _emphasis_, ok? - -Ok, at least _this_ should work. - -THIS__SHOULD__STAY - -Here is some __strong__ stuff. - -THIS___SHOULD___STAY? diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 6e96ea32c..a7b3c56fa 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -20,6 +20,7 @@ """ from markdown.test_tools import TestCase +import textwrap class TestNotEmphasis(TestCase): @@ -147,7 +148,7 @@ def test_complex_emphasis_smart_underscore(self): def test_complex_emphasis_smart_underscore_mid_word(self): self.assertMarkdownRenders( 'This is text __bold_italic bold___ with more text', - '

This is text __bold_italic bold___ with more text

' + '

This is text bold_italic bold_ with more text

' ) def test_nested_emphasis(self): @@ -191,3 +192,33 @@ def test_link_emphasis_inner_outer(self): '**[**text**](url)**', '

text

' ) + + def test_underscore_legacy(self): + + self.assertMarkdownRenders( + textwrap.dedent( + """ + THIS_SHOULD_STAY_AS_IS + + Here is some _emphasis_, ok? + + Ok, at least _this_ should work. + + THIS__SHOULD__STAY + + Here is some __strong__ stuff. + + THIS___SHOULD___STAY? + """ + ), + textwrap.dedent( + """ +

THIS_SHOULD_STAY_AS_IS

+

Here is some emphasis, ok?

+

Ok, at least this should work.

+

THIS__SHOULD__STAY

+

Here is some strong stuff.

+

THIS___SHOULD___STAY?

+ """ + ).strip() + )