diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 39efe9a73..a78ebb402 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,29 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE -import re - -# _emphasis_ -EMPHASIS_RE = r'(_)([^_]+)\1' - -# __strong__ -STRONG_RE = r'(_{2})(.+?)\1' - -# __strong_em___ -STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' - - -class LegacyUnderscoreProcessor(UnderscoreProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -50,8 +28,8 @@ def extendMarkdown(self, md): | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ - # flake8: noqa: E501 48-50 - md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + # flake8: noqa: E501 27-29 + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 0f3533b2e..02a05aecf 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -41,7 +41,8 @@ from __future__ import annotations from . import util -from typing import TYPE_CHECKING, Any, Collection, NamedTuple +from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast +from collections import deque import re import xml.etree.ElementTree as etree from html import entities @@ -89,9 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70) - inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60) - inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) return inlinePatterns @@ -107,36 +107,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro ESCAPE_RE = r'\\(.)' """ Match a backslash escaped character (`\\<` or `\\*`). """ -EMPHASIS_RE = r'(\*)([^\*]+)\1' -""" Match emphasis with an asterisk (`*emphasis*`). """ - -STRONG_RE = r'(\*{2})(.+?)\1' -""" Match strong with an asterisk (`**strong**`). """ - -SMART_STRONG_RE = r'(?)` or `[text](url "title")`). """ @@ -149,9 +119,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro IMAGE_REFERENCE_RE = IMAGE_LINK_RE """ Match start of image reference (`![alt text][2]`). """ -NOT_STRONG_RE = r'((^|(?<=\s))(\*{1,3}|_{1,3})(?=\s|$))' -""" Match a stand-alone `*` or `_`. """ - AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>' """ Match an automatic link (``). """ @@ -592,151 +559,383 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class AsteriskProcessor(InlineProcessor): - """Emphasis processor for handling strong and em matches inside asterisks.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" - def build_single(self, m: re.Match[str], tag: str, idx: int) -> etree.Element: - """Return single tag.""" - el1 = etree.Element(tag) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - return el1 - - def build_double(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tag.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el2, None, idx) - el1.append(el2) - if len(m.groups()) == 3: - text = m.group(3) - self.parse_sub_patterns(text, el1, el2, idx) - return el1 - - def build_double2(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tags (variant 2): `text text`.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - text = m.group(3) - el1.append(el2) - self.parse_sub_patterns(text, el2, None, idx) - return el1 - - def parse_sub_patterns( - self, data: str, parent: etree.Element, last: etree.Element | None, idx: int + def __init__( + self, + token: str, + tags: str, + md: Markdown | None = None, + smart: bool = False, + double: bool = False ) -> None: """ - Parses sub patterns. - - `data`: text to evaluate. + Initialize. - `parent`: Parent to attach text and sub elements to. - - `last`: Last appended child to parent. Can also be None if parent has no children. + Arguments: + token: A single character token. + tags: A tag or two tags seprated by comma. When two are specified, the first will be the + one that takes double tokens. + md: the Markdown object + smart: Enable intelligent word logic. + double: If only one tag is specified, indicate whether it requires double tokens. - `idx`: Current pattern index that was used to evaluate the parent. """ - offset = 0 - pos = 0 - - length = len(data) - while pos < length: - # Find the start of potential emphasis or strong tokens - if self.compiled_re.match(data, pos): - matched = False - # See if the we can match an emphasis/strong pattern - for index, item in enumerate(self.PATTERNS): - # Only evaluate patterns that are after what was used on the parent - if index <= idx: - continue - m = item.pattern.match(data, pos) - if m: - # Append child nodes to parent - # Text nodes should be appended to the last - # child if present, and if not, it should - # be added as the parent's text node. - text = data[offset:m.start(0)] - if text: - if last is not None: - last.tail = text - else: - parent.text = text - el = self.build_element(m, item.builder, item.tags, index) - parent.append(el) - last = el - # Move our position past the matched hunk - offset = pos = m.end(0) - matched = True - if not matched: - # We matched nothing, move on to the next character - pos += 1 + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + + self.smart = smart + self.tags = tags.split(',') + self.double = len(tags) != 2 and double + super().__init__(self._build_patterns(token), md) + + def _build_patterns(self, token: str) -> str: + """Build regular expression patterns.""" + + # Build up patterns + self.token = token + etoken = re.escape(token) + avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - # Increment position as no potential emphasis start was found. - pos += 1 - - # Append any leftover text as a text node. - text = data[offset:] - if text: - if last is not None: - last.tail = text + # This case is not currently used + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + # Patterns for "dumb" cases. + else: + if len(self.tags) == 2: + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - parent.text = text - - def build_element(self, m: re.Match[str], builder: str, tags: str, index: int) -> etree.Element: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + + return fr'{etoken}' + + def _build_element( + self, + data: str, + start: int = 0, + offset: int = 0 + ) -> tuple[etree.Element, int]: """Element builder.""" - if builder == 'double2': - return self.build_double2(m, tags, index) - elif builder == 'double': - return self.build_double(m, tags, index) + regions = self.regions + el: etree.Element | None = None + last: Any = None + previous: Any = None + greater: Any = None + lesser: Any = None + + triple = set() + outer: list[etree.Element] = [] + outer_r: list[tuple[int, int, int, int, int]] = [] + + if len(self.tags) == 2: + greater, lesser = self.tags + elif self.double: + greater = self.tags[0] + lesser = None else: - return self.build_single(m, tags, index) - - def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]: - """Parse patterns.""" - - el = None - start = None - end = None - - for index, item in enumerate(self.PATTERNS): - m1 = item.pattern.match(data, m.start(0)) - if m1: - start = m1.start(0) - end = m1.end(0) - el = self.build_element(m1, item.builder, item.tags, index) + lesser = self.tags[0] + greater = None + + # Iterate regions creating the elements they represent + end = len(regions) + idx = 0 + for idx, i in enumerate(range(start, end), 1): + r = regions[i] + # Not contained within region + if idx and r[0] > regions[start][3]: + idx -= 1 break - return el, start, end + # Get the appropriate element(s) + if r[4] == 3: + el1 = etree.Element(greater) + el2 = etree.Element(lesser) + elif r[4] == 2: + el1 = etree.Element(greater) + el2 = None + else: + el1 = etree.Element(lesser) + el2 = None + + # Populate the elements with their text + if idx > 1: + if last.text is None: + if previous[2] < r[0]: + last.text = data[previous[1]+offset:previous[2]+offset] + else: + last.text = data[previous[1]+offset:r[0]+offset] + if last is not outer[-1] and last.tail is None: + if r[0] < outer_r[-1][3]: + last.tail = data[previous[3]+offset:r[0]+offset] + else: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset] + + # First element + if el is None: + el = el1 + last = el + outer.append(el) + outer_r.append(r) + + # Subsequent elements + else: + # Is the current outer element no longer wrapping this one? + while len(outer_r) > 1 and r[3] > outer_r[-1][3]: + outer.pop() + outer_r.pop() + # Double nested element (triple token) + if outer[-1] in triple: + outer[-1][-1].append(el1) + + # Non-nested + else: + outer[-1].append(el1) + + # Is this element wrapping the next? + if i + 1 < end: + if r[3] > regions[i + 1][3]: + outer.append(el1) + outer_r.append(r) + + # Track the last element we parsed. + last = el1 + + # Nest secondary element if there is one. + # Track triple tokens (double elements) + # so we can identify quickly and properly nest. + if el2 is not None: + el1.append(el2) + last = el2 + triple.add(el1) + + # Track the previous region. + previous = r + + # Populate remaining elements with their text + while outer: + if last.text is None: + last.text = data[previous[1]+offset:previous[2]+offset] + if last.tail is None and last is not outer[-1]: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + last = outer.pop() + previous = outer_r.pop() + + return cast('etree.Element', el), idx + + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + stack = self.stack + regions = self.regions + + # Process the next region(s) in the cache + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + + # Determine next offset + self.cache_index += count + if self.cache_index < len(regions): + self.cache_pos = regions[self.cache_index][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + + # Nothing left to process + else: + regions.clear() + stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + + # Whether element is valid or not, we'll advance past the end + return el, start + offset, end + offset + + def handleMatch( # type: ignore[override] + self, + m: re.Match[str], + data: str + ) -> tuple[etree.Element | None, int | None, int | None]: + """Parse delimiter pattern.""" + + # Do we have entries we haven't returned yet? + if self.regions: + return self.get_cached_result(m.start(0), data) + + # If token is not an opening, quit + m2 = self.boundary.match(data, m.start(0)) + if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] + if m2 is not None: + m = m2 + # Advance past the full length of the delimiter found + return None, m.start(0), m.end(0) + + # Get the stack and regions + stack = self.stack + regions = self.regions + + # Data offset + offset = m2.end(0) + # Stack of opening delimiters + stack.append((m2.start(0), offset, len(m2.group(0)))) + + # Pair tokens until the stack is empty or we can no longer find tokens. + while stack: + m2 = self.boundary.search(data, offset) + if m2 is None: + break + offset = m2.end(0) + + # Get current and last delimiter size + current = len(m2.group(0)) + last = stack[-1][-1] + + # Some delimiters may be ambiguous and look like both a start or an end + is_start = m2.lastgroup[0] != 'e' # type: ignore[index] + is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] + ambiguous = is_start and is_end + + # Find closing tokens + # Looking for: + # - `*em*` + # - `**strong**` + # - `***strong,em***` + # - `*em**` + # - `*em***` + # - `**strong***` + # + # Avoid ambiguous tokens that could be a start or an end. + # Consume starts until the end token is fully consumed. + # If we don't consume the entire end, see if next rule consumes it. + if is_end and ((not ambiguous and current > last) or (current == last)): + is_start = False + + # Consume previous tokens until the delimiter is consumed + s = m2.start(0) + while current and last <= current: + delimiter = stack.pop() + + # Build up region for pair and adjust accounting. + regions.append((delimiter[0], delimiter[1], s, s + delimiter[-1], delimiter[-1])) + s += delimiter[-1] + current -= delimiter[-1] + if not stack: + is_end = False + break + last = stack[-1][-1] + + # Do we still have more to consume? + is_end = current and stack and last > current + + # Looking for: + # - `***em*` + # - `***strong**` + # - `**em*` + if is_end and (last == 3 or not ambiguous) and last > current: + is_start = False + delimiter = stack.pop() + new = last - current + regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) + stack.append((delimiter[0], delimiter[0] + new, new)) + + # Find opening tokens + if is_start: + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + stack.append((m2.start(0), m2.end(0), current)) + + # Build the HTML elements + if regions: + # Regions may be out of order. + regions.sort(key=lambda x: x[0]) + start, end = regions[0][0], regions[0][3] + el, count = self._build_element(data) + + # Cache unprocessed regions to avoid repeated searches + if count < len(regions): + self.cache_index = count + self.cache_pos = self.regions[count][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + else: + # Cleanup + stack.clear() + regions.clear() -class UnderscoreProcessor(AsteriskProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" + return el, start, end - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ + # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. + start = m.start(0) + end = stack[-1][1] if stack else m.end(0) + stack.clear() + return None, start, end class LinkInlineProcessor(InlineProcessor): diff --git a/tests/misc/underscores.html b/tests/misc/underscores.html deleted file mode 100644 index 72d51b8b5..000000000 --- a/tests/misc/underscores.html +++ /dev/null @@ -1,6 +0,0 @@ -

THIS_SHOULD_STAY_AS_IS

-

Here is some emphasis, ok?

-

Ok, at least this should work.

-

THIS__SHOULD__STAY

-

Here is some strong stuff.

-

THISSHOULDSTAY?

\ No newline at end of file diff --git a/tests/misc/underscores.txt b/tests/misc/underscores.txt deleted file mode 100644 index 3c7f4bdd9..000000000 --- a/tests/misc/underscores.txt +++ /dev/null @@ -1,11 +0,0 @@ -THIS_SHOULD_STAY_AS_IS - -Here is some _emphasis_, ok? - -Ok, at least _this_ should work. - -THIS__SHOULD__STAY - -Here is some __strong__ stuff. - -THIS___SHOULD___STAY? diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 6e96ea32c..a7b3c56fa 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -20,6 +20,7 @@ """ from markdown.test_tools import TestCase +import textwrap class TestNotEmphasis(TestCase): @@ -147,7 +148,7 @@ def test_complex_emphasis_smart_underscore(self): def test_complex_emphasis_smart_underscore_mid_word(self): self.assertMarkdownRenders( 'This is text __bold_italic bold___ with more text', - '

This is text __bold_italic bold___ with more text

' + '

This is text bold_italic bold_ with more text

' ) def test_nested_emphasis(self): @@ -191,3 +192,33 @@ def test_link_emphasis_inner_outer(self): '**[**text**](url)**', '

text

' ) + + def test_underscore_legacy(self): + + self.assertMarkdownRenders( + textwrap.dedent( + """ + THIS_SHOULD_STAY_AS_IS + + Here is some _emphasis_, ok? + + Ok, at least _this_ should work. + + THIS__SHOULD__STAY + + Here is some __strong__ stuff. + + THIS___SHOULD___STAY? + """ + ), + textwrap.dedent( + """ +

THIS_SHOULD_STAY_AS_IS

+

Here is some emphasis, ok?

+

Ok, at least this should work.

+

THIS__SHOULD__STAY

+

Here is some strong stuff.

+

THIS___SHOULD___STAY?

+ """ + ).strip() + )