From ef245b53240baf33c5ae05bc2d206495beb89aaf Mon Sep 17 00:00:00 2001 From: python-slugify-4 Date: Wed, 14 Oct 2026 14:16:55 +0000 Subject: [PATCH] fix(modern): keep combining marks within unicode words Co-authored-by: CommandCodeBot --- slugify/slugify.py | 20 ++++++++++++++++++-- tests/test_modern_unicode.py | 8 ++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 tests/test_modern_unicode.py diff --git a/slugify/slugify.py b/slugify/slugify.py index 27df54b..4e767d4 100644 --- a/slugify/slugify.py +++ b/slugify/slugify.py @@ -66,6 +66,19 @@ def _transliterate(text: str, backend: Backend) -> str: return result +def _replace_disallowed_unicode(match: re.Match[str], text: str) -> str: + replacement = [] + for index, character in enumerate(match.group()): + position = match.start() + index + category = unicodedata.category(character) + attached = category in ('Mn', 'Mc') and ( + (position > 0 and text[position - 1].isalnum()) or + (position + 1 < len(text) and text[position + 1].isalnum()) + ) + replacement.append(character if attached else DEFAULT_SEPARATOR) + return ''.join(replacement) + + def _modern_truncate(text: str, max_length: int, word_boundary: bool, separator: str, save_order: bool) -> str: """Budget internal dash-separated tokens before mapping output delimiters. @@ -165,11 +178,14 @@ def _modern_slugify( text = _transliterate(unicodedata.normalize('NFKD', text), backend) text = unicodedata.normalize('NFKC' if allow_unicode else 'NFKD', text) if lowercase: - text = text.lower() + text = unicodedata.normalize('NFC', text.lower()) text = QUOTE_PATTERN.sub('', text) text = NUMBERS_PATTERN.sub('', text) pattern = regex_pattern or (DISALLOWED_UNICODE_CHARS_PATTERN if allow_unicode else DISALLOWED_CHARS_PATTERN) - text = re.sub(pattern, DEFAULT_SEPARATOR, text) + if allow_unicode and regex_pattern is None: + text = re.sub(pattern, lambda match: _replace_disallowed_unicode(match, text), text) + else: + text = re.sub(pattern, DEFAULT_SEPARATOR, text) text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR) if stopwords: diff --git a/tests/test_modern_unicode.py b/tests/test_modern_unicode.py new file mode 100644 index 0000000..1aa4e16 --- /dev/null +++ b/tests/test_modern_unicode.py @@ -0,0 +1,8 @@ +from slugify import slugify + + +def test_combining_mark_does_not_split_word() -> None: + output = slugify('İstanbul', allow_unicode=True, algorithm='modern') + + assert '-' not in output + assert output.endswith('stanbul')