From e7c15aad3066f20ba47a6cad84cba8692d8f8dc5 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:45:31 +0200 Subject: [PATCH] Overhaul Discord markdown rendering Replace the basic markdown parser with feature-complete frontend and email renderers supporting lists, paragraphs, timestamps, ANSI code blocks, Unicode and custom emoji, Discord links, mentions, navigation tags, entity-aware previews, and safer URL handling. --- backend/email/markdown.go | 781 ++------ backend/email/markdown_legacy_url.go | 188 ++ backend/email/markdown_links.go | 497 +++++ backend/email/markdown_parse.go | 1721 +++++++++++++++++ backend/email/markdown_url.go | 827 ++++++++ backend/go.mod | 2 +- frontend/package-lock.json | 10 + frontend/package.json | 1 + frontend/src/components/EmbedPreview.tsx | 26 +- frontend/src/components/FormInputRow.tsx | 4 + frontend/src/components/PanelPreview.tsx | 28 +- .../src/components/discord/DiscordContent.tsx | 392 +++- .../src/components/discord/DiscordEmbed.tsx | 16 +- .../src/components/discord/DiscordMessage.tsx | 17 +- .../src/components/discord/DiscordText.tsx | 23 +- .../discord/container/Container.tsx | 1 + .../components/discord/container/Section.tsx | 1 + .../src/components/discord/discord-markup.css | 195 ++ frontend/src/components/discord/types.ts | 5 + .../gallery/GalleryListingPreview.tsx | 75 +- .../components/modals/GalleryPreviewModal.tsx | 5 +- .../src/components/modals/TagEditorModal.tsx | 6 + frontend/src/hooks/usePreviewEntities.ts | 20 + frontend/src/lib/discord-ansi.ts | 43 + frontend/src/lib/discord-emoji/index.ts | 39 + frontend/src/lib/discord-emoji/table.ts | 79 + frontend/src/lib/discord-markdown.ts | 312 --- frontend/src/lib/discord-markdown/engine.ts | 162 ++ frontend/src/lib/discord-markdown/index.ts | 148 ++ .../src/lib/discord-markdown/legacy-url.ts | 121 ++ frontend/src/lib/discord-markdown/links.ts | 414 ++++ frontend/src/lib/discord-markdown/rules.ts | 792 ++++++++ frontend/src/lib/discord-markdown/types.ts | 41 + frontend/src/lib/discord-time.ts | 119 ++ frontend/src/pages/Notifications.tsx | 6 +- frontend/src/pages/manage/tickets/view.tsx | 14 +- .../src/pages/manage/transcripts/view.tsx | 4 +- 37 files changed, 6125 insertions(+), 1010 deletions(-) create mode 100644 backend/email/markdown_legacy_url.go create mode 100644 backend/email/markdown_links.go create mode 100644 backend/email/markdown_parse.go create mode 100644 backend/email/markdown_url.go create mode 100644 frontend/src/components/discord/discord-markup.css create mode 100644 frontend/src/hooks/usePreviewEntities.ts create mode 100644 frontend/src/lib/discord-ansi.ts create mode 100644 frontend/src/lib/discord-emoji/index.ts create mode 100644 frontend/src/lib/discord-emoji/table.ts delete mode 100644 frontend/src/lib/discord-markdown.ts create mode 100644 frontend/src/lib/discord-markdown/engine.ts create mode 100644 frontend/src/lib/discord-markdown/index.ts create mode 100644 frontend/src/lib/discord-markdown/legacy-url.ts create mode 100644 frontend/src/lib/discord-markdown/links.ts create mode 100644 frontend/src/lib/discord-markdown/rules.ts create mode 100644 frontend/src/lib/discord-markdown/types.ts create mode 100644 frontend/src/lib/discord-time.ts diff --git a/backend/email/markdown.go b/backend/email/markdown.go index 20e47e5f..02845f74 100644 --- a/backend/email/markdown.go +++ b/backend/email/markdown.go @@ -1,71 +1,14 @@ package email -// Renders Discord message formatting as HTML or plain text. -// -// Go's regexp has no lookahead, so the paired-delimiter rules are hand-scanned -// rather than matched. Input is not escaped before parsing — that would turn -// `<@123>` into `<@123>` and break every `<` rule; the emitters escape leaves -// and attributes instead. - import ( "fmt" "html" - "net/url" "regexp" "strconv" "strings" "time" ) -type mdKind int - -const ( - kindText mdKind = iota - kindCode - kindCodeBlock - kindUserMention - kindRoleMention - kindChannelMention - kindGlobalMention - kindEmoji - kindTimestamp - kindMark - kindLink - kindHeading - kindSubtext - kindQuote -) - -type mdMark int - -const ( - markBold mdMark = iota - markItalic - markUnderline - markStrike - markSpoiler -) - -type mdNode struct { - kind mdKind - value string // text, code, mention id, emoji name, "everyone"/"here" - lang string // codeBlock - mark mdMark // mark - url string // link - level int // heading, 1-3 - unix int64 // timestamp - animated bool // emoji - children []mdNode -} - -func textNode(value string) mdNode { - return mdNode{kind: kindText, value: value} -} - -func markNode(m mdMark, body string) mdNode { - return mdNode{kind: kindMark, mark: m, children: parseInline(body)} -} - func RenderDiscordMarkdown(src string) string { var sb strings.Builder renderHTML(parseDiscordMarkdown(src), &sb) @@ -78,473 +21,14 @@ func RenderDiscordMarkdownText(src string) string { return strings.TrimSpace(sb.String()) } -func parseDiscordMarkdown(input string) []mdNode { - if input == "" { - return nil - } - return parseBlocks(reCarriageReturn.ReplaceAllString(input, "\n"), true) -} - -var ( - reCarriageReturn = regexp.MustCompile(`\r\n?`) - reTripleQuote = regexp.MustCompile("(?:^|\n) *>>> ") - reFence = regexp.MustCompile("(?s)```(?:([A-Za-z0-9+#._-]*)\n)?(.*?)```\n?") - reHeading = regexp.MustCompile(`^ {0,3}(#{1,3}) (.*)$`) - reSubtext = regexp.MustCompile(`^ {0,3}-# (.*)$`) -) - -func parseBlocks(src string, allowTriple bool) []mdNode { - // `>>> ` quotes everything after it, fences included, so it runs first. A nested - // `>>>` is literal, which is what allowTriple=false on the recursion gives. - if allowTriple { - if loc := reTripleQuote.FindStringIndex(src); loc != nil { - var out []mdNode - if head := src[:loc[0]]; head != "" { - out = parseBlocks(head, false) - } - return append(out, mdNode{kind: kindQuote, children: parseBlocks(src[loc[1]:], false)}) - } - } - - var out []mdNode - last := 0 - - for _, m := range reFence.FindAllStringSubmatchIndex(src, -1) { - if m[0] > last { - // Drop the newline that ended the preceding text run. - out = append(out, parseLines(strings.TrimSuffix(src[last:m[0]], "\n"))...) - } - - var lang, code string - if m[2] >= 0 { - lang = src[m[2]:m[3]] - } - if m[4] >= 0 { - code = src[m[4]:m[5]] - } - // One trailing newline only; trimming all whitespace would eat indentation. - out = append(out, mdNode{kind: kindCodeBlock, lang: lang, value: strings.TrimSuffix(code, "\n")}) - last = m[1] - } - - if last < len(src) { - out = append(out, parseLines(src[last:])...) - } - return out -} - -func parseLines(src string) []mdNode { - if src == "" { - return nil - } - - var out []mdNode - var buf []string - lines := strings.Split(src, "\n") - - flush := func() { - if len(buf) > 0 { - out = append(out, parseInline(strings.Join(buf, "\n"))...) - buf = nil - } - } - - for i := 0; i < len(lines); i++ { - line := lines[i] - - if m := reHeading.FindStringSubmatch(line); m != nil { - flush() - out = append(out, mdNode{kind: kindHeading, level: len(m[1]), children: parseInline(m[2])}) - continue - } - - if m := reSubtext.FindStringSubmatch(line); m != nil { - flush() - out = append(out, mdNode{kind: kindSubtext, children: parseInline(m[1])}) - continue - } - - if quoted, ok := matchQuoteLine(line); ok { - flush() - body := []string{quoted} - // Stops at the first non-quote line, so `> a\n\n> b` is two quotes. - for i+1 < len(lines) { - ahead, ok := matchQuoteLine(lines[i+1]) - if !ok { - break - } - body = append(body, ahead) - i++ - } - out = append(out, mdNode{kind: kindQuote, children: parseInline(strings.Join(body, "\n"))}) - continue - } - - buf = append(buf, line) - } - - flush() - return out -} - -func matchQuoteLine(line string) (string, bool) { - i := 0 - for i < 3 && i < len(line) && line[i] == ' ' { - i++ - } - if i >= len(line) || line[i] != '>' { - return "", false - } - i++ - - // `>>> ` is the triple quote, handled in parseBlocks. - if strings.HasPrefix(line[i:], ">>") { - return "", false - } - - start := i - for i < len(line) && line[i] == ' ' { - i++ - } - if i == start { - return "", false - } - return line[i:], true -} - -type inlineRule func(src string, at int) (mdNode, int, bool) - -var ( - // Populated in init rather than inline: the rules call parseInline, which reads - // this map back, and Go rejects that as an initialization cycle. - inlineRules map[byte][]inlineRule - - // First character of every rule above, plus the escape marker. - triggers [256]bool -) - -func init() { - // Rules under different keys can't collide, so only the order within a key - // matters: `**` before `*`, `__` before `_`, role mentions before user mentions. - inlineRules = map[byte][]inlineRule{ - '`': {matchCodeSpan}, - '|': {matchSpoiler}, - '*': {matchBold, matchItalicStar}, - '_': {matchUnderline, matchItalicUnderscore}, - '~': {matchStrike}, - '<': {matchEmoji, matchTimestamp, matchRoleMention, matchUserMention, matchChannelMention, matchAngleLink}, - '@': {matchGlobalMention}, - '[': {matchMaskedLink}, - 'h': {matchBareLink}, - } - - triggers['\\'] = true - for c := range inlineRules { - triggers[c] = true - } -} - -var reEscape = regexp.MustCompile(`^\\([^0-9A-Za-z\s])`) - -func parseInline(src string) []mdNode { - var out []mdNode - var buf strings.Builder - cursor := 0 - - flush := func() { - if buf.Len() > 0 { - out = append(out, textNode(buf.String())) - buf.Reset() - } - } - - for cursor < len(src) { - c := src[cursor] - - if c == '\\' { - if m := reEscape.FindStringSubmatch(src[cursor:]); m != nil { - buf.WriteString(m[1]) - cursor += len(m[0]) - continue - } - } - - matched := false - for _, rule := range inlineRules[c] { - node, end, ok := rule(src, cursor) - // Require forward progress; a zero-length match would spin forever. - if !ok || end <= cursor { - continue - } - flush() // before the append, or text comes out in the wrong order - out = append(out, node) - cursor = end - matched = true - break - } - if matched { - continue - } - - next := cursor + 1 - for next < len(src) && !triggers[src[next]] { - next++ - } - buf.WriteString(src[cursor:next]) - cursor = next - } - - flush() - return out -} - -// matchPair handles the delimiters whose body is any run of characters. guard rejects -// a closing run followed by another delimiter character; strike and spoiler skip it -// because Discord allows a stray `~`, making `~~~a~~~` render as ~a~. -func matchPair(src string, at int, delim string, m mdMark, guard bool) (mdNode, int, bool) { - if !strings.HasPrefix(src[at:], delim) { - return mdNode{}, 0, false - } - - n := len(delim) - // The body needs at least one character, hence the extra +1. - for i := at + n + 1; i+n <= len(src); i++ { - if !strings.HasPrefix(src[i:], delim) { - continue - } - if guard && i+n < len(src) && src[i+n] == delim[0] { - continue - } - return markNode(m, src[at+n:i]), i + n, true - } - return mdNode{}, 0, false -} - -func matchBold(src string, at int) (mdNode, int, bool) { - return matchPair(src, at, "**", markBold, true) -} - -func matchUnderline(src string, at int) (mdNode, int, bool) { - return matchPair(src, at, "__", markUnderline, true) -} - -func matchStrike(src string, at int) (mdNode, int, bool) { - return matchPair(src, at, "~~", markStrike, false) -} - -func matchSpoiler(src string, at int) (mdNode, int, bool) { - return matchPair(src, at, "||", markSpoiler, false) -} - -// The opening guard is what keeps `5 * 3 = 15` from italicising. -func matchItalicStar(src string, at int) (mdNode, int, bool) { - if at+1 >= len(src) || src[at+1] == '*' || isSpaceByte(src[at+1]) { - return mdNode{}, 0, false - } - - for i := at + 1; i < len(src); i++ { - if src[i] != '*' { - continue - } - if i+1 < len(src) && src[i+1] == '*' { - // Part of the body, not a closing delimiter. - i++ - continue - } - return markNode(markItalic, src[at+1:i]), i + 1, true - } - return mdNode{}, 0, false -} - -// The word-character guards are what keep snake_case_words plain. A lone `_` can't -// appear in the body, so a failed closing guard fails the whole match. -func matchItalicUnderscore(src string, at int) (mdNode, int, bool) { - if at > 0 && isWordByte(src[at-1]) { - return mdNode{}, 0, false - } - - for i := at + 1; i < len(src); i++ { - if src[i] != '_' { - continue - } - if i+1 < len(src) && src[i+1] == '_' { - i++ - continue - } - if i == at+1 { - return mdNode{}, 0, false - } - if i+1 < len(src) && isWordByte(src[i+1]) { - return mdNode{}, 0, false - } - return markNode(markItalic, src[at+1:i]), i + 1, true - } - return mdNode{}, 0, false -} - -// Contents are terminal and never re-parsed. Requiring a non-backtick before the -// close, and rejecting one followed by a backtick, is what stops an unclosed ``` -// fence being eaten as a one-backtick span. -func matchCodeSpan(src string, at int) (mdNode, int, bool) { - n := 1 - if at+1 < len(src) && src[at+1] == '`' { - n = 2 - } - delim := src[at : at+n] - - for i := at + n + 1; i+n <= len(src); i++ { - if !strings.HasPrefix(src[i:], delim) { - continue - } - if src[i-1] == '`' { - continue - } - if i+n < len(src) && src[i+n] == '`' { - continue - } - return mdNode{kind: kindCode, value: src[at+n : i]}, i + n, true - } - return mdNode{}, 0, false -} - -var ( - reEmoji = regexp.MustCompile(`^<(a)?:(\w+):(\d+)>`) - reTimestamp = regexp.MustCompile(`^`) - reRoleMention = regexp.MustCompile(`^<@&(\d+)>`) - reUserMention = regexp.MustCompile(`^<@!?(\d+)>`) - reChannelMention = regexp.MustCompile(`^<#(\d+)>`) - reAngleLink = regexp.MustCompile(`^<(https?://[^\s<>]+)>`) - reGlobalMention = regexp.MustCompile(`^@(everyone|here)\b`) - reMaskedLink = regexp.MustCompile(`^\[((?:[^\][]|\[[^\]]*\])+)\]\((?:<(https?://[^\s<>]+)>|((?:[^()\s]|\([^()\s]*\))+))\)`) - // The final class excludes trailing punctuation, so `(https://example.com/x.)` - // keeps the `.)`. - reBareLink = regexp.MustCompile(`^https?://[^\s<]*[^<.,:;"')\]\s]`) -) - -func matchEmoji(src string, at int) (mdNode, int, bool) { - m := reEmoji.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return mdNode{kind: kindEmoji, value: m[2], url: m[3], animated: m[1] == "a"}, at + len(m[0]), true -} - -// Past this a formatted date is nonsense, so the tag is left literal. -const maxTimestampSeconds = 8.64e12 - -func matchTimestamp(src string, at int) (mdNode, int, bool) { - m := reTimestamp.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - - unix, err := strconv.ParseInt(m[1], 10, 64) - if err != nil || unix > maxTimestampSeconds || unix < -maxTimestampSeconds { - return textNode(m[0]), at + len(m[0]), true - } - return mdNode{kind: kindTimestamp, unix: unix}, at + len(m[0]), true -} - -func matchRoleMention(src string, at int) (mdNode, int, bool) { - m := reRoleMention.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return mdNode{kind: kindRoleMention, value: m[1]}, at + len(m[0]), true -} - -func matchUserMention(src string, at int) (mdNode, int, bool) { - m := reUserMention.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return mdNode{kind: kindUserMention, value: m[1]}, at + len(m[0]), true -} - -func matchChannelMention(src string, at int) (mdNode, int, bool) { - m := reChannelMention.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return mdNode{kind: kindChannelMention, value: m[1]}, at + len(m[0]), true -} - -func matchAngleLink(src string, at int) (mdNode, int, bool) { - m := reAngleLink.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return buildLink(m[1], m[0], nil), at + len(m[0]), true -} - -// The left guard stops support@here.example.com growing an @here pill. -func matchGlobalMention(src string, at int) (mdNode, int, bool) { - if at > 0 && (isWordByte(src[at-1]) || src[at-1] == '@') { - return mdNode{}, 0, false - } - m := reGlobalMention.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - return mdNode{kind: kindGlobalMention, value: m[1]}, at + len(m[0]), true -} - -func matchMaskedLink(src string, at int) (mdNode, int, bool) { - m := reMaskedLink.FindStringSubmatch(src[at:]) - if m == nil { - return mdNode{}, 0, false - } - rawURL := m[2] - if rawURL == "" { - rawURL = m[3] - } - return buildLink(rawURL, m[0], &m[1]), at + len(m[0]), true -} - -func matchBareLink(src string, at int) (mdNode, int, bool) { - m := reBareLink.FindString(src[at:]) - if m == "" { - return mdNode{}, 0, false - } - return buildLink(m, m, nil), at + len(m), true -} - -// buildLink falls back to the raw source for unsafe URLs, so the text stays visible. -func buildLink(rawURL, raw string, label *string) mdNode { - if !isSafeURL(rawURL) { - return textNode(raw) - } - - children := []mdNode{textNode(rawURL)} - if label != nil { - children = parseInline(*label) - } - return mdNode{kind: kindLink, url: rawURL, children: children} -} - -// url.Parse accepts "/foo" with an empty scheme, so a host is required too. -func isSafeURL(raw string) bool { - u, err := url.Parse(raw) - if err != nil { - return false - } - return (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" -} - -func isWordByte(c byte) bool { - return c == '_' || (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -} - -func isSpaceByte(c byte) bool { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' -} - -// Inline styles throughout, since email clients strip