From 1fbcd5ad6af57a2ba529ee8ad6ca6e50b2615457 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 23:32:46 +0000 Subject: [PATCH 1/5] fix(chat): render portable Codex source references --- .../src/features/threads/ThreadFeed.tsx | 17 +- .../src/features/threads/thread-work-log.tsx | 26 +- .../chat/MessagesTimeline.logic.test.ts | 93 +++++ .../components/chat/MessagesTimeline.logic.ts | 7 +- packages/shared/package.json | 4 + packages/shared/src/codexCitations.ts | 348 ++++++++++++++++++ .../shared/src/markdownListIndentation.ts | 196 +++++++--- packages/shared/src/markdownPipeline.test.ts | 69 +++- packages/shared/src/markdownPipeline.ts | 27 ++ 9 files changed, 717 insertions(+), 70 deletions(-) create mode 100644 packages/shared/src/codexCitations.ts diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index ff5bf3147e021..ca475f5c27069 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -32,6 +32,7 @@ import { type OrchestrationV2ProjectedTurnItem, type RunId, } from "@t3tools/contracts"; +import { renderCodexCitationsAsMarkdown } from "@t3tools/shared/codexCitations"; import { renderAssistantCitationsAsText } from "@t3tools/shared/assistantCitations"; import { encodeComposerContextFragment } from "@t3tools/shared/composerContextClipboard"; import { parseThreadLinkHref, resolveThreadLinkRef } from "@t3tools/shared/threadLinks"; @@ -59,6 +60,7 @@ import { import { resolveViewedImageAsset } from "@t3tools/client-runtime/work-log/presentation"; import { renderCodexInlineDirectivesAsMarkdown, + renderCodexDirectivesForCopy, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; @@ -798,6 +800,7 @@ function ArtifactTemplateCard(props: { const AssistantMarkdownContent = memo(function AssistantMarkdownContent(props: { readonly markdown: string; + readonly isStreaming: boolean; readonly environmentId: EnvironmentId; readonly markdownStyles: MarkdownStyleSet; readonly linkHandlers: MarkdownLinkHandlers; @@ -805,7 +808,11 @@ const AssistantMarkdownContent = memo(function AssistantMarkdownContent(props: { readonly renderImage: MarkdownImageRenderer; readonly skills?: ReadonlyArray | undefined; }) { - const liveMarkdown = useLiveThreadLinkLabels(props.markdown, props.environmentId); + const citationMarkdown = useMemo( + () => renderCodexCitationsAsMarkdown(props.markdown, { isStreaming: props.isStreaming }), + [props.markdown, props.isStreaming], + ); + const liveMarkdown = useLiveThreadLinkLabels(citationMarkdown, props.environmentId); const segments = useMemo(() => splitCodexArtifactTemplateMarkdown(liveMarkdown), [liveMarkdown]); return segments.map((segment) => { @@ -994,7 +1001,7 @@ function renderFeedEntry( readonly markdownLinkHandlers: MarkdownLinkHandlers; readonly renderMarkdownImage: MarkdownImageRenderer; readonly renderViewedImage: MarkdownImageRenderer; - readonly renderReasoning: (text: string) => ReactNode; + readonly renderReasoning: (text: string, isStreaming: boolean) => ReactNode; readonly iconSubtleColor: string | import("react-native").ColorValue; readonly screenColor: string; readonly userBubbleColor: string | import("react-native").ColorValue; @@ -1380,6 +1387,7 @@ function renderFeedEntry( ( + const renderReasoning = (text: string, isStreaming: boolean) => ( void; readonly renderImage: MarkdownImageRenderer; - readonly renderReasoning: (text: string) => ReactNode; + readonly renderReasoning: (text: string, isStreaming: boolean) => ReactNode; readonly onPressPreview: (source: FilePreviewSource) => void; } @@ -525,7 +527,12 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) { return ( {props.activities.map((row) => ( - {props.renderReasoning(row.workEntry.detail ?? "")} + + {props.renderReasoning( + row.workEntry.detail ?? "", + row.projectedItem.item.type === "reasoning" && row.projectedItem.item.streaming, + )} + ))} ); @@ -1056,7 +1063,18 @@ const ThreadWorkLogRow = function ThreadWorkLogRow( props.onToggleRow(row.id, props.anchorKey, row.groupStandIn?.groupId); } }} - onLongPress={() => props.onCopyRow(row.id, row.getCopyText())} + onLongPress={() => + props.onCopyRow( + row.id, + reasoning + ? renderCodexDirectivesForCopy( + renderCodexCitationsAsMarkdown(row.getCopyText(), { + isStreaming: reasoning.streaming, + }), + ) + : row.getCopyText(), + ) + } > {row.live && !expanded ? ( {reasoning ? ( - props.renderReasoning(reasoning.text) + props.renderReasoning(reasoning.text, reasoning.streaming) ) : call ? ( [ call.command, diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index d55a621d87514..2a80c5f74a732 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -670,6 +670,99 @@ describe("resolveAssistantMessageCopyState", () => { visible: true, }); }); + + it("copies readable citations without changing unrelated Markdown or literal examples", () => { + const example = "\uE200cite\uE202excluded\uE201"; + const before = [ + "# Report", + "", + "🦊 \\*literal\\* & **bold** ", + `Example: \`${example}\` and [source ${example}]().`, + `${example}`, + "", + "```md", + example, + "```", + "", + "> Evidence ", + ].join("\n"); + const after = "\n\n- untouched spacing\n"; + const text = `${before}\uE200cite\uE202__opaque_id__\uE201${after}`; + const input = Object.freeze({ text, showCopyButton: true, streaming: false }); + + expect(resolveAssistantMessageCopyState(input)).toEqual({ + text: `${before}\\[Source 1: \\_\\_opaque\\_id\\_\\_\\]${after}`, + visible: true, + }); + expect(input.text).toBe(text); + }); + + it("copies recovered nested list citations at their original positions", () => { + const text = [ + "🦊 Before.", + "", + "- First **evidence** \uE200cite\uE202turn0view0\uE201.", + "", + " - Second \uE200cite\uE202turn1view0\uE201.", + "", + "After \uE200cite\uE202turn0view0\uE201.", + ].join("\r\n"); + + expect( + resolveAssistantMessageCopyState({ text, showCopyButton: true, streaming: false }), + ).toEqual({ + text: [ + "🦊 Before.", + "", + "- First **evidence** \\[Source 1: turn0view0\\].", + "", + " - Second \\[Source 2: turn1view0\\].", + "", + "After \\[Source 1: turn0view0\\].", + ].join("\r\n"), + visible: true, + }); + }); + + it("keeps source numbering across artifacts without numbering directive contents", () => { + const literal = "\uE200cite\uE202excluded\uE201"; + const text = [ + `Created :codex-file-citation{path="outputs/${literal}.txt" purpose="output"}.`, + "", + "Before \uE200cite\uE202turn0view0\uE201.", + "", + `::artifact-template{skill_name="artifact-template-report" skill_directory="/skills/report" display_name="${literal}" artifact_kind="document"}`, + "", + "After \uE200cite\uE202turn1search2\uE202turn0view0\uE202L8-L13\uE201.", + ].join("\n"); + + expect( + resolveAssistantMessageCopyState({ text, showCopyButton: true, streaming: false }), + ).toEqual({ + text: [ + `Created [${literal}.txt]().`, + "", + "Before \\[Source 1: turn0view0\\].", + "", + `${literal} (Document template)`, + "", + "After \\[Source 2: turn1search2; Source 1: turn0view0; L8-L13\\].", + ].join("\n"), + visible: true, + }); + }); + + it("leaves streaming copy hidden and retains an unfinished marker when stopped", () => { + const tail = "\uE200cite\uE202next"; + const text = `Evidence \uE200cite\uE202turn0view0\uE201 and ${tail}`; + + expect( + resolveAssistantMessageCopyState({ text, showCopyButton: true, streaming: true }), + ).toEqual({ text, visible: false }); + expect( + resolveAssistantMessageCopyState({ text, showCopyButton: true, streaming: false }), + ).toEqual({ text: `Evidence \\[Source 1: turn0view0\\] and ${tail}`, visible: true }); + }); }); describe("deriveMessagesTimelineRows", () => { diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 1024f11a213d1..9ea85e00186c6 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -8,6 +8,7 @@ import { collectAssistantCitations, } from "@t3tools/shared/assistantCitations"; import { shallow } from "zustand/vanilla/shallow"; +import { renderCodexCitationsAsMarkdown } from "@t3tools/shared/codexCitations"; import { renderCodexDirectivesForCopy } from "@t3tools/shared/codexMarkdownDirectives"; import { commandDisplayText, @@ -667,7 +668,11 @@ export function resolveAssistantMessageCopyState({ const hasText = text !== null && text.trim().length > 0; const visible = showCopyButton && hasText && !streaming; return { - text: hasText ? (visible ? renderCodexDirectivesForCopy(text) : text) : null, + text: hasText + ? visible + ? renderCodexDirectivesForCopy(renderCodexCitationsAsMarkdown(text)) + : text + : null, visible, }; } diff --git a/packages/shared/package.json b/packages/shared/package.json index b90af1a9b2cc1..443e08f509987 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -431,6 +431,10 @@ "types": "./src/codexArtifactTemplates.ts", "import": "./src/codexArtifactTemplates.ts" }, + "./codexCitations": { + "types": "./src/codexCitations.ts", + "import": "./src/codexCitations.ts" + }, "./codexFileCitations": { "types": "./src/codexFileCitations.ts", "import": "./src/codexFileCitations.ts" diff --git a/packages/shared/src/codexCitations.ts b/packages/shared/src/codexCitations.ts new file mode 100644 index 0000000000000..32fe59445f6c6 --- /dev/null +++ b/packages/shared/src/codexCitations.ts @@ -0,0 +1,348 @@ +import type { Extension, Tokenizer } from "micromark-util-types"; +import remarkGfm from "remark-gfm"; +import remarkParse from "remark-parse"; +import { unified, type Processor } from "unified"; + +import { remarkCodexDirectives } from "./codexMarkdownDirectives.ts"; +import { remarkNormalizeListItemIndentation } from "./markdownListIndentation.ts"; + +const START = "\uE200"; +const SEPARATOR = "\uE202"; +const END = "\uE201"; +const PREFIX = `${START}cite${SEPARATOR}`; +const TOKEN = "codexCitation"; +const SOURCE_ID = /^[A-Za-z0-9_-]+$/; +const LOCATOR = /^L\d+(?:-L\d+)?$/; +const RAW_HTML_CONTAINERS = new Set(["a", "code", "pre", "script", "style", "textarea"]); + +declare module "micromark-util-types" { + interface TokenTypeMap { + codexCitation: "codexCitation"; + } +} + +export interface CodexCitation { + readonly raw: string; + readonly sources: ReadonlyArray<{ readonly id: string; readonly number: number }>; + readonly locator?: string; +} + +interface CitationOptions { + readonly isStreaming?: boolean; +} + +interface MarkdownPoint { + readonly line: number; + readonly column: number; + readonly offset?: number; +} + +interface MarkdownNode { + type?: string; + value?: string; + position?: { readonly start: MarkdownPoint; readonly end: MarkdownPoint }; + data?: { + codexCitationMarkdown?: string; + hName?: string; + hProperties?: Record; + }; + children?: MarkdownNode[]; +} + +const citationTokens = new WeakMap(); + +export const CODEX_CITATION_HAST_PROPERTIES = ["dataCodexCitation"] as const; + +function parseCitation(raw: string) { + if (!raw.startsWith(PREFIX) || !raw.endsWith(END)) return null; + const ids = raw.slice(PREFIX.length, -END.length).split(SEPARATOR); + if (!ids.every((id) => SOURCE_ID.test(id))) return null; + const last = ids.at(-1); + const locator = + ids.length > 1 && last !== undefined && LOCATOR.test(last) ? ids.pop() : undefined; + return { ids: [...new Set(ids)], ...(locator === undefined ? {} : { locator }) }; +} + +function isSourceCharacter(code: number | null): code is number { + return ( + code !== null && + ((code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + code === 45 || + code === 95) + ); +} + +const tokenizeCitation: Tokenizer = function (effects, ok, nok) { + let prefixIndex = 0; + let segmentHasCharacters = false; + return prefix; + + function finish(code: number | null) { + effects.exit(TOKEN); + return ok(code); + } + + function prefix(code: number | null) { + if (code === null && prefixIndex > 0) return finish(code); + if (code !== PREFIX.charCodeAt(prefixIndex)) return nok(code); + if (prefixIndex === 0) effects.enter(TOKEN); + effects.consume(code); + prefixIndex += 1; + return prefixIndex === PREFIX.length ? source : prefix; + } + + function source(code: number | null) { + if (code === null) return finish(code); + if (isSourceCharacter(code)) { + segmentHasCharacters = true; + effects.consume(code); + return source; + } + if (!segmentHasCharacters) return nok(code); + if (code === SEPARATOR.charCodeAt(0)) { + segmentHasCharacters = false; + effects.consume(code); + return source; + } + if (code === END.charCodeAt(0)) { + effects.consume(code); + return finish; + } + return nok(code); + } +}; + +const CITATION_SYNTAX: Extension = { + text: { [START.charCodeAt(0)]: { tokenize: tokenizeCitation } }, +}; + +export function codexCitationText(citation: CodexCitation): string { + const sources = citation.sources.map(({ id, number }) => `Source ${number}: ${id}`); + if (citation.locator !== undefined) sources.push(citation.locator); + return `[${sources.join("; ")}]`; +} + +/** Keeps the opaque source IDs readable without inventing link destinations. */ +export function codexCitationMarkdown(citation: CodexCitation): string { + return codexCitationText(citation).replace(/[\\[\]*_`<&]/g, "\\$&"); +} + +function updateHtmlContext(value: string, tags: string[]): void { + if (value.startsWith("])*>/gi, + )) { + const tag = match[2]?.toLowerCase(); + if (tag === undefined || !RAW_HTML_CONTAINERS.has(tag)) continue; + if (match[1]) { + const index = tags.lastIndexOf(tag); + if (index !== -1) tags.splice(index); + } else if (!match[0].endsWith("/>")) { + tags.push(tag); + } + } +} + +function transformCitations(tree: MarkdownNode, source: string, options: CitationOptions): void { + const numbers = new Map(); + const htmlTags: string[] = []; + + function visit(node: MarkdownNode, excluded = false): MarkdownNode[] { + if (node.type === "html") { + if (!excluded) updateHtmlContext(node.value ?? "", htmlTags); + return [node]; + } + + const raw = citationTokens.get(node); + if (raw !== undefined && node.position !== undefined) { + citationTokens.delete(node); + const end = { + line: node.position.start.line, + column: node.position.start.column + raw.length, + ...(node.position.start.offset === undefined + ? {} + : { offset: node.position.start.offset + raw.length }), + }; + const parsed = parseCitation(raw); + const hidden = + parsed === null && + !raw.endsWith(END) && + options.isStreaming === true && + end.offset === source.length; + if (excluded || htmlTags.length > 0 || (parsed === null && !hidden)) return [node]; + + const citation = parsed && { + raw, + sources: parsed.ids.map((id) => { + let number = numbers.get(id); + if (number === undefined) { + number = numbers.size + 1; + numbers.set(id, number); + } + return { id, number }; + }), + ...(parsed.locator === undefined ? {} : { locator: parsed.locator }), + }; + const replacement: MarkdownNode = { + type: "text", + value: citation ? codexCitationText(citation) : "", + position: { start: node.position.start, end }, + data: { + codexCitationMarkdown: citation ? codexCitationMarkdown(citation) : "", + ...(citation + ? { + hName: "span", + hProperties: { dataCodexCitation: JSON.stringify(citation) }, + } + : {}), + }, + }; + // mdast can append following text to this token's text node. + const tail = node.value?.slice(raw.length); + return tail + ? [ + replacement, + { + type: "text", + value: tail, + position: { start: end, end: node.position.end }, + }, + ] + : [replacement]; + } + + const excludeChildren = + excluded || + node.type === "link" || + node.type === "linkReference" || + node.type === "image" || + node.type === "imageReference" || + node.type === "code" || + node.type === "inlineCode" || + node.type === "textDirective" || + node.type === "leafDirective" || + node.type === "containerDirective"; + if (node.children) { + node.children = node.children.flatMap((child) => visit(child, excludeChildren)); + } + return [node]; + } + + visit(tree); +} + +/** Tokenizes citations before Markdown can interpret underscores inside source IDs. */ +function attachCodexCitations(this: Processor, options: CitationOptions = {}) { + const data = this.data(); + (data.micromarkExtensions ??= []).push(CITATION_SYNTAX); + (data.fromMarkdownExtensions ??= []).push({ + enter: { + codexCitation(token) { + const raw = this.sliceSerialize(token); + const node = { type: "text" as const, value: raw }; + citationTokens.set(node, raw); + this.enter(node, token); + }, + }, + exit: { + codexCitation(token) { + this.exit(token); + }, + }, + }); + + return (tree: unknown, file: { readonly value: unknown }) => { + const source = String(file.value); + if (source.includes(START)) transformCitations(tree as MarkdownNode, source, options); + }; +} + +export const remarkCodexCitations = attachCodexCitations; + +function createCitationParser(isStreaming: boolean) { + return unified() + .use(remarkParse) + .use(remarkGfm) + .use(remarkNormalizeListItemIndentation) + .use(remarkCodexDirectives) + .use(remarkCodexCitations, { isStreaming }) + .freeze(); +} + +const citationParser = createCitationParser(false); +const streamingCitationParser = createCitationParser(true); + +/** Replaces only citation source ranges; all other Markdown remains byte-for-byte intact. */ +export function renderCodexCitationsAsMarkdown( + markdown: string, + options: CitationOptions = {}, +): string { + if (!markdown.includes(START)) return markdown; + const parser = options.isStreaming ? streamingCitationParser : citationParser; + const tree = parser.runSync(parser.parse(markdown), { value: markdown }) as MarkdownNode; + const replacements: { readonly start: number; readonly end: number; readonly text: string }[] = + []; + function collect(node: MarkdownNode): void { + const text = node.data?.codexCitationMarkdown; + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (text !== undefined && start !== undefined && end !== undefined) { + replacements.push({ start, end, text }); + } + for (const child of node.children ?? []) collect(child); + } + collect(tree); + let rendered = ""; + let cursor = 0; + for (const replacement of replacements) { + rendered += markdown.slice(cursor, replacement.start) + replacement.text; + cursor = replacement.end; + } + return rendered + markdown.slice(cursor); +} + +export function codexCitationFromHastProperties( + properties: Readonly> | null | undefined, +): CodexCitation | null { + const serialized = properties?.dataCodexCitation; + if (typeof serialized !== "string") return null; + let value: unknown; + try { + value = JSON.parse(serialized); + } catch { + return null; + } + if (typeof value !== "object" || value === null || !("raw" in value)) return null; + if (typeof value.raw !== "string" || !("sources" in value) || !Array.isArray(value.sources)) { + return null; + } + const parsed = parseCitation(value.raw); + if (!parsed || parsed.ids.length !== value.sources.length) return null; + const locator = "locator" in value ? value.locator : undefined; + if (locator !== parsed.locator) return null; + const sources: { readonly id: string; readonly number: number }[] = []; + const rawSources: readonly unknown[] = value.sources; + for (const [index, source] of rawSources.entries()) { + if ( + typeof source !== "object" || + source === null || + !("id" in source) || + typeof source.id !== "string" || + source.id !== parsed.ids[index] || + !("number" in source) || + typeof source.number !== "number" || + !Number.isSafeInteger(source.number) || + source.number <= 0 + ) { + return null; + } + sources.push({ id: source.id, number: source.number }); + } + return { + raw: value.raw, + sources, + ...(parsed.locator === undefined ? {} : { locator: parsed.locator }), + }; +} diff --git a/packages/shared/src/markdownListIndentation.ts b/packages/shared/src/markdownListIndentation.ts index cbf2090547edc..73b1ab28df507 100644 --- a/packages/shared/src/markdownListIndentation.ts +++ b/packages/shared/src/markdownListIndentation.ts @@ -1,14 +1,18 @@ +interface MarkdownPoint { + readonly line?: number; + readonly column?: number; + readonly offset?: number; +} + interface MarkdownPosition { - readonly start?: { - readonly line?: number; - readonly offset?: number; - }; + readonly start: MarkdownPoint; + readonly end: MarkdownPoint; } interface MarkdownAstNode { readonly type: string; - readonly value?: unknown; - readonly position?: MarkdownPosition; + value?: unknown; + position?: MarkdownPosition; children?: MarkdownAstNode[]; } @@ -20,21 +24,82 @@ interface MarkdownParser { parse(markdown: string): unknown; } -interface RecoveredMarkdown { - readonly blocks: MarkdownAstNode[]; - readonly source: string; +const INLINE_PARSE_PREFIX = "t3-markdown-inline-prefix:"; + +function lineRanges(value: string) { + const lines: { start: number; end: number; next: number }[] = []; + let start = 0; + for (const match of value.matchAll(/\r\n|\r|\n/g)) { + const next = match.index + match[0].length; + lines.push({ start, end: match.index, next }); + start = next; + } + lines.push({ start, end: value.length, next: value.length }); + return lines; } -const INLINE_PARSE_PREFIX = "t3-markdown-inline-prefix:"; +/** Indented code removes line prefixes but otherwise preserves each line's literal suffix. */ +function codeSourceOffsets(node: MarkdownAstNode, source: string): number[] | null { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (typeof node.value !== "string" || start === undefined || end === undefined) return null; + + const value = node.value; + const original = source.slice(start, end); + const valueLines = lineRanges(value); + const sourceLines = lineRanges(original); + if (valueLines.length !== sourceLines.length) return null; + + const offsets: number[] = []; + for (const [index, line] of valueLines.entries()) { + const sourceLine = sourceLines[index]; + if (!sourceLine) return null; + const text = value.slice(line.start, line.end); + const indentLength = /^[\t ]*/.exec(text)?.[0].length ?? 0; + const literal = text.slice(indentLength); + let sourceCursor = sourceLine.end - literal.length; + if ( + sourceCursor < sourceLine.start || + original.slice(sourceCursor, sourceLine.end) !== literal || + (line.end === line.next) !== (sourceLine.end === sourceLine.next) + ) { + return null; + } + + for (let column = indentLength; column <= text.length; column += 1) { + offsets[line.start + column] = start + sourceCursor + column - indentLength; + } + for (let column = indentLength - 1; column >= 0; column -= 1) { + sourceCursor -= 1; + if (sourceCursor < sourceLine.start) return null; + if (original[sourceCursor] === text[column]) { + offsets[line.start + column] = start + sourceCursor; + } else if (original[sourceCursor] === "\t" && /^ *$/.test(text.slice(0, column + 1))) { + // Consuming indentation can leave part of a tab as generated spaces. + for (let remaining = 0; remaining <= column; remaining += 1) { + offsets[line.start + remaining] = start + sourceCursor; + } + break; + } else { + return null; + } + } + for (let offset = line.end; offset < line.next; offset += 1) { + offsets[offset] = + start + sourceLine.end + Math.min(offset - line.end, sourceLine.next - sourceLine.end - 1); + } + } + return offsets; +} function isSameLineOverIndentedCode( node: MarkdownAstNode, - parent: MarkdownAstNode | undefined, + parent: MarkdownAstNode, markdown: string, ): boolean { if ( node.type !== "code" || - parent?.type !== "listItem" || + parent.type !== "listItem" || typeof node.value !== "string" || !/^[\t ]/.test(node.value) ) { @@ -56,13 +121,20 @@ function isSameLineOverIndentedCode( return sourceCharacter !== "`" && sourceCharacter !== "~"; } -function parseRecoveredMarkdown(value: string, parser: MarkdownParser): RecoveredMarkdown { - // A text prefix forces block-looking input into a paragraph while preserving - // the processor's configured inline extensions (for example, GFM syntax). - // Later root children are kept as blocks so blank-line-separated content is - // never discarded. - const source = `${INLINE_PARSE_PREFIX}${value}`; - const document = parser.parse(source) as MarkdownAstNode; +function recoverIndentedCode( + node: MarkdownAstNode, + parser: MarkdownParser, + source: string, + pointAt: (offset: number) => MarkdownPoint, +): MarkdownAstNode[] | null { + const offsets = codeSourceOffsets(node, source); + if (!offsets || typeof node.value !== "string") return null; + const value = node.value.trim(); + const trimStart = node.value.length - node.value.trimStart().length; + // The prefix keeps block-looking first-line content inline while retaining + // the processor's configured grammar, including citation tokens and GFM. + const recoveredSource = `${INLINE_PARSE_PREFIX}${value}`; + const document = parser.parse(recoveredSource) as MarkdownAstNode; const blocks = document.children; const paragraph = blocks?.[0]; const children = paragraph?.type === "paragraph" ? paragraph.children : undefined; @@ -74,65 +146,69 @@ function parseRecoveredMarkdown(value: string, parser: MarkdownParser): Recovere typeof first.value !== "string" || !first.value.startsWith(INLINE_PARSE_PREFIX) ) { - return { blocks: [{ type: "text", value }], source }; + return null; } - const firstValue = first.value.slice(INLINE_PARSE_PREFIX.length); - return { - blocks: [ - { - ...paragraph, - type: "paragraph", - children: [...(firstValue ? [{ ...first, value: firstValue }] : []), ...children.slice(1)], - }, - ...blocks.slice(1), - ], - source, - }; -} + first.value = first.value.slice(INLINE_PARSE_PREFIX.length); + if (!first.value) children.shift(); -function blocksFromIndentedCode(node: MarkdownAstNode, parser: MarkdownParser): RecoveredMarkdown { - const value = typeof node.value === "string" ? node.value.trim() : ""; - const recovered = parseRecoveredMarkdown(value, parser); - const first = recovered.blocks[0]; - return { - ...recovered, - blocks: - first && node.position - ? [{ ...first, position: node.position }, ...recovered.blocks.slice(1)] - : recovered.blocks, + const originalPoint = (point: MarkdownPoint) => { + if (point.offset === undefined) return null; + const valueOffset = trimStart + Math.max(0, point.offset - INLINE_PARSE_PREFIX.length); + const originalOffset = offsets[valueOffset]; + return originalOffset === undefined ? null : pointAt(originalOffset); + }; + const remap = (current: MarkdownAstNode): boolean => { + if (current.position) { + const start = originalPoint(current.position.start); + const end = originalPoint(current.position.end); + if (!start || !end) return false; + current.position = { start, end }; + } + return current.children?.every(remap) ?? true; }; + // Preserve node identity for syntax extensions that associate metadata with nodes. + return remap(document) ? blocks : null; } /** - * CommonMark treats four or more spaces after a list marker as an indented code - * block. + * Recover accidental same-line list alignment without changing explicit fences + * or conventional indented code. Recovered positions still address the input, + * so later plugins can safely replace source ranges or recognize a streaming tail. */ function attachListItemIndentationNormalizer(this: MarkdownParser) { - return (tree: MarkdownAstNode, file: MarkdownFile) => { - if (typeof file.value !== "string") { - return; - } - const markdown = file.value; - - const visit = (node: MarkdownAstNode, source: string) => { - if (!node.children) { - return; + return (tree: unknown, file: MarkdownFile) => { + if (typeof file.value !== "string") return; + const source = file.value; + let sourceLineStarts: number[] | undefined; + const pointAt = (offset: number) => { + sourceLineStarts ??= lineRanges(source).map((line) => line.start); + let low = 0; + let high = sourceLineStarts.length; + while (low + 1 < high) { + const middle = (low + high) >>> 1; + if ((sourceLineStarts[middle] ?? 0) <= offset) low = middle; + else high = middle; } + return { line: low + 1, column: offset - (sourceLineStarts[low] ?? 0) + 1, offset }; + }; + + const visit = (node: MarkdownAstNode) => { + if (!node.children) return; node.children = node.children.flatMap((child) => { if (isSameLineOverIndentedCode(child, node, source)) { - const recovered = blocksFromIndentedCode(child, this); - for (const block of recovered.blocks) { - visit(block, recovered.source); + const recovered = recoverIndentedCode(child, this, source, pointAt); + if (recovered) { + recovered.forEach(visit); + return recovered; } - return recovered.blocks; } - visit(child, source); + visit(child); return [child]; }); }; - visit(tree, markdown); + visit(tree as MarkdownAstNode); }; } diff --git a/packages/shared/src/markdownPipeline.test.ts b/packages/shared/src/markdownPipeline.test.ts index b546fcc57c650..75f9ab18e95de 100644 --- a/packages/shared/src/markdownPipeline.test.ts +++ b/packages/shared/src/markdownPipeline.test.ts @@ -2,7 +2,11 @@ import { expect, it } from "vite-plus/test"; import { unified } from "unified"; import remarkParse from "remark-parse"; import remarkRehype from "remark-rehype"; -import { CHAT_MARKDOWN_REHYPE_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS } from "./markdownPipeline.ts"; +import { + CHAT_MARKDOWN_REHYPE_PLUGINS, + CHAT_MARKDOWN_REMARK_PLUGINS, + CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS, +} from "./markdownPipeline.ts"; it("preserves in-app thread links through the shared Markdown sanitizer", () => { const processor = unified() @@ -55,3 +59,66 @@ it("keeps backslashes that CommonMark would read as escapes in prose Windows pat String.raw`C:\me\.t3 and C:\me\.t3`, ); }); + +type MarkdownNode = { type: string; value?: string; tagName?: string; children?: MarkdownNode[] }; + +function markdownText(node: MarkdownNode): string { + return node.value ?? node.children?.map(markdownText).join("") ?? ""; +} + +it("renders stable source numbers, grouped locators and opaque IDs through both shared pipelines", () => { + const markdown = + "First \uE200cite\uE202turn0view0\uE201. " + + "Grouped \uE200cite\uE202__opaque_id__\uE202turn0view0\uE202__opaque_id__\uE202L8-L13\uE201. " + + "Sole \uE200cite\uE202L42\uE201\uE200cite\uE202L42\uE202L9\uE201."; + const expected = + "First [Source 1: turn0view0]. " + + "Grouped [Source 2: __opaque_id__; Source 1: turn0view0; L8-L13]. " + + "Sole [Source 3: L42][Source 3: L42; L9]."; + const processor = unified() + .use(remarkParse) + .use(CHAT_MARKDOWN_REMARK_PLUGINS) + .use(remarkRehype, { allowDangerousHtml: true }) + .use(CHAT_MARKDOWN_REHYPE_PLUGINS); + const tree = processor.runSync(processor.parse(markdown), { value: markdown }); + expect(markdownText(tree)).toBe(expected); + expect(JSON.stringify(tree)).not.toContain('"href"'); + + const withBreaks = unified() + .use(remarkParse) + .use(CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS) + .use(remarkRehype, { allowDangerousHtml: true }) + .use(CHAT_MARKDOWN_REHYPE_PLUGINS); + expect(markdownText(withBreaks.runSync(withBreaks.parse(markdown), { value: markdown }))).toBe( + expected, + ); + + const nextMessage = "\uE200cite\uE202L42\uE201"; + expect( + markdownText(processor.runSync(processor.parse(nextMessage), { value: nextMessage })), + ).toBe("[Source 1: L42]"); +}); + +it("keeps citation examples literal without numbering them or suppressing later prose", () => { + const example = "\uE200cite\uE202excluded\uE201"; + const unsupported = "\uE200image\uE202turn0view0\uE201"; + const malformed = "\uE200cite\uE202two words\uE201"; + const markdown = [ + `Inline \`${example}\`, [link ${example}](https://example.com), ${example}.`, + "", + "```md", + example, + "```", + "", + `${unsupported} ${malformed} Actual \uE200cite\uE202turn0view0\uE201.`, + ].join("\n"); + const processor = unified() + .use(remarkParse) + .use(CHAT_MARKDOWN_REMARK_PLUGINS) + .use(remarkRehype, { allowDangerousHtml: true }) + .use(CHAT_MARKDOWN_REHYPE_PLUGINS); + expect(markdownText(processor.runSync(processor.parse(markdown), { value: markdown }))).toBe( + `Inline ${example}, link ${example}, ${example}.\n${example}\n\n` + + `${unsupported} ${malformed} Actual [Source 1: turn0view0].`, + ); +}); diff --git a/packages/shared/src/markdownPipeline.ts b/packages/shared/src/markdownPipeline.ts index a32825840647f..175cffdad3572 100644 --- a/packages/shared/src/markdownPipeline.ts +++ b/packages/shared/src/markdownPipeline.ts @@ -12,6 +12,7 @@ import { remarkCodexDirectives, } from "./codexMarkdownDirectives.ts"; import { isWindowsDrivePathHref } from "./markdownLinks.ts"; +import { CODEX_CITATION_HAST_PROPERTIES, remarkCodexCitations } from "./codexCitations.ts"; import { THREAD_LINK_PROTOCOL } from "./threadLinks.ts"; type MarkdownImageHastNode = { @@ -140,6 +141,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = { ], blockquote: [...(defaultSchema.attributes?.blockquote ?? []), "dataAlert"], div: [...(defaultSchema.attributes?.div ?? []), ...CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES], + span: [...(defaultSchema.attributes?.span ?? []), ...CODEX_CITATION_HAST_PROPERTIES], a: [...(defaultSchema.attributes?.a ?? []), "dataPullRequestAutolink"], img: [ ...(defaultSchema.attributes?.img ?? []), @@ -167,6 +169,7 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [ remarkGithubAlerts, remarkNormalizeListItemIndentation, remarkCodexDirectives, + remarkCodexCitations, remarkPreserveCodeMeta, remarkNormalizeLinksAndTagInlineCode, ]; @@ -177,11 +180,35 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [ remarkGithubAlerts, remarkNormalizeListItemIndentation, remarkCodexDirectives, + remarkCodexCitations, remarkBreaks, remarkPreserveCodeMeta, remarkNormalizeLinksAndTagInlineCode, ]; +const streamingRemarkPlugins = (plugins: PluggableList): PluggableList => + plugins.map((plugin) => + plugin === remarkCodexCitations ? [remarkCodexCitations, { isStreaming: true }] : plugin, + ); + +const STREAMING_REMARK_PLUGINS = streamingRemarkPlugins(CHAT_MARKDOWN_REMARK_PLUGINS); +const STREAMING_REMARK_PLUGINS_WITH_BREAKS = streamingRemarkPlugins( + CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS, +); + +export function chatMarkdownRemarkPlugins({ + isStreaming = false, + lineBreaks = false, +}: { + readonly isStreaming?: boolean; + readonly lineBreaks?: boolean; +} = {}): PluggableList { + if (isStreaming) { + return lineBreaks ? STREAMING_REMARK_PLUGINS_WITH_BREAKS : STREAMING_REMARK_PLUGINS; + } + return lineBreaks ? CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS : CHAT_MARKDOWN_REMARK_PLUGINS; +} + export const CHAT_MARKDOWN_REHYPE_PLUGINS: PluggableList = [ rehypePreserveBareAnchorPlaceholders, rehypeRaw, From 39bb845386542b8228179be44557ccd56eb0557a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 23:52:02 +0000 Subject: [PATCH 2/5] fix(chat): preserve citations through incremental cache clones --- apps/web/package.json | 2 + apps/web/src/markdown-incremental.test.tsx | 42 ++++++++++++++++++- packages/shared/src/codexCitations.ts | 14 ++++--- .../shared/src/markdownListIndentation.ts | 1 - pnpm-lock.yaml | 6 +++ 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index e6b1bc8471da6..7315d85b1df59 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -93,6 +93,8 @@ "compression": "^1.8.2", "oxc-transform-react": "~0.152.0", "playwright": "1.60.0", + "remark-parse": "catalog:", + "remark-rehype": "catalog:", "tailwindcss": "^4.0.0", "unified": "catalog:", "vite": "catalog:", diff --git a/apps/web/src/markdown-incremental.test.tsx b/apps/web/src/markdown-incremental.test.tsx index 3682deb281fa9..948f0ceecef2a 100644 --- a/apps/web/src/markdown-incremental.test.tsx +++ b/apps/web/src/markdown-incremental.test.tsx @@ -1,9 +1,14 @@ -import { CHAT_MARKDOWN_REHYPE_PLUGINS } from "@t3tools/shared/markdownPipeline"; +import { + CHAT_MARKDOWN_REHYPE_PLUGINS, + chatMarkdownRemarkPlugins, +} from "@t3tools/shared/markdownPipeline"; import type { Root } from "mdast"; import { renderToStaticMarkup } from "react-dom/server"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; -import type { Plugin } from "unified"; +import remarkParse from "remark-parse"; +import remarkRehype from "remark-rehype"; +import { unified, type Plugin } from "unified"; import { describe, expect, it } from "vite-plus/test"; import { remarkCodexDirectives } from "@t3tools/shared/codexMarkdownDirectives"; @@ -47,6 +52,39 @@ function render(source: string, incremental?: Plugin<[], Root>, parsedSources?: const prefix = "# Before\n\n```ts\nconst values = [1, 2];\n```\n\n"; describe("incremental Markdown parsing", () => { + it("preserves citation IDs and numbering across a cached closed fence", () => { + const incremental = createIncrementalMarkdownPlugin(); + const parse = (source: string, plugin?: Plugin<[], Root>) => { + const processor = unified() + .use(remarkParse) + .use(chatMarkdownRemarkPlugins({ isStreaming: true })) + .use(plugin ? [plugin] : []) + .use(remarkRehype, { allowDangerousHtml: true }) + .use(CHAT_MARKDOWN_REHYPE_PLUGINS); + return processor.runSync(processor.parse(source), { value: source }); + }; + type Node = { type: string; value?: string; children?: Node[] }; + const text = (node: Node): string => node.value ?? node.children?.map(text).join("") ?? ""; + const before = "Before \uE200cite\uE202turn0search1\uE201.\n\n```js\ncode\n```\n\n"; + const after = + "After \uE200cite\uE202turn0search2\uE201, again \uE200cite\uE202turn0search1\uE201."; + for (const [tail, expectedTail] of [ + ["After", "After"], + [after, "After [Source 2: turn0search2], again [Source 1: turn0search1]."], + [ + `${after} Also \uE200cite\uE202turn0search2\uE201.`, + "After [Source 2: turn0search2], again [Source 1: turn0search1]. Also [Source 2: turn0search2].", + ], + ]) { + const source = before + tail; + const full = parse(source); + const cached = parse(source, incremental); + expect(text(full)).toBe(`Before [Source 1: turn0search1].\ncode\n\n${expectedTail}`); + expect(text(cached)).toBe(text(full)); + expect(cached).toEqual(full); + } + }); + it("keeps the document prefix cached when list recovery parses contain fences", () => { const source = prefix + diff --git a/packages/shared/src/codexCitations.ts b/packages/shared/src/codexCitations.ts index 32fe59445f6c6..e2b698b5acf9b 100644 --- a/packages/shared/src/codexCitations.ts +++ b/packages/shared/src/codexCitations.ts @@ -43,14 +43,13 @@ interface MarkdownNode { position?: { readonly start: MarkdownPoint; readonly end: MarkdownPoint }; data?: { codexCitationMarkdown?: string; + codexCitationRaw?: string; hName?: string; hProperties?: Record; }; children?: MarkdownNode[]; } -const citationTokens = new WeakMap(); - export const CODEX_CITATION_HAST_PROPERTIES = ["dataCodexCitation"] as const; function parseCitation(raw: string) { @@ -155,9 +154,9 @@ function transformCitations(tree: MarkdownNode, source: string, options: Citatio return [node]; } - const raw = citationTokens.get(node); + const raw = node.data?.codexCitationRaw; if (raw !== undefined && node.position !== undefined) { - citationTokens.delete(node); + delete node.data?.codexCitationRaw; const end = { line: node.position.start.line, column: node.position.start.column + raw.length, @@ -241,8 +240,11 @@ function attachCodexCitations(this: Processor, options: CitationOptions = {}) { enter: { codexCitation(token) { const raw = this.sliceSerialize(token); - const node = { type: "text" as const, value: raw }; - citationTokens.set(node, raw); + const node: Extract[0], { type: "text" }> & MarkdownNode = { + type: "text", + value: raw, + data: { codexCitationRaw: raw }, + }; this.enter(node, token); }, }, diff --git a/packages/shared/src/markdownListIndentation.ts b/packages/shared/src/markdownListIndentation.ts index 73b1ab28df507..b7e8322bfdaa5 100644 --- a/packages/shared/src/markdownListIndentation.ts +++ b/packages/shared/src/markdownListIndentation.ts @@ -167,7 +167,6 @@ function recoverIndentedCode( } return current.children?.every(remap) ?? true; }; - // Preserve node identity for syntax extensions that associate metadata with nodes. return remap(document) ? blocks : null; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index feb2eca59aac0..5df8b38a960ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1040,6 +1040,12 @@ importers: playwright: specifier: 1.60.0 version: 1.60.0 + remark-parse: + specifier: 'catalog:' + version: 11.0.0 + remark-rehype: + specifier: 'catalog:' + version: 11.1.2 tailwindcss: specifier: 4.3.3 version: 4.3.3 From 4a8ebd818a9ed58a6dce511eb8104c6baac5ac38 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 23:59:18 +0000 Subject: [PATCH 3/5] fix(chat): index portable citation references for thread find --- packages/shared/src/markdownPipeline.test.ts | 4 ++++ packages/shared/src/threadFindText.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/markdownPipeline.test.ts b/packages/shared/src/markdownPipeline.test.ts index 75f9ab18e95de..3957796b8ca4a 100644 --- a/packages/shared/src/markdownPipeline.test.ts +++ b/packages/shared/src/markdownPipeline.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vite-plus/test"; import { unified } from "unified"; import remarkParse from "remark-parse"; import remarkRehype from "remark-rehype"; +import { searchableMessageSegments } from "./threadFindText.ts"; import { CHAT_MARKDOWN_REHYPE_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS, @@ -97,6 +98,9 @@ it("renders stable source numbers, grouped locators and opaque IDs through both expect( markdownText(processor.runSync(processor.parse(nextMessage), { value: nextMessage })), ).toBe("[Source 1: L42]"); + expect( + searchableMessageSegments({ role: "assistant", text: markdown, streaming: false }), + ).toEqual([expected]); }); it("keeps citation examples literal without numbering them or suppressing later prose", () => { diff --git a/packages/shared/src/threadFindText.ts b/packages/shared/src/threadFindText.ts index a2e5b4dbb982d..865a82708960d 100644 --- a/packages/shared/src/threadFindText.ts +++ b/packages/shared/src/threadFindText.ts @@ -114,7 +114,7 @@ function markdownThreadFindText( : lineBreaks ? assistantBreaksProcessor : assistantProcessor; - const tree = processor.runSync(processor.parse(markdown)); + const tree = processor.runSync(processor.parse(markdown), { value: markdown }); const paths = [ ...extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(markdown)), ...extractInlineCodeSpans(markdown).flatMap((span) => inlineCodeFilePathCandidate(span) ?? []), From ef3d2ebe00fea560e98dfed67a12a77b8fc1ff8c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 00:27:09 +0000 Subject: [PATCH 4/5] fix(shared): expose citation helpers with their consumers --- packages/shared/src/codexCitations.ts | 48 ++------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/packages/shared/src/codexCitations.ts b/packages/shared/src/codexCitations.ts index e2b698b5acf9b..4ac4531cad724 100644 --- a/packages/shared/src/codexCitations.ts +++ b/packages/shared/src/codexCitations.ts @@ -117,14 +117,14 @@ const CITATION_SYNTAX: Extension = { text: { [START.charCodeAt(0)]: { tokenize: tokenizeCitation } }, }; -export function codexCitationText(citation: CodexCitation): string { +function codexCitationText(citation: CodexCitation): string { const sources = citation.sources.map(({ id, number }) => `Source ${number}: ${id}`); if (citation.locator !== undefined) sources.push(citation.locator); return `[${sources.join("; ")}]`; } /** Keeps the opaque source IDs readable without inventing link destinations. */ -export function codexCitationMarkdown(citation: CodexCitation): string { +function codexCitationMarkdown(citation: CodexCitation): string { return codexCitationText(citation).replace(/[\\[\]*_`<&]/g, "\\$&"); } @@ -304,47 +304,3 @@ export function renderCodexCitationsAsMarkdown( } return rendered + markdown.slice(cursor); } - -export function codexCitationFromHastProperties( - properties: Readonly> | null | undefined, -): CodexCitation | null { - const serialized = properties?.dataCodexCitation; - if (typeof serialized !== "string") return null; - let value: unknown; - try { - value = JSON.parse(serialized); - } catch { - return null; - } - if (typeof value !== "object" || value === null || !("raw" in value)) return null; - if (typeof value.raw !== "string" || !("sources" in value) || !Array.isArray(value.sources)) { - return null; - } - const parsed = parseCitation(value.raw); - if (!parsed || parsed.ids.length !== value.sources.length) return null; - const locator = "locator" in value ? value.locator : undefined; - if (locator !== parsed.locator) return null; - const sources: { readonly id: string; readonly number: number }[] = []; - const rawSources: readonly unknown[] = value.sources; - for (const [index, source] of rawSources.entries()) { - if ( - typeof source !== "object" || - source === null || - !("id" in source) || - typeof source.id !== "string" || - source.id !== parsed.ids[index] || - !("number" in source) || - typeof source.number !== "number" || - !Number.isSafeInteger(source.number) || - source.number <= 0 - ) { - return null; - } - sources.push({ id: source.id, number: source.number }); - } - return { - raw: value.raw, - sources, - ...(parsed.locator === undefined ? {} : { locator: parsed.locator }), - }; -} From e5437f004fe3ca143b6882403029d2c483831071 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 20 Oct 2026 00:41:32 +0000 Subject: [PATCH 5/5] fix(shared): keep copied source references out of math syntax --- .../components/chat/MessagesTimeline.logic.test.ts | 14 +++++++------- packages/shared/src/codexCitations.ts | 2 +- packages/shared/src/markdownPipeline.test.ts | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index 2a80c5f74a732..1bf29d1123761 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -691,7 +691,7 @@ describe("resolveAssistantMessageCopyState", () => { const input = Object.freeze({ text, showCopyButton: true, streaming: false }); expect(resolveAssistantMessageCopyState(input)).toEqual({ - text: `${before}\\[Source 1: \\_\\_opaque\\_id\\_\\_\\]${after}`, + text: `${before}[Source 1: \\_\\_opaque\\_id\\_\\_\\]${after}`, visible: true, }); expect(input.text).toBe(text); @@ -714,11 +714,11 @@ describe("resolveAssistantMessageCopyState", () => { text: [ "🦊 Before.", "", - "- First **evidence** \\[Source 1: turn0view0\\].", + "- First **evidence** [Source 1: turn0view0\\].", "", - " - Second \\[Source 2: turn1view0\\].", + " - Second [Source 2: turn1view0\\].", "", - "After \\[Source 1: turn0view0\\].", + "After [Source 1: turn0view0\\].", ].join("\r\n"), visible: true, }); @@ -742,11 +742,11 @@ describe("resolveAssistantMessageCopyState", () => { text: [ `Created [${literal}.txt]().`, "", - "Before \\[Source 1: turn0view0\\].", + "Before [Source 1: turn0view0\\].", "", `${literal} (Document template)`, "", - "After \\[Source 2: turn1search2; Source 1: turn0view0; L8-L13\\].", + "After [Source 2: turn1search2; Source 1: turn0view0; L8-L13\\].", ].join("\n"), visible: true, }); @@ -761,7 +761,7 @@ describe("resolveAssistantMessageCopyState", () => { ).toEqual({ text, visible: false }); expect( resolveAssistantMessageCopyState({ text, showCopyButton: true, streaming: false }), - ).toEqual({ text: `Evidence \\[Source 1: turn0view0\\] and ${tail}`, visible: true }); + ).toEqual({ text: `Evidence [Source 1: turn0view0\\] and ${tail}`, visible: true }); }); }); diff --git a/packages/shared/src/codexCitations.ts b/packages/shared/src/codexCitations.ts index 4ac4531cad724..b7caef64e9901 100644 --- a/packages/shared/src/codexCitations.ts +++ b/packages/shared/src/codexCitations.ts @@ -125,7 +125,7 @@ function codexCitationText(citation: CodexCitation): string { /** Keeps the opaque source IDs readable without inventing link destinations. */ function codexCitationMarkdown(citation: CodexCitation): string { - return codexCitationText(citation).replace(/[\\[\]*_`<&]/g, "\\$&"); + return codexCitationText(citation).replace(/[\\\]*_`<&]/g, "\\$&"); } function updateHtmlContext(value: string, tags: string[]): void { diff --git a/packages/shared/src/markdownPipeline.test.ts b/packages/shared/src/markdownPipeline.test.ts index 3957796b8ca4a..2edd4668611fd 100644 --- a/packages/shared/src/markdownPipeline.test.ts +++ b/packages/shared/src/markdownPipeline.test.ts @@ -2,6 +2,8 @@ import { expect, it } from "vite-plus/test"; import { unified } from "unified"; import remarkParse from "remark-parse"; import remarkRehype from "remark-rehype"; +import { renderCodexCitationsAsMarkdown } from "./codexCitations.ts"; +import { parseMarkdownWithMath } from "./markdownMath.ts"; import { searchableMessageSegments } from "./threadFindText.ts"; import { CHAT_MARKDOWN_REHYPE_PLUGINS, @@ -85,6 +87,11 @@ it("renders stable source numbers, grouped locators and opaque IDs through both expect(markdownText(tree)).toBe(expected); expect(JSON.stringify(tree)).not.toContain('"href"'); + const copiedTree = parseMarkdownWithMath(renderCodexCitationsAsMarkdown(markdown)); + expect(copiedTree.children).toMatchObject([ + { type: "paragraph", children: [{ type: "text", value: expected }] }, + ]); + const withBreaks = unified() .use(remarkParse) .use(CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS)