diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 8e8c1c4955b29..4b8721936ab95 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -92,6 +92,7 @@ import { renderCodexInlineDirectivesAsMarkdown, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; +import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; import { imageMimeType } from "@t3tools/shared/image"; import { videoMimeType } from "@t3tools/shared/video"; @@ -1217,7 +1218,7 @@ function AssistantMarkdownBlocks(props: { return ( { markdownRef.current = node; }; - const sharedRemarkPlugins = + const sharedRemarkPlugins = withExtraPlugins( CHAT_MARKDOWN_REMARK_PLUGIN_SETS[ componentState.mathEnabled ? lineBreaks @@ -1846,7 +1854,11 @@ function ChatMarkdown({ : lineBreaks ? "breaks" : "plain" - ]; + ], + hideWebCitations && text.includes(CODEX_WEB_CITATION_START) + ? WEB_CITATION_REMARK_PLUGINS + : EMPTY_REMARK_PLUGINS, + ); // The incremental parser keeps per-instance state, so its output is not shared. const uncachedPlugins = incrementalParsing; const remarkPlugins = incrementalParsing diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 90690dc0810ec..a3256ab28fb32 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -11,6 +11,7 @@ import { } from "@t3tools/client-runtime/user-message"; import type { HandoffTimelineRun } from "@t3tools/client-runtime/handoff"; import { renderCodexDirectivesForCopy } from "@t3tools/shared/codexMarkdownDirectives"; +import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations"; import { commandDisplayText, commandProgramName, @@ -690,7 +691,11 @@ export function resolveAssistantMessageCopyState({ const hasText = text !== null && text.trim().length > 0; const visible = showCopyButton && hasText && !streaming; return { - text: hasText ? (visible ? renderCodexDirectivesForCopy(text) : text) : null, + text: hasText + ? visible + ? hideCodexWebCitations(renderCodexDirectivesForCopy(text)) + : text + : null, visible, }; } diff --git a/apps/web/src/components/chat/TimelineAssistantRows.tsx b/apps/web/src/components/chat/TimelineAssistantRows.tsx index 36c04da893275..d326d57369abf 100644 --- a/apps/web/src/components/chat/TimelineAssistantRows.tsx +++ b/apps/web/src/components/chat/TimelineAssistantRows.tsx @@ -57,6 +57,7 @@ export function AssistantTimelineRow({ row }: { row: Extract; +} + +// Positions and text node boundaries differ between a displayed tree and a reparsed copy, and a +// copy can carry an invisible word joiner; structure and visible text must not differ. +function shape(node: TestNode): TestNode { + const children: Array = []; + for (const child of (node.children ?? []).map(shape)) { + const previous = children.at(-1); + if (child.type === "text" && child.value === "") continue; + if (child.type === "text" && previous?.type === "text") { + children[children.length - 1] = { type: "text", value: `${previous.value}${child.value}` }; + } else { + children.push(child); + } + } + return { + type: node.type, + ...(node.value === undefined ? {} : { value: node.value.replaceAll("\u2060", "") }), + ...(node.url === undefined ? {} : { url: node.url }), + ...(node.children === undefined ? {} : { children }), + }; +} + +function parsed(markdown: string): TestNode { + return shape(unified().use(remarkParse).use(remarkGfm).parse(markdown)); +} + +function displayed(markdown: string): TestNode { + const processor = unified().use(remarkParse).use(remarkGfm).use(remarkHideCodexWebCitations); + return shape(processor.runSync(processor.parse(markdown)) as Root); +} + +describe("Codex web-search citation markers", () => { + it.each([ + ["at the end of a sentence", `Node 24 is current. ${CITE}`, "Node 24 is current."], + ["with several sources", `Node 24 is current. ${CITE_TWO}`, "Node 24 is current."], + ["between sentences", `One. ${CITE} Two. ${CITE}${CITE_TWO} Three.`, "One. Two. Three."], + ["before punctuation", `Node 24 ${CITE}, then 25 ${CITE}.`, "Node 24, then 25."], + ["closing bold before a word", `**Result ${CITE}**next`, "**Result**next"], + [ + "closing bold before a link", + `**Result ${CITE}**[next](https://a.dev)`, + "**Result**[next](https://a.dev)", + ], + ["closing bold after punctuation", `**Result. ${CITE}** next`, "**Result.** next"], + [ + "closing bold between punctuation and a word", + `**Result. ${CITE}**next`, + "**Result.\u2060**next", + ], + [ + "closing bold between a link and a word", + `**[docs](https://a.dev) ${CITE}**next`, + "**[docs](https://a.dev)\u2060**next", + ], + [ + "opening bold between a word and punctuation", + `word**${CITE} (Result)** next`, + "word**\u2060(Result)** next", + ], + ["opening bold", `**${CITE} Result** next`, "**Result** next"], + ["after inline code inside bold", `**Use \`npm\` ${CITE}** now`, "**Use `npm`** now"], + ["as all of the bold text", `Done **${CITE}** now`, "Done now"], + ["closing strikethrough", `~~old ${CITE}~~ new`, "~~old~~ new"], + ["before bold", `See ${CITE}**this**`, "See **this**"], + ["after a link", `[docs](https://a.dev)${CITE} say so`, "[docs](https://a.dev) say so"], + [ + "after a bare URL", + `See https://nodejs.org${CITE} for more`, + "See https://nodejs.org for more", + ], + [ + "in a heading and a list", + `# News ${CITE}\n\n- item ${CITE}\n- other`, + "# News\n\n- item\n- other", + ], + [ + "in a table", + `| a | b |\n| - | - |\n| one ${CITE} | two |`, + "| a | b |\n| - | - |\n| one | two |", + ], + ["between code spans", `\`one\`${CITE}\`two\``, "`one`\u2060`two`"], + [ + "in a shortcut reference link", + `[docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + `[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + ], + [ + "in a collapsed reference link", + `[docs ${CITE}][]\n\n[docs ${CITE}]: https://a.dev`, + `[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + ], + ["in a quote", `> first\n> ${CITE} second ${CITE}`, "> first\n> second"], + ])("are hidden %s, in display and in copied text alike", (_name, markdown, expected) => { + expect(hideCodexWebCitations(markdown)).toBe(expected); + expect(displayed(markdown)).toEqual(parsed(expected)); + }); + + it.each([ + ["inline code", `Literally \`${CITE}\` here`], + ["a code fence", `\`\`\`\n${CITE}\n\`\`\``], + ["an unfinished marker", "Still streaming citeturn0search0"], + ["a link destination", `[docs](https://a.dev/${CITE})`], + ["another private-use marker", "See fileciteturn0file0 here"], + ])("are kept in %s", (_name, markdown) => { + expect(hideCodexWebCitations(markdown)).toBe(markdown); + expect(displayed(markdown)).toEqual(parsed(markdown)); + }); + + it("stay in Markdown that does not opt in, such as a user's prompt", () => { + const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS); + const tree = processor.runSync(processor.parse(`Explain this marker: ${CITE}`)) as Root; + + expect(shape(tree).children?.[0]?.children).toEqual([ + { type: "text", value: `Explain this marker: ${CITE}` }, + ]); + }); +}); diff --git a/packages/shared/src/codexWebCitations.ts b/packages/shared/src/codexWebCitations.ts new file mode 100644 index 0000000000000..83582586e596d --- /dev/null +++ b/packages/shared/src/codexWebCitations.ts @@ -0,0 +1,196 @@ +import remarkGfm from "remark-gfm"; +import remarkParse from "remark-parse"; +import { unified } from "unified"; + +// Codex web search cites sources as U+E200 "cite" (U+E202 source id)+ U+E201. The ids resolve +// against search results T3 Code never receives, so a marker can only be hidden, not linked. +export const CODEX_WEB_CITATION_START = "cite"; +const CITATION = /cite[^]*/gu; +const ENDS_PHRASE = /[\s.,;:!?)\]}]/u; +const DELIMITED = new Set(["emphasis", "strong", "delete"]); +const DELIMITER = /[*_~]/u; +const PUNCTUATION = /[\p{P}\p{S}]/u; +const WORD_JOINER = "\u2060"; + +interface MarkdownNode { + type: string; + value?: string; + url?: string; + referenceType?: string; + position?: { start: { offset?: number }; end: { offset?: number } }; + children?: MarkdownNode[]; +} + +interface SourceEdit { + readonly start: number; + readonly end: number; + readonly text: string; +} + +interface SourceRange { + readonly start: number; + readonly end: number; +} + +interface SourceRewrite { + readonly source: string; + readonly edits: SourceEdit[]; +} + +/** + * `first` and `last` say whether the text opens or closes its parent. Space left against that + * edge would sit inside an emphasis delimiter and stop it from parsing, so it goes too. + */ +function withoutCitations(text: string, first: boolean, last: boolean): string { + let kept = ""; + let cursor = 0; + for (const match of text.matchAll(CITATION)) { + kept += text.slice(cursor, match.index); + cursor = match.index + match[0].length; + const next = text[cursor]; + // With nothing after it in this node, the space separates the text from the next sibling. + if (next === undefined ? last : ENDS_PHRASE.test(next)) kept = kept.replace(/[ \t]+$/u, ""); + if (kept.endsWith("\n") || (first && kept === "")) { + while (text[cursor] === " " || text[cursor] === "\t") cursor += 1; + } + } + return kept + text.slice(cursor); +} + +function isWordCharacter(character: string | undefined): boolean { + return character !== undefined && !/\s/u.test(character) && !PUNCTUATION.test(character); +} + +/** + * A delimiter with punctuation inside it and a word character outside is literal text. A marker + * at that edge was what kept the delimiter working, so an invisible word joiner takes its place. + */ +function keepingDelimiters( + shown: string, + text: string, + range: SourceRange, + parent: SourceRange, + source: string, + first: boolean, + last: boolean, +): string { + let kept = shown; + if (last && text.endsWith("\uE201")) { + let outside = parent.end; + while (DELIMITER.test(source[outside] ?? "")) outside += 1; + const inside = shown.at(-1) ?? source[range.start - 1] ?? ""; + if (PUNCTUATION.test(inside) && isWordCharacter(source[outside])) kept += WORD_JOINER; + } + if (first && text.startsWith(CODEX_WEB_CITATION_START)) { + let outside = parent.start - 1; + while (DELIMITER.test(source[outside] ?? "")) outside -= 1; + const inside = shown[0] ?? source[range.end] ?? ""; + if (PUNCTUATION.test(inside) && isWordCharacter(source[outside])) kept = WORD_JOINER + kept; + } + return kept; +} + +/** Removing all the text between two code spans would join their backticks into one span. */ +function keepingCodeSpans(shown: string, range: SourceRange, source: string): string { + const joins = shown === "" && source[range.start - 1] === "`" && source[range.end] === "`"; + return joins ? WORD_JOINER : shown; +} + +function sourceRange(node: MarkdownNode): SourceRange | undefined { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + return start === undefined || end === undefined ? undefined : { start, end }; +} + +// Only text nodes are edited, so a marker written in code stays literal. With `rewrite`, text is +// read from the source instead of the node, whose value has escapes and references resolved. +function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { + const children = parent.children ?? []; + const parentRange = DELIMITED.has(parent.type) ? sourceRange(parent) : undefined; + parent.children = children.filter((child, index) => { + const range = sourceRange(child); + + if (child.type === "text") { + const text = rewrite + ? range + ? rewrite.source.slice(range.start, range.end) + : "" + : child.value; + if (!text?.includes(CODEX_WEB_CITATION_START)) return true; + const first = index === 0; + const last = index === children.length - 1; + const shown = withoutCitations(text, first, last); + child.value = shown; + if (rewrite && range) { + rewrite.edits.push({ + ...range, + text: keepingCodeSpans( + parentRange + ? keepingDelimiters(shown, text, range, parentRange, rewrite.source, first, last) + : shown, + range, + rewrite.source, + ), + }); + } + return shown !== ""; + } + + // A bare URL takes a marker that directly follows it into the autolink, whose text is the URL. + // A destination written out is left alone: the source rewrite edits text only. + if ( + child.url !== undefined && + child.type === "link" && + child.children?.some((label) => label.value?.includes(CODEX_WEB_CITATION_START)) + ) { + child.url = child.url.replace(CITATION, ""); + } + if (!child.children) return true; + + const editsBefore = rewrite?.edits.length ?? 0; + hideCitations(child, rewrite); + // `[label]` and `[label][]` find their definition by the label, so the original is spelled out. + if ( + rewrite && + range && + rewrite.edits.length > editsBefore && + child.type === "linkReference" && + child.referenceType !== "full" + ) { + const reference = child.referenceType === "collapsed" ? 2 : 0; + const label = rewrite.source.slice(range.start + 1, range.end - reference - 1); + rewrite.edits.push({ start: range.end - reference, end: range.end, text: `[${label}]` }); + } + if (child.children.length > 0 || !DELIMITED.has(child.type)) return true; + // Empty delimiters would show as literal asterisks. + if (rewrite && range) { + rewrite.edits.length = editsBefore; + rewrite.edits.push({ ...range, text: "" }); + } + return false; + }); +} + +/** Add only where an assistant message renders: a user may have typed a marker on purpose. */ +export function remarkHideCodexWebCitations() { + return (tree: unknown) => { + hideCitations(tree as MarkdownNode); + }; +} + +const markdownParser = unified().use(remarkParse).use(remarkGfm).freeze(); + +/** Assistant Markdown without web-search citation markers, for copying and native display. */ +export function hideCodexWebCitations(markdown: string): string { + if (!markdown.includes(CODEX_WEB_CITATION_START)) return markdown; + + const rewrite: SourceRewrite = { source: markdown, edits: [] }; + hideCitations(markdownParser.parse(markdown) as MarkdownNode, rewrite); + let shown = ""; + let cursor = 0; + for (const edit of rewrite.edits) { + shown += markdown.slice(cursor, edit.start) + edit.text; + cursor = edit.end; + } + return shown + markdown.slice(cursor); +}