From b52493afe69f267e976ef0ecb4b412dd8c2ac417 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 19:42:14 +0000 Subject: [PATCH 1/4] fix(shared): hide Codex web-search citation markers in assistant messages Codex web search leaves private-use citation markers in assistant text (U+E200 "cite", U+E202-separated source ids, U+E201). T3 Code never receives the search results those ids point at, so it printed the raw markers on web, desktop and mobile and kept them in copied text. Complete markers are now left out of assistant messages where they are shown and copied. The stored message is unchanged. - Removal follows the parsed Markdown: only text nodes are edited, so markers in code stay literal, and space left against the edge of bold, italic or strikethrough is dropped with the marker so the delimiters still parse in copied text and native display. - It is opt-in. Web enables it on the assistant row only, so a prompt a user typed with a literal marker still shows in full. - A bare URL directly followed by a marker no longer links to a URL with the marker in it. Refs pingdotgg/t3code#30979 Co-authored-by: Julius Marminge Co-Authored-By: Claude Fable 5.2 --- .../src/features/threads/ThreadFeed.tsx | 5 +- apps/web/src/components/ChatMarkdown.tsx | 16 ++- .../components/chat/MessagesTimeline.logic.ts | 7 +- .../components/chat/TimelineAssistantRows.tsx | 1 + packages/shared/package.json | 4 + packages/shared/src/codexWebCitations.test.ts | 106 ++++++++++++++++ packages/shared/src/codexWebCitations.ts | 113 ++++++++++++++++++ 7 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 packages/shared/src/codexWebCitations.test.ts create mode 100644 packages/shared/src/codexWebCitations.ts diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 8e8c1c4955b29..4b8721936ab95 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -92,6 +92,7 @@ import { renderCodexInlineDirectivesAsMarkdown, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; +import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; import { imageMimeType } from "@t3tools/shared/image"; import { videoMimeType } from "@t3tools/shared/video"; @@ -1217,7 +1218,7 @@ function AssistantMarkdownBlocks(props: { return ( { markdownRef.current = node; }; - const sharedRemarkPlugins = + const sharedRemarkPlugins = withExtraPlugins( CHAT_MARKDOWN_REMARK_PLUGIN_SETS[ componentState.mathEnabled ? lineBreaks @@ -1846,7 +1854,11 @@ function ChatMarkdown({ : lineBreaks ? "breaks" : "plain" - ]; + ], + hideWebCitations && text.includes(CODEX_WEB_CITATION_START) + ? WEB_CITATION_REMARK_PLUGINS + : EMPTY_REMARK_PLUGINS, + ); // The incremental parser keeps per-instance state, so its output is not shared. const uncachedPlugins = incrementalParsing; const remarkPlugins = incrementalParsing diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 90690dc0810ec..a3256ab28fb32 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -11,6 +11,7 @@ import { } from "@t3tools/client-runtime/user-message"; import type { HandoffTimelineRun } from "@t3tools/client-runtime/handoff"; import { renderCodexDirectivesForCopy } from "@t3tools/shared/codexMarkdownDirectives"; +import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations"; import { commandDisplayText, commandProgramName, @@ -690,7 +691,11 @@ export function resolveAssistantMessageCopyState({ const hasText = text !== null && text.trim().length > 0; const visible = showCopyButton && hasText && !streaming; return { - text: hasText ? (visible ? renderCodexDirectivesForCopy(text) : text) : null, + text: hasText + ? visible + ? hideCodexWebCitations(renderCodexDirectivesForCopy(text)) + : text + : null, visible, }; } diff --git a/apps/web/src/components/chat/TimelineAssistantRows.tsx b/apps/web/src/components/chat/TimelineAssistantRows.tsx index 36c04da893275..d326d57369abf 100644 --- a/apps/web/src/components/chat/TimelineAssistantRows.tsx +++ b/apps/web/src/components/chat/TimelineAssistantRows.tsx @@ -57,6 +57,7 @@ export function AssistantTimelineRow({ row }: { row: Extract; +} + +// Positions and text node boundaries differ between a displayed tree and a reparsed copy; +// structure and text must not. +function shape(node: TestNode): TestNode { + const children: Array = []; + for (const child of (node.children ?? []).map(shape)) { + const previous = children.at(-1); + if (child.type === "text" && previous?.type === "text") { + children[children.length - 1] = { type: "text", value: `${previous.value}${child.value}` }; + } else { + children.push(child); + } + } + return { + type: node.type, + ...(node.value === undefined ? {} : { value: node.value }), + ...(node.url === undefined ? {} : { url: node.url }), + ...(node.children === undefined ? {} : { children }), + }; +} + +function parsed(markdown: string): TestNode { + return shape(unified().use(remarkParse).use(remarkGfm).parse(markdown)); +} + +function displayed(markdown: string): TestNode { + const processor = unified().use(remarkParse).use(remarkGfm).use(remarkHideCodexWebCitations); + return shape(processor.runSync(processor.parse(markdown)) as Root); +} + +describe("Codex web-search citation markers", () => { + it.each([ + ["at the end of a sentence", `Node 24 is current. ${CITE}`, "Node 24 is current."], + ["with several sources", `Node 24 is current. ${CITE_TWO}`, "Node 24 is current."], + ["between sentences", `One. ${CITE} Two. ${CITE}${CITE_TWO} Three.`, "One. Two. Three."], + ["before punctuation", `Node 24 ${CITE}, then 25 ${CITE}.`, "Node 24, then 25."], + ["closing bold before a word", `**Result ${CITE}**next`, "**Result**next"], + [ + "closing bold before a link", + `**Result ${CITE}**[next](https://a.dev)`, + "**Result**[next](https://a.dev)", + ], + ["opening bold", `**${CITE} Result** next`, "**Result** next"], + ["after inline code inside bold", `**Use \`npm\` ${CITE}** now`, "**Use `npm`** now"], + ["as all of the bold text", `Done **${CITE}** now`, "Done now"], + ["closing strikethrough", `~~old ${CITE}~~ new`, "~~old~~ new"], + ["before bold", `See ${CITE}**this**`, "See **this**"], + ["after a link", `[docs](https://a.dev)${CITE} say so`, "[docs](https://a.dev) say so"], + [ + "after a bare URL", + `See https://nodejs.org${CITE} for more`, + "See https://nodejs.org for more", + ], + [ + "in a heading and a list", + `# News ${CITE}\n\n- item ${CITE}\n- other`, + "# News\n\n- item\n- other", + ], + [ + "in a table", + `| a | b |\n| - | - |\n| one ${CITE} | two |`, + "| a | b |\n| - | - |\n| one | two |", + ], + ["in a quote", `> first\n> ${CITE} second ${CITE}`, "> first\n> second"], + ])("are hidden %s, in display and in copied text alike", (_name, markdown, expected) => { + expect(hideCodexWebCitations(markdown)).toBe(expected); + expect(displayed(markdown)).toEqual(parsed(expected)); + }); + + it.each([ + ["inline code", `Literally \`${CITE}\` here`], + ["a code fence", `\`\`\`\n${CITE}\n\`\`\``], + ["an unfinished marker", "Still streaming citeturn0search0"], + ["another private-use marker", "See fileciteturn0file0 here"], + ])("are kept in %s", (_name, markdown) => { + expect(hideCodexWebCitations(markdown)).toBe(markdown); + expect(displayed(markdown)).toEqual(parsed(markdown)); + }); + + it("stay in Markdown that does not opt in, such as a user's prompt", () => { + const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS); + const tree = processor.runSync(processor.parse(`Explain this marker: ${CITE}`)) as Root; + + expect(shape(tree).children?.[0]?.children).toEqual([ + { type: "text", value: `Explain this marker: ${CITE}` }, + ]); + }); +}); diff --git a/packages/shared/src/codexWebCitations.ts b/packages/shared/src/codexWebCitations.ts new file mode 100644 index 0000000000000..07f92892ea449 --- /dev/null +++ b/packages/shared/src/codexWebCitations.ts @@ -0,0 +1,113 @@ +import remarkGfm from "remark-gfm"; +import remarkParse from "remark-parse"; +import { unified } from "unified"; + +// Codex web search cites sources as U+E200 "cite" (U+E202 source id)+ U+E201. The ids resolve +// against search results T3 Code never receives, so a marker can only be hidden, not linked. +export const CODEX_WEB_CITATION_START = "cite"; +const CITATION = /cite[^]*/gu; +const ENDS_PHRASE = /[\s.,;:!?)\]}]/u; +const EMPTIES_WITH_ITS_TEXT = new Set(["emphasis", "strong", "delete"]); + +interface MarkdownNode { + type: string; + value?: string; + url?: string; + position?: { start: { offset?: number }; end: { offset?: number } }; + children?: MarkdownNode[]; +} + +interface SourceEdit { + readonly start: number; + readonly end: number; + readonly text: string; +} + +interface SourceRewrite { + readonly source: string; + readonly edits: SourceEdit[]; +} + +/** + * `first` and `last` say whether the text opens or closes its parent. Space left against that + * edge would sit inside an emphasis delimiter and stop it from parsing, so it goes too. + */ +function withoutCitations(text: string, first: boolean, last: boolean): string { + let kept = ""; + let cursor = 0; + for (const match of text.matchAll(CITATION)) { + kept += text.slice(cursor, match.index); + cursor = match.index + match[0].length; + const next = text[cursor]; + // With nothing after it in this node, the space separates the text from the next sibling. + if (next === undefined ? last : ENDS_PHRASE.test(next)) kept = kept.replace(/[ \t]+$/u, ""); + if (kept.endsWith("\n") || (first && kept === "")) { + while (text[cursor] === " " || text[cursor] === "\t") cursor += 1; + } + } + return kept + text.slice(cursor); +} + +// Only text nodes are edited, so a marker written in code stays literal. With `rewrite`, text is +// read from the source instead of the node, whose value has escapes and references resolved. +function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { + const children = parent.children ?? []; + parent.children = children.filter((child, index) => { + const start = child.position?.start.offset; + const end = child.position?.end.offset; + const range = start === undefined || end === undefined ? undefined : { start, end }; + + if (child.type === "text") { + const text = rewrite + ? range + ? rewrite.source.slice(range.start, range.end) + : "" + : child.value; + if (!text?.includes(CODEX_WEB_CITATION_START)) return true; + const shown = withoutCitations(text, index === 0, index === children.length - 1); + child.value = shown; + if (range) rewrite?.edits.push({ ...range, text: shown }); + return shown !== ""; + } + + // A bare URL takes a marker that directly follows it into the autolink. + if (child.type === "link" && child.url?.includes(CODEX_WEB_CITATION_START)) { + child.url = child.url.replace(CITATION, ""); + } + if (!child.children) return true; + + const editsBefore = rewrite?.edits.length ?? 0; + hideCitations(child, rewrite); + if (child.children.length > 0 || !EMPTIES_WITH_ITS_TEXT.has(child.type)) return true; + // Empty delimiters would show as literal asterisks. + if (rewrite && range) { + rewrite.edits.length = editsBefore; + rewrite.edits.push({ ...range, text: "" }); + } + return false; + }); +} + +/** Add only where an assistant message renders: a user may have typed a marker on purpose. */ +export function remarkHideCodexWebCitations() { + return (tree: unknown) => { + hideCitations(tree as MarkdownNode); + }; +} + +const markdownParser = unified().use(remarkParse).use(remarkGfm).freeze(); + +/** Assistant Markdown without web-search citation markers, for copying and native display. */ +export function hideCodexWebCitations(markdown: string): string { + if (!markdown.includes(CODEX_WEB_CITATION_START)) return markdown; + + const rewrite: SourceRewrite = { source: markdown, edits: [] }; + hideCitations(markdownParser.parse(markdown) as MarkdownNode, rewrite); + let shown = ""; + let cursor = 0; + for (const edit of rewrite.edits) { + shown += markdown.slice(cursor, edit.start) + edit.text; + cursor = edit.end; + } + return shown + markdown.slice(cursor); +} From 07d3f68ac5fb4a0861b239c86f450a8a0d4d7fd8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 19:54:44 +0000 Subject: [PATCH 2/4] fix(shared): copied bold text keeps its delimiters when a citation follows punctuation Removing a citation marker from the edge of bold, italic or strikethrough text could leave punctuation just inside the delimiter and a word character just outside it. CommonMark reads such a delimiter as literal text, so `**Result. **next` was copied, and shown on mobile, as `**Result.**next` with visible asterisks. In that one arrangement the rewritten source now keeps an invisible word joiner where the marker was, which is what kept the delimiter valid before. The web display tree is unaffected. Co-Authored-By: Claude Fable 5.2 --- packages/shared/src/codexWebCitations.test.ts | 23 ++++++- packages/shared/src/codexWebCitations.ts | 69 +++++++++++++++++-- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/packages/shared/src/codexWebCitations.test.ts b/packages/shared/src/codexWebCitations.test.ts index 4a0f7e37d65bf..5d7324fb1ed28 100644 --- a/packages/shared/src/codexWebCitations.test.ts +++ b/packages/shared/src/codexWebCitations.test.ts @@ -17,12 +17,13 @@ interface TestNode { readonly children?: ReadonlyArray; } -// Positions and text node boundaries differ between a displayed tree and a reparsed copy; -// structure and text must not. +// Positions and text node boundaries differ between a displayed tree and a reparsed copy, and a +// copy can carry an invisible word joiner; structure and visible text must not differ. function shape(node: TestNode): TestNode { const children: Array = []; for (const child of (node.children ?? []).map(shape)) { const previous = children.at(-1); + if (child.type === "text" && child.value === "") continue; if (child.type === "text" && previous?.type === "text") { children[children.length - 1] = { type: "text", value: `${previous.value}${child.value}` }; } else { @@ -31,7 +32,7 @@ function shape(node: TestNode): TestNode { } return { type: node.type, - ...(node.value === undefined ? {} : { value: node.value }), + ...(node.value === undefined ? {} : { value: node.value.replaceAll("\u2060", "") }), ...(node.url === undefined ? {} : { url: node.url }), ...(node.children === undefined ? {} : { children }), }; @@ -58,6 +59,22 @@ describe("Codex web-search citation markers", () => { `**Result ${CITE}**[next](https://a.dev)`, "**Result**[next](https://a.dev)", ], + ["closing bold after punctuation", `**Result. ${CITE}** next`, "**Result.** next"], + [ + "closing bold between punctuation and a word", + `**Result. ${CITE}**next`, + "**Result.\u2060**next", + ], + [ + "closing bold between a link and a word", + `**[docs](https://a.dev) ${CITE}**next`, + "**[docs](https://a.dev)\u2060**next", + ], + [ + "opening bold between a word and punctuation", + `word**${CITE} (Result)** next`, + "word**\u2060(Result)** next", + ], ["opening bold", `**${CITE} Result** next`, "**Result** next"], ["after inline code inside bold", `**Use \`npm\` ${CITE}** now`, "**Use `npm`** now"], ["as all of the bold text", `Done **${CITE}** now`, "Done now"], diff --git a/packages/shared/src/codexWebCitations.ts b/packages/shared/src/codexWebCitations.ts index 07f92892ea449..4880dd88919a8 100644 --- a/packages/shared/src/codexWebCitations.ts +++ b/packages/shared/src/codexWebCitations.ts @@ -7,7 +7,10 @@ import { unified } from "unified"; export const CODEX_WEB_CITATION_START = "cite"; const CITATION = /cite[^]*/gu; const ENDS_PHRASE = /[\s.,;:!?)\]}]/u; -const EMPTIES_WITH_ITS_TEXT = new Set(["emphasis", "strong", "delete"]); +const DELIMITED = new Set(["emphasis", "strong", "delete"]); +const DELIMITER = /[*_~]/u; +const PUNCTUATION = /[\p{P}\p{S}]/u; +const WORD_JOINER = "\u2060"; interface MarkdownNode { type: string; @@ -23,6 +26,11 @@ interface SourceEdit { readonly text: string; } +interface SourceRange { + readonly start: number; + readonly end: number; +} + interface SourceRewrite { readonly source: string; readonly edits: SourceEdit[]; @@ -48,14 +56,52 @@ function withoutCitations(text: string, first: boolean, last: boolean): string { return kept + text.slice(cursor); } +function isWordCharacter(character: string | undefined): boolean { + return character !== undefined && !/\s/u.test(character) && !PUNCTUATION.test(character); +} + +/** + * A delimiter with punctuation inside it and a word character outside is literal text. A marker + * at that edge was what kept the delimiter working, so an invisible word joiner takes its place. + */ +function keepingDelimiters( + shown: string, + text: string, + range: SourceRange, + parent: SourceRange, + source: string, + first: boolean, + last: boolean, +): string { + let kept = shown; + if (last && text.endsWith("\uE201")) { + let outside = parent.end; + while (DELIMITER.test(source[outside] ?? "")) outside += 1; + const inside = shown.at(-1) ?? source[range.start - 1] ?? ""; + if (PUNCTUATION.test(inside) && isWordCharacter(source[outside])) kept += WORD_JOINER; + } + if (first && text.startsWith(CODEX_WEB_CITATION_START)) { + let outside = parent.start - 1; + while (DELIMITER.test(source[outside] ?? "")) outside -= 1; + const inside = shown[0] ?? source[range.end] ?? ""; + if (PUNCTUATION.test(inside) && isWordCharacter(source[outside])) kept = WORD_JOINER + kept; + } + return kept; +} + +function sourceRange(node: MarkdownNode): SourceRange | undefined { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + return start === undefined || end === undefined ? undefined : { start, end }; +} + // Only text nodes are edited, so a marker written in code stays literal. With `rewrite`, text is // read from the source instead of the node, whose value has escapes and references resolved. function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { const children = parent.children ?? []; + const parentRange = DELIMITED.has(parent.type) ? sourceRange(parent) : undefined; parent.children = children.filter((child, index) => { - const start = child.position?.start.offset; - const end = child.position?.end.offset; - const range = start === undefined || end === undefined ? undefined : { start, end }; + const range = sourceRange(child); if (child.type === "text") { const text = rewrite @@ -64,9 +110,18 @@ function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { : "" : child.value; if (!text?.includes(CODEX_WEB_CITATION_START)) return true; - const shown = withoutCitations(text, index === 0, index === children.length - 1); + const first = index === 0; + const last = index === children.length - 1; + const shown = withoutCitations(text, first, last); child.value = shown; - if (range) rewrite?.edits.push({ ...range, text: shown }); + if (rewrite && range) { + rewrite.edits.push({ + ...range, + text: parentRange + ? keepingDelimiters(shown, text, range, parentRange, rewrite.source, first, last) + : shown, + }); + } return shown !== ""; } @@ -78,7 +133,7 @@ function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { const editsBefore = rewrite?.edits.length ?? 0; hideCitations(child, rewrite); - if (child.children.length > 0 || !EMPTIES_WITH_ITS_TEXT.has(child.type)) return true; + if (child.children.length > 0 || !DELIMITED.has(child.type)) return true; // Empty delimiters would show as literal asterisks. if (rewrite && range) { rewrite.edits.length = editsBefore; From 2dae72c74749a85643daa7bc67e10e05805d35a7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 20:11:09 +0000 Subject: [PATCH 3/4] fix(shared): a citation marker in a written-out link destination is left as is Display removed a marker from any link's URL, but copied text and native display only edit text, so the two disagreed for a link whose destination was written out with a marker in it. Only an autolink, whose text is its URL, has its URL cleaned now, which is the case a bare URL followed by a marker needs. Co-Authored-By: Claude Fable 5.2 --- packages/shared/src/codexWebCitations.test.ts | 1 + packages/shared/src/codexWebCitations.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/codexWebCitations.test.ts b/packages/shared/src/codexWebCitations.test.ts index 5d7324fb1ed28..eb55a552cc7d6 100644 --- a/packages/shared/src/codexWebCitations.test.ts +++ b/packages/shared/src/codexWebCitations.test.ts @@ -106,6 +106,7 @@ describe("Codex web-search citation markers", () => { ["inline code", `Literally \`${CITE}\` here`], ["a code fence", `\`\`\`\n${CITE}\n\`\`\``], ["an unfinished marker", "Still streaming citeturn0search0"], + ["a link destination", `[docs](https://a.dev/${CITE})`], ["another private-use marker", "See fileciteturn0file0 here"], ])("are kept in %s", (_name, markdown) => { expect(hideCodexWebCitations(markdown)).toBe(markdown); diff --git a/packages/shared/src/codexWebCitations.ts b/packages/shared/src/codexWebCitations.ts index 4880dd88919a8..f9af501dcd88b 100644 --- a/packages/shared/src/codexWebCitations.ts +++ b/packages/shared/src/codexWebCitations.ts @@ -125,8 +125,13 @@ function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { return shown !== ""; } - // A bare URL takes a marker that directly follows it into the autolink. - if (child.type === "link" && child.url?.includes(CODEX_WEB_CITATION_START)) { + // A bare URL takes a marker that directly follows it into the autolink, whose text is the URL. + // A destination written out is left alone: the source rewrite edits text only. + if ( + child.url !== undefined && + child.type === "link" && + child.children?.some((label) => label.value?.includes(CODEX_WEB_CITATION_START)) + ) { child.url = child.url.replace(CITATION, ""); } if (!child.children) return true; From 225737f67124af274f14d53fac221dc0a6606904 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 20:30:35 +0000 Subject: [PATCH 4/4] fix(shared): hiding a citation keeps reference links resolving and code spans apart Copied and native Markdown rewrote a shortcut or collapsed reference link's label without its definition, so the link stopped resolving, and removing a marker between two code spans joined their backticks into one span. The original label is now written out as the reference, and an invisible word joiner keeps the spans separate. Co-Authored-By: Claude Fable 5.2 --- packages/shared/src/codexWebCitations.test.ts | 11 +++++++ packages/shared/src/codexWebCitations.ts | 29 +++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/shared/src/codexWebCitations.test.ts b/packages/shared/src/codexWebCitations.test.ts index eb55a552cc7d6..1dd6557448780 100644 --- a/packages/shared/src/codexWebCitations.test.ts +++ b/packages/shared/src/codexWebCitations.test.ts @@ -96,6 +96,17 @@ describe("Codex web-search citation markers", () => { `| a | b |\n| - | - |\n| one ${CITE} | two |`, "| a | b |\n| - | - |\n| one | two |", ], + ["between code spans", `\`one\`${CITE}\`two\``, "`one`\u2060`two`"], + [ + "in a shortcut reference link", + `[docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + `[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + ], + [ + "in a collapsed reference link", + `[docs ${CITE}][]\n\n[docs ${CITE}]: https://a.dev`, + `[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`, + ], ["in a quote", `> first\n> ${CITE} second ${CITE}`, "> first\n> second"], ])("are hidden %s, in display and in copied text alike", (_name, markdown, expected) => { expect(hideCodexWebCitations(markdown)).toBe(expected); diff --git a/packages/shared/src/codexWebCitations.ts b/packages/shared/src/codexWebCitations.ts index f9af501dcd88b..83582586e596d 100644 --- a/packages/shared/src/codexWebCitations.ts +++ b/packages/shared/src/codexWebCitations.ts @@ -16,6 +16,7 @@ interface MarkdownNode { type: string; value?: string; url?: string; + referenceType?: string; position?: { start: { offset?: number }; end: { offset?: number } }; children?: MarkdownNode[]; } @@ -89,6 +90,12 @@ function keepingDelimiters( return kept; } +/** Removing all the text between two code spans would join their backticks into one span. */ +function keepingCodeSpans(shown: string, range: SourceRange, source: string): string { + const joins = shown === "" && source[range.start - 1] === "`" && source[range.end] === "`"; + return joins ? WORD_JOINER : shown; +} + function sourceRange(node: MarkdownNode): SourceRange | undefined { const start = node.position?.start.offset; const end = node.position?.end.offset; @@ -117,9 +124,13 @@ function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { if (rewrite && range) { rewrite.edits.push({ ...range, - text: parentRange - ? keepingDelimiters(shown, text, range, parentRange, rewrite.source, first, last) - : shown, + text: keepingCodeSpans( + parentRange + ? keepingDelimiters(shown, text, range, parentRange, rewrite.source, first, last) + : shown, + range, + rewrite.source, + ), }); } return shown !== ""; @@ -138,6 +149,18 @@ function hideCitations(parent: MarkdownNode, rewrite?: SourceRewrite): void { const editsBefore = rewrite?.edits.length ?? 0; hideCitations(child, rewrite); + // `[label]` and `[label][]` find their definition by the label, so the original is spelled out. + if ( + rewrite && + range && + rewrite.edits.length > editsBefore && + child.type === "linkReference" && + child.referenceType !== "full" + ) { + const reference = child.referenceType === "collapsed" ? 2 : 0; + const label = rewrite.source.slice(range.start + 1, range.end - reference - 1); + rewrite.edits.push({ start: range.end - reference, end: range.end, text: `[${label}]` }); + } if (child.children.length > 0 || !DELIMITED.has(child.type)) return true; // Empty delimiters would show as literal asterisks. if (rewrite && range) {