From 093201261721322385aa9d9d2767da7f100d1fdc Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 16:42:07 +0000 Subject: [PATCH 1/9] test(shared): reproduce #30979 web citation markers shown literally These tests reproduce #30979: OpenAI web-search citation markers (U+E200 cite U+E202 turnNviewM U+E201) in Codex assistant messages appear in the web chat render, the mobile Markdown render, and copied text instead of being left out. AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- .../src/codexMarkdownDirectives.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index 4db9e7c4b86d2..c4e456778b8cb 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -30,12 +30,20 @@ const FOLLOW_UP = ':codex-followup[Prepare print version]{prompt="Prepare the document for printing."}'; const ARTIFACT_TEMPLATE = '::artifact-template{skill_name="artifact-template-hello-world" skill_directory="/Users/test/.codex/skills/artifact-template-hello-world" display_name="Hello World" artifact_kind="document"}'; +// OpenAI web-search citation markers: U+E200 opens, U+E202 separates fields, U+E201 closes. +const WEB_CITATION_MARKDOWN = + "Prices rose 3%. \uE200cite\uE202turn816200view0\uE201 Rates held. \uE200cite\uE202turn0search0\uE202turn0search3\uE201"; +const WEB_CITATION_MARKER = /[\uE200-\uE202]|turn\d+(?:view|search)\d+/u; function parse(markdown: string): TestNode { const processor = unified().use(remarkParse).use(remarkCodexDirectives); return processor.runSync(processor.parse(markdown), { value: markdown }) as TestNode; } +function textContent(node: TestNode): string { + return node.value ?? (node.children ?? []).map(textContent).join(""); +} + function parseOrdinaryMarkdown(markdown: string): TestNode { return unified().use(remarkParse).parse(markdown) as TestNode; } @@ -72,6 +80,14 @@ describe("remarkCodexDirectives", () => { }); }); + it("does not display web citation markers", () => { + const text = textContent(parse(WEB_CITATION_MARKDOWN)); + + expect(text).toContain("Prices rose 3%."); + expect(text).toContain("Rates held."); + expect(text).not.toMatch(WEB_CITATION_MARKER); + }); + it.each([ "Meeting at 10:30", ":::note\ncontent\n:::", @@ -108,6 +124,19 @@ describe.each([ }); }); +describe.each([ + { name: "renderCodexDirectivesForCopy", render: renderCodexDirectivesForCopy }, + { name: "renderCodexInlineDirectivesAsMarkdown", render: renderCodexInlineDirectivesAsMarkdown }, +])("$name web citation markers", ({ render }) => { + it("leaves the markers out of the rendered Markdown", () => { + const markdown = render(WEB_CITATION_MARKDOWN); + + expect(markdown).toContain("Prices rose 3%."); + expect(markdown).toContain("Rates held."); + expect(markdown).not.toMatch(WEB_CITATION_MARKER); + }); +}); + describe("native Markdown adapters", () => { it("renders follow-up suggestions as their label Markdown", () => { expect( From 2ffb1f3861b4b25970dea9db7d83145d6c12d48c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 16:48:17 +0000 Subject: [PATCH 2/9] fix(shared): hide web citation markers in assistant messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI web search puts private-use citation markers (U+E200 cite U+E202 id… U+E201) in Codex assistant text, and T3 showed them as tofu boxes plus "cite turn…view0". T3 has no sources for those ids, so complete markers are now left out of chat display, mobile Markdown, and copied text. Markers in code spans and blocks stay literal; an unterminated marker mid-stream stays as-is until it closes. AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- .../shared/src/codexMarkdownDirectives.ts | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 6ff396eddb12a..52a2602420b15 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -22,6 +22,9 @@ const UNDERSCORE = 95; const CODEX_FILE_CITATION_NAME = "codex-file-citation"; const CODEX_FOLLOW_UP_NAME = "codex-followup"; const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; +const WEB_CITATION_OPEN = "\uE200"; +// OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids. +const WEB_CITATION_MARKER = /[ \t]?\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", @@ -248,6 +251,10 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { } function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { + if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { + node.value = node.value.replace(WEB_CITATION_MARKER, ""); + return; + } if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) { renderFileCitation(node, source, insideLink); return; @@ -296,10 +303,21 @@ interface DirectiveMatch { readonly labelRange?: readonly [start: number, end: number]; } -function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void { +function collectDirectiveMatches( + node: MarkdownAstNode, + source: string, + matches: DirectiveMatch[], +): void { const start = node.position?.start.offset; const end = node.position?.end.offset; if (start !== undefined && end !== undefined) { + if (node.type === "text") { + for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) { + const markerStart = start + marker.index; + matches.push({ start: markerStart, end: markerStart + marker[0].length, markdown: "" }); + } + return; + } if (node.data?.codexFileCitationMarkdown !== undefined) { matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown }); return; @@ -315,7 +333,7 @@ function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[ return; } } - for (const child of node.children ?? []) collectDirectiveMatches(child, matches); + for (const child of node.children ?? []) collectDirectiveMatches(child, source, matches); } function renderDirectiveMatches( @@ -323,7 +341,7 @@ function renderDirectiveMatches( replacementFor: (match: DirectiveMatch) => string | undefined, ): string { const matches: DirectiveMatch[] = []; - collectDirectiveMatches(parseCodexMarkdown(markdown), matches); + collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); let rendered = markdown; for (const match of matches.sort((left, right) => right.start - left.start)) { const replacement = match.labelRange @@ -339,7 +357,8 @@ function renderDirectiveMatches( export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && - !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) + !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) && + !markdown.includes(WEB_CITATION_OPEN) ) { return markdown; } @@ -351,7 +370,8 @@ export function renderCodexDirectivesForCopy(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) && - !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) + !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) && + !markdown.includes(WEB_CITATION_OPEN) ) { return markdown; } @@ -371,7 +391,7 @@ export function splitCodexArtifactTemplateMarkdown( } const matches: DirectiveMatch[] = []; - collectDirectiveMatches(parseCodexMarkdown(markdown), matches); + collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); const templates = matches .filter( (match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } => From 4a6ce83babd685f00e131065443928d980bbcd0a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 16:55:15 +0000 Subject: [PATCH 3/9] fix(shared): keep web citation markers out of bare-URL links GFM autolinks absorb a marker that directly follows a bare URL into the link's href, so the label rendered cleanly but the link was broken. Co-Authored-By: Claude Opus 5.5 --- packages/shared/src/codexMarkdownDirectives.test.ts | 10 ++++++++++ packages/shared/src/codexMarkdownDirectives.ts | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index c4e456778b8cb..380fbaa52ab20 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vite-plus/test"; +import remarkGfm from "remark-gfm"; import remarkParse from "remark-parse"; import { unified } from "unified"; @@ -88,6 +89,15 @@ describe("remarkCodexDirectives", () => { expect(text).not.toMatch(WEB_CITATION_MARKER); }); + it("keeps web citation markers out of a bare URL they follow", () => { + const markdown = "See https://nodejs.orgciteturn0search0 for details."; + const processor = unified().use(remarkParse).use(remarkGfm).use(remarkCodexDirectives); + const tree = processor.runSync(processor.parse(markdown), { value: markdown }) as TestNode; + const link = tree.children?.[0]?.children?.find((node) => node.type === "link"); + + expect(link?.url).toBe("https://nodejs.org"); + }); + it.each([ "Meeting at 10:30", ":::note\ncontent\n:::", diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 52a2602420b15..e73797055715b 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -255,6 +255,10 @@ function transformCodexDirectives(node: MarkdownAstNode, source: string, insideL node.value = node.value.replace(WEB_CITATION_MARKER, ""); return; } + // GFM autolinks take a marker that directly follows a bare URL into the href. + if (node.type === "link" && node.url?.includes(WEB_CITATION_OPEN)) { + node.url = node.url.replace(WEB_CITATION_MARKER, ""); + } if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) { renderFileCitation(node, source, insideLink); return; From cec6f2a24a57f3a2d33296589d032a8a207b5aff Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 16:57:24 +0000 Subject: [PATCH 4/9] fix: hide web citation markers in follow-up labels and mobile copy Follow-up labels kept their source text verbatim in the string adapters, and mobile "Copy message" copied the raw text instead of using the copy adapter the web client uses. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/features/threads/ThreadFeed.tsx | 3 ++- packages/shared/src/codexMarkdownDirectives.test.ts | 12 ++++++------ packages/shared/src/codexMarkdownDirectives.ts | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 1abfb6061a6bb..c8d67d790ecf4 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -88,6 +88,7 @@ import { } from "@t3tools/client-runtime/markdown-images"; import { resolveViewedImageAsset } from "@t3tools/client-runtime/work-log/presentation"; import { + renderCodexDirectivesForCopy, renderCodexInlineDirectivesAsMarkdown, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; @@ -2285,7 +2286,7 @@ function renderFeedEntry( { - it("leaves the markers out of the rendered Markdown", () => { - const markdown = render(WEB_CITATION_MARKDOWN); - - expect(markdown).toContain("Prices rose 3%."); - expect(markdown).toContain("Rates held."); - expect(markdown).not.toMatch(WEB_CITATION_MARKER); + it.each([ + [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], + [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], + ["`citeturn0search0`", "`citeturn0search0`"], + ])("leaves the markers out of the rendered Markdown, except in code", (input, expected) => { + expect(render(input)).toBe(expected); }); }); diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index e73797055715b..b269ddcf45ff0 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -349,7 +349,7 @@ function renderDirectiveMatches( let rendered = markdown; for (const match of matches.sort((left, right) => right.start - left.start)) { const replacement = match.labelRange - ? markdown.slice(...match.labelRange) + ? markdown.slice(...match.labelRange).replace(WEB_CITATION_MARKER, "") : replacementFor(match); if (replacement !== undefined) { rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end); From 1c6ec66ee27c6b8cd62f482a3e3effaf1368e980 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 17:08:16 +0000 Subject: [PATCH 5/9] fix(shared): keep code in follow-up labels when hiding web citations The string adapters replaced across a follow-up label's whole source, so a citation marker shown as a literal in inline code was deleted too. Follow-ups now drop only their directive syntax and the label goes through the same text node pass as everything else. Co-Authored-By: Claude Opus 5.5 --- packages/shared/src/codexMarkdownDirectives.test.ts | 1 + packages/shared/src/codexMarkdownDirectives.ts | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index 4be05ea23eb48..85a48c5be79ac 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -142,6 +142,7 @@ describe.each([ [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], ["`citeturn0search0`", "`citeturn0search0`"], + [':codex-followup[Show `citeturn0search0`]{prompt="Show it"}', "Show `citeturn0search0`"], ])("leaves the markers out of the rendered Markdown, except in code", (input, expected) => { expect(render(input)).toBe(expected); }); diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index b269ddcf45ff0..71101b6ee687e 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -304,7 +304,6 @@ interface DirectiveMatch { readonly end: number; readonly markdown?: string; readonly template?: CodexArtifactTemplate; - readonly labelRange?: readonly [start: number, end: number]; } function collectDirectiveMatches( @@ -333,8 +332,10 @@ function collectDirectiveMatches( const labelStart = node.children?.[0]?.position?.start.offset; const labelEnd = node.children?.at(-1)?.position?.end.offset; if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) { - matches.push({ start, end, labelRange: [labelStart, labelEnd] }); - return; + matches.push( + { start, end: labelStart, markdown: "" }, + { start: labelEnd, end, markdown: "" }, + ); } } for (const child of node.children ?? []) collectDirectiveMatches(child, source, matches); @@ -348,9 +349,7 @@ function renderDirectiveMatches( collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); let rendered = markdown; for (const match of matches.sort((left, right) => right.start - left.start)) { - const replacement = match.labelRange - ? markdown.slice(...match.labelRange).replace(WEB_CITATION_MARKER, "") - : replacementFor(match); + const replacement = replacementFor(match); if (replacement !== undefined) { rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end); } From 0c96fe27edee49e56277458058e1ed8d678ee9cb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 17:19:22 +0000 Subject: [PATCH 6/9] perf(shared): keep the streamed href scan off the citation-marker parse Web extracts file-link hrefs through the inline directive renderer on every streamed delta; checking it for citation markers made each delta of a cited answer a full Markdown parse that cannot change those hrefs. Mobile display, the one caller that needs marker-only text cleaned, gets its own entry point. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/features/threads/ThreadFeed.tsx | 4 ++-- packages/shared/src/codexMarkdownDirectives.test.ts | 3 ++- packages/shared/src/codexMarkdownDirectives.ts | 10 ++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index c8d67d790ecf4..7413a18be82b2 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -89,7 +89,7 @@ import { import { resolveViewedImageAsset } from "@t3tools/client-runtime/work-log/presentation"; import { renderCodexDirectivesForCopy, - renderCodexInlineDirectivesAsMarkdown, + renderCodexMarkdownForNativeDisplay, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; @@ -1258,7 +1258,7 @@ function AssistantMarkdownBlocks(props: { return ( { it.each([ [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 71101b6ee687e..0ae164f3f0642 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -360,8 +360,7 @@ function renderDirectiveMatches( export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && - !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) && - !markdown.includes(WEB_CITATION_OPEN) + !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) ) { return markdown; } @@ -369,6 +368,13 @@ export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string return renderDirectiveMatches(markdown, (match) => match.markdown); } +// Kept apart from the inline renderer: its href-only callers run on every streamed delta. +export function renderCodexMarkdownForNativeDisplay(markdown: string): string { + return markdown.includes(WEB_CITATION_OPEN) + ? renderDirectiveMatches(markdown, (match) => match.markdown) + : renderCodexInlineDirectivesAsMarkdown(markdown); +} + export function renderCodexDirectivesForCopy(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && From 322ee6133f8b48fac6502642baa6e1d334085684 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 23:17:37 +0000 Subject: [PATCH 7/9] fix(shared): keep word separators and clean link destinations when hiding web citations A marker between two words no longer joins them, and markers in link destinations and reference definitions are removed in copy and native display. AI-assisted; verified with the tests and tools listed above. --- .../src/codexMarkdownDirectives.test.ts | 3 ++ .../shared/src/codexMarkdownDirectives.ts | 49 ++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index 2ddab36c2c47a..b3783fdf73b48 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -141,6 +141,9 @@ describe.each([ ])("$name web citation markers", ({ render }) => { it.each([ [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], + ["red \uE200cite\uE202turn0search0\uE201blue", "red blue"], + ["[s](https://a.dev\uE200cite\uE202turn0search0\uE201)", "[s](https://a.dev)"], + ["[s]\n\n[s]: https://a.dev\uE200cite\uE202turn0search0\uE201", "[s]\n\n[s]: https://a.dev"], [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], ["`citeturn0search0`", "`citeturn0search0`"], [':codex-followup[Show `citeturn0search0`]{prompt="Show it"}', "Show `citeturn0search0`"], diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 0ae164f3f0642..3be29a83ddee2 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -24,7 +24,8 @@ const CODEX_FOLLOW_UP_NAME = "codex-followup"; const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; const WEB_CITATION_OPEN = "\uE200"; // OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids. -const WEB_CITATION_MARKER = /[ \t]?\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; +const WEB_CITATION_MARKER = /([ \t]?)\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; +const WORD_CHARACTER = /[\p{L}\p{N}]/u; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", @@ -250,14 +251,45 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { delete node.url; } +// The space before a marker goes with it, unless dropping it would join two words. +function webCitationReplacement(text: string, space: string, markerEnd: number): string { + return space && WORD_CHARACTER.test(text[markerEnd] ?? "") ? space : ""; +} + +function removeWebCitationMarkers(text: string): string { + return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => + webCitationReplacement(text, space, offset + marker.length), + ); +} + +function collectWebCitationMarkers( + source: string, + start: number, + end: number, + matches: DirectiveMatch[], +): void { + for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) { + const markerStart = start + marker.index; + const markerEnd = markerStart + marker[0].length; + matches.push({ + start: markerStart, + end: markerEnd, + markdown: webCitationReplacement(source, marker[1] ?? "", markerEnd), + }); + } +} + function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { - node.value = node.value.replace(WEB_CITATION_MARKER, ""); + node.value = removeWebCitationMarkers(node.value); return; } // GFM autolinks take a marker that directly follows a bare URL into the href. - if (node.type === "link" && node.url?.includes(WEB_CITATION_OPEN)) { - node.url = node.url.replace(WEB_CITATION_MARKER, ""); + if ( + (node.type === "link" || node.type === "definition") && + node.url?.includes(WEB_CITATION_OPEN) + ) { + node.url = removeWebCitationMarkers(node.url); } if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) { renderFileCitation(node, source, insideLink); @@ -315,12 +347,13 @@ function collectDirectiveMatches( const end = node.position?.end.offset; if (start !== undefined && end !== undefined) { if (node.type === "text") { - for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) { - const markerStart = start + marker.index; - matches.push({ start: markerStart, end: markerStart + marker[0].length, markdown: "" }); - } + collectWebCitationMarkers(source, start, end, matches); return; } + if (node.type === "link" || node.type === "definition") { + const destinationStart = node.children?.at(-1)?.position?.end.offset ?? start; + collectWebCitationMarkers(source, destinationStart, end, matches); + } if (node.data?.codexFileCitationMarkdown !== undefined) { matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown }); return; From 639a6caa1e52312e796ba08c9556cfe26574e71a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 23:35:23 +0000 Subject: [PATCH 8/9] fix(shared): keep the space before a citation marker that precedes Markdown A marker followed by emphasis, a link, or a code span no longer joins the words around it. The space goes only before whitespace, closing punctuation, or the end of the text. AI-assisted; verified with the tests and tools listed above. --- .../src/codexMarkdownDirectives.test.ts | 4 +++ .../shared/src/codexMarkdownDirectives.ts | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index b3783fdf73b48..8b90cdb019dcf 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -88,6 +88,7 @@ describe("remarkCodexDirectives", () => { expect(text).toContain("Prices rose 3%."); expect(text).toContain("Rates held."); expect(text).not.toMatch(WEB_CITATION_MARKER); + expect(textContent(parse("red \uE200cite\uE202turn0search0\uE201**blue**"))).toBe("red blue"); }); it("keeps web citation markers out of a bare URL they follow", () => { @@ -142,6 +143,9 @@ describe.each([ it.each([ [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], ["red \uE200cite\uE202turn0search0\uE201blue", "red blue"], + ["red \uE200cite\uE202turn0search0\uE201**blue**", "red **blue**"], + ["red \uE200cite\uE202turn0search0\uE201[blue](https://a.dev)", "red [blue](https://a.dev)"], + ["red \uE200cite\uE202turn0search0\uE201`blue`", "red `blue`"], ["[s](https://a.dev\uE200cite\uE202turn0search0\uE201)", "[s](https://a.dev)"], ["[s]\n\n[s]: https://a.dev\uE200cite\uE202turn0search0\uE201", "[s]\n\n[s]: https://a.dev"], [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 3be29a83ddee2..cf7dc6c103415 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -25,7 +25,7 @@ const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; const WEB_CITATION_OPEN = "\uE200"; // OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids. const WEB_CITATION_MARKER = /([ \t]?)\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; -const WORD_CHARACTER = /[\p{L}\p{N}]/u; +const DROPS_PRECEDING_SPACE = /^(?:\s|[.,;:!?)\]]|$)/u; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", @@ -251,15 +251,17 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { delete node.url; } -// The space before a marker goes with it, unless dropping it would join two words. -function webCitationReplacement(text: string, space: string, markerEnd: number): string { - return space && WORD_CHARACTER.test(text[markerEnd] ?? "") ? space : ""; +// The space before a marker goes with it only where nothing it separates follows. +function webCitationReplacement(space: string, following: string): string { + return DROPS_PRECEDING_SPACE.test(following) ? "" : space; } -function removeWebCitationMarkers(text: string): string { - return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => - webCitationReplacement(text, space, offset + marker.length), - ); +// `after` is the source text past this string, for a marker that ends it. +function removeWebCitationMarkers(text: string, after = ""): string { + return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => { + const markerEnd = offset + marker.length; + return webCitationReplacement(space, text.slice(markerEnd, markerEnd + 1) || after); + }); } function collectWebCitationMarkers( @@ -274,14 +276,18 @@ function collectWebCitationMarkers( matches.push({ start: markerStart, end: markerEnd, - markdown: webCitationReplacement(source, marker[1] ?? "", markerEnd), + markdown: webCitationReplacement(marker[1] ?? "", source.slice(markerEnd, markerEnd + 1)), }); } } function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { - node.value = removeWebCitationMarkers(node.value); + const end = node.position?.end.offset; + node.value = removeWebCitationMarkers( + node.value, + end === undefined ? "" : source.slice(end, end + 1), + ); return; } // GFM autolinks take a marker that directly follows a bare URL into the href. From 7c7443687f3dbe63a3f2f4d473016f100244e497 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 23:55:48 +0000 Subject: [PATCH 9/9] fix(shared): keep emphasis and file citations intact when hiding web citations A marker before closing emphasis no longer leaves a space that breaks the emphasis, and a marker inside a file citation no longer cuts off the text after it in copied and mobile Markdown. AI-assisted; verified with the tests and tools listed above. --- .../src/codexMarkdownDirectives.test.ts | 5 +++ .../shared/src/codexMarkdownDirectives.ts | 31 ++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index 8b90cdb019dcf..c09914a5f05f6 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -146,6 +146,11 @@ describe.each([ ["red \uE200cite\uE202turn0search0\uE201**blue**", "red **blue**"], ["red \uE200cite\uE202turn0search0\uE201[blue](https://a.dev)", "red [blue](https://a.dev)"], ["red \uE200cite\uE202turn0search0\uE201`blue`", "red `blue`"], + ["**Result \uE200cite\uE202turn0search0\uE201**.", "**Result**."], + [ + ':codex-file-citation{path="src/foo.ts" note="\uE200cite\uE202turn0search0\uE201"} trailing', + "[foo.ts]() trailing", + ], ["[s](https://a.dev\uE200cite\uE202turn0search0\uE201)", "[s](https://a.dev)"], ["[s]\n\n[s]: https://a.dev\uE200cite\uE202turn0search0\uE201", "[s]\n\n[s]: https://a.dev"], [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index cf7dc6c103415..06f3130cd222a 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -25,7 +25,8 @@ const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; const WEB_CITATION_OPEN = "\uE200"; // OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids. const WEB_CITATION_MARKER = /([ \t]?)\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; -const DROPS_PRECEDING_SPACE = /^(?:\s|[.,;:!?)\]]|$)/u; +// Past any closing emphasis or code delimiters, nothing the space separates follows. +const DROPS_PRECEDING_SPACE = /[*_~`]*(?:\s|[.,;:!?)\]]|$)/uy; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", @@ -251,16 +252,18 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { delete node.url; } -// The space before a marker goes with it only where nothing it separates follows. -function webCitationReplacement(space: string, following: string): string { - return DROPS_PRECEDING_SPACE.test(following) ? "" : space; +function webCitationReplacement(space: string, text: string, markerEnd: number): string { + DROPS_PRECEDING_SPACE.lastIndex = markerEnd; + return DROPS_PRECEDING_SPACE.test(text) ? "" : space; } -// `after` is the source text past this string, for a marker that ends it. -function removeWebCitationMarkers(text: string, after = ""): string { +// A marker that ends `text` is judged by the source past `textEnd`. +function removeWebCitationMarkers(text: string, source = "", textEnd = 0): string { return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => { const markerEnd = offset + marker.length; - return webCitationReplacement(space, text.slice(markerEnd, markerEnd + 1) || after); + return markerEnd < text.length + ? webCitationReplacement(space, text, markerEnd) + : webCitationReplacement(space, source, textEnd); }); } @@ -276,17 +279,17 @@ function collectWebCitationMarkers( matches.push({ start: markerStart, end: markerEnd, - markdown: webCitationReplacement(marker[1] ?? "", source.slice(markerEnd, markerEnd + 1)), + markdown: webCitationReplacement(marker[1] ?? "", source, markerEnd), }); } } function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { - const end = node.position?.end.offset; node.value = removeWebCitationMarkers( node.value, - end === undefined ? "" : source.slice(end, end + 1), + source, + node.position?.end.offset ?? source.length, ); return; } @@ -356,10 +359,6 @@ function collectDirectiveMatches( collectWebCitationMarkers(source, start, end, matches); return; } - if (node.type === "link" || node.type === "definition") { - const destinationStart = node.children?.at(-1)?.position?.end.offset ?? start; - collectWebCitationMarkers(source, destinationStart, end, matches); - } if (node.data?.codexFileCitationMarkdown !== undefined) { matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown }); return; @@ -368,6 +367,10 @@ function collectDirectiveMatches( matches.push({ start, end, template: node.data.codexArtifactTemplate }); return; } + if (node.type === "link" || node.type === "definition") { + const destinationStart = node.children?.at(-1)?.position?.end.offset ?? start; + collectWebCitationMarkers(source, destinationStart, end, matches); + } const labelStart = node.children?.[0]?.position?.start.offset; const labelEnd = node.children?.at(-1)?.position?.end.offset; if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) {