From df1b36493c92132edaa70f49a8342fc01bc3c1e9 Mon Sep 17 00:00:00 2001 From: Wyatt Kane Date: Sun, 8 Nov 2026 14:34:27 -0500 Subject: [PATCH] fix(shared): keep words separated when hiding citations --- .../src/codexMarkdownDirectives.test.ts | 30 +++++++++ .../shared/src/codexMarkdownDirectives.ts | 66 +++++++++++++++++-- 2 files changed, 89 insertions(+), 7 deletions(-) diff --git a/packages/shared/src/codexMarkdownDirectives.test.ts b/packages/shared/src/codexMarkdownDirectives.test.ts index 2ddab36c2c47a..eadea26885e9b 100644 --- a/packages/shared/src/codexMarkdownDirectives.test.ts +++ b/packages/shared/src/codexMarkdownDirectives.test.ts @@ -149,6 +149,36 @@ describe.each([ }); }); +describe.each([ + { name: "remarkCodexDirectives", render: (markdown: string) => textContent(parse(markdown)) }, + { + name: "renderCodexDirectivesForCopy", + render: (markdown: string) => + textContent(parseOrdinaryMarkdown(renderCodexDirectivesForCopy(markdown))), + }, + { + name: "renderCodexMarkdownForNativeDisplay", + render: (markdown: string) => + textContent(parseOrdinaryMarkdown(renderCodexMarkdownForNativeDisplay(markdown))), + }, +])("$name citation separators", ({ render }) => { + it.each([ + ["red citeturn0search0blue", "red blue"], + ["red citeturn0search0**blue**", "red blue"], + ["**red citeturn0search0**blue", "red blue"], + ["red\tciteturn0search0blue", "red\tblue"], + ["red citeturn0search0 blue", "red blue"], + ["**red citeturn0search0** blue", "red blue"], + ['red citeturn0search0:codex-followup[blue]{prompt="Continue"}', "red blue"], + [':codex-followup[red citeturn0search0]{prompt="Continue"}blue', "red blue"], + ["red citeturn0search0", "red"], + ["red citeturn0search0\n\nblue", "redblue"], + ["## red citeturn0search0\n\nblue", "redblue"], + ])("preserves word boundaries in %s", (input, expected) => { + expect(render(input)).toBe(expected); + }); +}); + describe("native Markdown adapters", () => { it("renders follow-up suggestions as their label Markdown", () => { expect( diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 0ae164f3f0642..4d51259a48659 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -250,9 +250,37 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { delete node.url; } -function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { +function webCitationReplacement(marker: string, followingCharacter: string | undefined): string { + const separator = marker[0]; + return (separator === " " || separator === "\t") && followingCharacter?.trim() ? separator : ""; +} + +function followingInlineOffset( + node: MarkdownAstNode, + childIndex: number, + followingOffset: number | undefined, +): number | undefined { + const nextStart = node.children?.[childIndex + 1]?.position?.start.offset; + if (nextStart !== undefined) return nextStart; + return node.type === "paragraph" || node.type === "heading" || node.type === "tableCell" + ? undefined + : followingOffset; +} + +function transformCodexDirectives( + node: MarkdownAstNode, + source: string, + insideLink = false, + followingOffset?: number, +): void { if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { - node.value = node.value.replace(WEB_CITATION_MARKER, ""); + node.value = node.value.replace(WEB_CITATION_MARKER, (marker, offset: number, text: string) => + webCitationReplacement( + marker, + text[offset + marker.length] ?? + (followingOffset === undefined ? undefined : source[followingOffset]), + ), + ); return; } // GFM autolinks take a marker that directly follows a bare URL into the href. @@ -272,8 +300,13 @@ function transformCodexDirectives(node: MarkdownAstNode, source: string, insideL } const childrenInsideLink = insideLink || node.type === "link" || node.type === "linkReference"; - for (const child of node.children ?? []) { - transformCodexDirectives(child, source, childrenInsideLink); + for (const [childIndex, child] of (node.children ?? []).entries()) { + transformCodexDirectives( + child, + source, + childrenInsideLink, + followingInlineOffset(node, childIndex, followingOffset), + ); } } @@ -310,6 +343,7 @@ function collectDirectiveMatches( node: MarkdownAstNode, source: string, matches: DirectiveMatch[], + followingOffset?: number, ): void { const start = node.position?.start.offset; const end = node.position?.end.offset; @@ -317,7 +351,16 @@ function collectDirectiveMatches( if (node.type === "text") { for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) { const markerStart = start + marker.index; - matches.push({ start: markerStart, end: markerStart + marker[0].length, markdown: "" }); + const markerEnd = markerStart + marker[0].length; + const separatorOffset = markerEnd < end ? markerEnd : followingOffset; + const separator = webCitationReplacement( + marker[0], + separatorOffset === undefined ? undefined : source[separatorOffset], + ); + matches.push({ start: markerStart, end: markerEnd, markdown: "" }); + if (separator && separatorOffset !== undefined) { + matches.push({ start: separatorOffset, end: separatorOffset, markdown: separator }); + } } return; } @@ -338,7 +381,14 @@ function collectDirectiveMatches( ); } } - for (const child of node.children ?? []) collectDirectiveMatches(child, source, matches); + for (const [childIndex, child] of (node.children ?? []).entries()) { + collectDirectiveMatches( + child, + source, + matches, + followingInlineOffset(node, childIndex, followingOffset), + ); + } } function renderDirectiveMatches( @@ -348,7 +398,9 @@ function renderDirectiveMatches( const matches: DirectiveMatch[] = []; collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); let rendered = markdown; - for (const match of matches.sort((left, right) => right.start - left.start)) { + for (const match of matches.sort( + (left, right) => right.start - left.start || right.end - left.end, + )) { const replacement = replacementFor(match); if (replacement !== undefined) { rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end);