diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index e7f4bfbc3d020..3f4e0e07a92fb 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -89,7 +89,8 @@ import { } from "@t3tools/client-runtime/markdown-images"; import { resolveViewedImageAsset } from "@t3tools/client-runtime/work-log/presentation"; import { - renderCodexInlineDirectivesAsMarkdown, + renderCodexDirectivesForCopy, + renderCodexMarkdownForNativeDisplay, splitCodexArtifactTemplateMarkdown, } from "@t3tools/shared/codexMarkdownDirectives"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; @@ -1208,7 +1209,7 @@ function AssistantMarkdownBlocks(props: { return ( { }); }); + it("does not display web citation markers", () => { + const text = textContent(parse(WEB_CITATION_MARKDOWN)); + + expect(text).toContain("Prices rose 3%."); + expect(text).toContain("Rates held."); + expect(text).not.toMatch(WEB_CITATION_MARKER); + expect(textContent(parse("red \uE200cite\uE202turn0search0\uE201**blue**"))).toBe("red blue"); + }); + + it("keeps web citation markers out of a bare URL they follow", () => { + const markdown = "See https://nodejs.orgciteturn0search0 for details."; + const processor = unified().use(remarkParse).use(remarkGfm).use(remarkCodexDirectives); + const tree = processor.runSync(processor.parse(markdown), { value: markdown }) as TestNode; + const link = tree.children?.[0]?.children?.find((node) => node.type === "link"); + + expect(link?.url).toBe("https://nodejs.org"); + }); + it.each([ "Meeting at 10:30", ":::note\ncontent\n:::", @@ -108,6 +136,31 @@ describe.each([ }); }); +describe.each([ + { name: "renderCodexDirectivesForCopy", render: renderCodexDirectivesForCopy }, + { name: "renderCodexMarkdownForNativeDisplay", render: renderCodexMarkdownForNativeDisplay }, +])("$name web citation markers", ({ render }) => { + it.each([ + [WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."], + ["red \uE200cite\uE202turn0search0\uE201blue", "red blue"], + ["red \uE200cite\uE202turn0search0\uE201**blue**", "red **blue**"], + ["red \uE200cite\uE202turn0search0\uE201[blue](https://a.dev)", "red [blue](https://a.dev)"], + ["red \uE200cite\uE202turn0search0\uE201`blue`", "red `blue`"], + ["**Result \uE200cite\uE202turn0search0\uE201**.", "**Result**."], + [ + ':codex-file-citation{path="src/foo.ts" note="\uE200cite\uE202turn0search0\uE201"} trailing', + "[foo.ts]() trailing", + ], + ["[s](https://a.dev\uE200cite\uE202turn0search0\uE201)", "[s](https://a.dev)"], + ["[s]\n\n[s]: https://a.dev\uE200cite\uE202turn0search0\uE201", "[s]\n\n[s]: https://a.dev"], + [':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"], + ["`citeturn0search0`", "`citeturn0search0`"], + [':codex-followup[Show `citeturn0search0`]{prompt="Show it"}', "Show `citeturn0search0`"], + ])("leaves the markers out of the rendered Markdown, except in code", (input, expected) => { + expect(render(input)).toBe(expected); + }); +}); + describe("native Markdown adapters", () => { it("renders follow-up suggestions as their label Markdown", () => { expect( diff --git a/packages/shared/src/codexMarkdownDirectives.ts b/packages/shared/src/codexMarkdownDirectives.ts index 6ff396eddb12a..06f3130cd222a 100644 --- a/packages/shared/src/codexMarkdownDirectives.ts +++ b/packages/shared/src/codexMarkdownDirectives.ts @@ -22,6 +22,11 @@ const UNDERSCORE = 95; const CODEX_FILE_CITATION_NAME = "codex-file-citation"; const CODEX_FOLLOW_UP_NAME = "codex-followup"; const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; +const WEB_CITATION_OPEN = "\uE200"; +// OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids. +const WEB_CITATION_MARKER = /([ \t]?)\uE200cite\uE202[^\uE200\uE201]*\uE201/gu; +// Past any closing emphasis or code delimiters, nothing the space separates follows. +const DROPS_PRECEDING_SPACE = /[*_~`]*(?:\s|[.,;:!?)\]]|$)/uy; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", @@ -247,7 +252,54 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { delete node.url; } +function webCitationReplacement(space: string, text: string, markerEnd: number): string { + DROPS_PRECEDING_SPACE.lastIndex = markerEnd; + return DROPS_PRECEDING_SPACE.test(text) ? "" : space; +} + +// A marker that ends `text` is judged by the source past `textEnd`. +function removeWebCitationMarkers(text: string, source = "", textEnd = 0): string { + return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => { + const markerEnd = offset + marker.length; + return markerEnd < text.length + ? webCitationReplacement(space, text, markerEnd) + : webCitationReplacement(space, source, textEnd); + }); +} + +function collectWebCitationMarkers( + source: string, + start: number, + end: number, + matches: DirectiveMatch[], +): void { + for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) { + const markerStart = start + marker.index; + const markerEnd = markerStart + marker[0].length; + matches.push({ + start: markerStart, + end: markerEnd, + markdown: webCitationReplacement(marker[1] ?? "", source, markerEnd), + }); + } +} + function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { + if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) { + node.value = removeWebCitationMarkers( + node.value, + source, + node.position?.end.offset ?? source.length, + ); + return; + } + // GFM autolinks take a marker that directly follows a bare URL into the href. + if ( + (node.type === "link" || node.type === "definition") && + node.url?.includes(WEB_CITATION_OPEN) + ) { + node.url = removeWebCitationMarkers(node.url); + } if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) { renderFileCitation(node, source, insideLink); return; @@ -293,13 +345,20 @@ interface DirectiveMatch { readonly end: number; readonly markdown?: string; readonly template?: CodexArtifactTemplate; - readonly labelRange?: readonly [start: number, end: number]; } -function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void { +function collectDirectiveMatches( + node: MarkdownAstNode, + source: string, + matches: DirectiveMatch[], +): void { const start = node.position?.start.offset; const end = node.position?.end.offset; if (start !== undefined && end !== undefined) { + if (node.type === "text") { + collectWebCitationMarkers(source, start, end, matches); + return; + } if (node.data?.codexFileCitationMarkdown !== undefined) { matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown }); return; @@ -308,14 +367,20 @@ function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[ matches.push({ start, end, template: node.data.codexArtifactTemplate }); return; } + if (node.type === "link" || node.type === "definition") { + const destinationStart = node.children?.at(-1)?.position?.end.offset ?? start; + collectWebCitationMarkers(source, destinationStart, end, matches); + } const labelStart = node.children?.[0]?.position?.start.offset; const labelEnd = node.children?.at(-1)?.position?.end.offset; if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) { - matches.push({ start, end, labelRange: [labelStart, labelEnd] }); - return; + matches.push( + { start, end: labelStart, markdown: "" }, + { start: labelEnd, end, markdown: "" }, + ); } } - for (const child of node.children ?? []) collectDirectiveMatches(child, matches); + for (const child of node.children ?? []) collectDirectiveMatches(child, source, matches); } function renderDirectiveMatches( @@ -323,12 +388,10 @@ function renderDirectiveMatches( replacementFor: (match: DirectiveMatch) => string | undefined, ): string { const matches: DirectiveMatch[] = []; - collectDirectiveMatches(parseCodexMarkdown(markdown), matches); + collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); let rendered = markdown; for (const match of matches.sort((left, right) => right.start - left.start)) { - const replacement = match.labelRange - ? markdown.slice(...match.labelRange) - : replacementFor(match); + const replacement = replacementFor(match); if (replacement !== undefined) { rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end); } @@ -347,11 +410,19 @@ export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string return renderDirectiveMatches(markdown, (match) => match.markdown); } +// Kept apart from the inline renderer: its href-only callers run on every streamed delta. +export function renderCodexMarkdownForNativeDisplay(markdown: string): string { + return markdown.includes(WEB_CITATION_OPEN) + ? renderDirectiveMatches(markdown, (match) => match.markdown) + : renderCodexInlineDirectivesAsMarkdown(markdown); +} + export function renderCodexDirectivesForCopy(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) && - !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) + !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) && + !markdown.includes(WEB_CITATION_OPEN) ) { return markdown; } @@ -371,7 +442,7 @@ export function splitCodexArtifactTemplateMarkdown( } const matches: DirectiveMatch[] = []; - collectDirectiveMatches(parseCodexMarkdown(markdown), matches); + collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches); const templates = matches .filter( (match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } =>