Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions apps/mobile/src/features/threads/ThreadFeed.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ import {
renderCodexInlineDirectivesAsMarkdown,
splitCodexArtifactTemplateMarkdown,
} from "@t3tools/shared/codexMarkdownDirectives";
import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations";
import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList";
import { imageMimeType } from "@t3tools/shared/image";
import { videoMimeType } from "@t3tools/shared/video";
Expand Down Expand Up @@ -1217,7 +1218,7 @@ function AssistantMarkdownBlocks(props: {
return (
<SelectableMarkdownText
key={`markdown:${offset}`}
markdown={renderCodexInlineDirectivesAsMarkdown(segment.markdown)}
markdown={hideCodexWebCitations(renderCodexInlineDirectivesAsMarkdown(segment.markdown))}
skills={props.skills}
models={props.models}
textStyle={props.markdownStyles.nativeTextStyle}
Expand Down Expand Up @@ -2283,7 +2284,7 @@ function renderFeedEntry(
<View className="mt-1 flex-row items-center gap-1">
<CopyTextButton
accessibilityLabel="Copy message"
text={renderedText}
text={hideCodexWebCitations(renderedText)}
tintColor={iconSubtleColor}
buttonSize={props.replyActionSize.button}
iconSize={props.replyActionSize.icon}
Expand Down
16 changes: 14 additions & 2 deletions apps/web/src/components/ChatMarkdown.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ import {
artifactTemplateFromHastProperties,
renderCodexInlineDirectivesAsMarkdown,
} from "@t3tools/shared/codexMarkdownDirectives";
import {
CODEX_WEB_CITATION_START,
remarkHideCodexWebCitations,
} from "@t3tools/shared/codexWebCitations";
import { renderSkillInlineMarkdownChildren } from "./chat/SkillInlineText";
import { MarkdownMath } from "./chat/MarkdownMath";
import {
Expand Down Expand Up @@ -230,6 +234,8 @@ const isAssetNotFoundError = Schema.is(AssetWorkspaceAssetNotFoundError);

interface ChatMarkdownProps {
renderInsights?: boolean;
/** For assistant messages only: a user's prompt shows what they typed. */
hideWebCitations?: boolean;
/** Compact markdown transforms remove blocks before rendering inside a label. */
inline?: boolean;
text: string;
Expand Down Expand Up @@ -1746,6 +1752,7 @@ const combinedPlugins = new WeakMap<PluggableList, WeakMap<PluggableList, Plugga

// One combined list per pair, so a stable extra list shares a renderer and its parse cache.
const INSIGHT_REMARK_PLUGINS: PluggableList = [remarkAssistantInsights];
const WEB_CITATION_REMARK_PLUGINS: PluggableList = [remarkHideCodexWebCitations];

function withExtraPlugins(shared: PluggableList, extra: PluggableList): PluggableList {
if (extra.length === 0) return shared;
Expand Down Expand Up @@ -1814,6 +1821,7 @@ function ChatMarkdown({
parseRawHtml = true,
extraRemarkPlugins = EMPTY_REMARK_PLUGINS,
renderInsights = false,
hideWebCitations = false,
...props
}: ChatMarkdownProps) {
const { componentState, handleCopy, markdownRef, localMediaPreview, closeLocalMediaPreview } =
Expand All @@ -1837,7 +1845,7 @@ function ChatMarkdown({
const bindMarkdownRef = (node: HTMLElement | null) => {
markdownRef.current = node;
};
const sharedRemarkPlugins =
const sharedRemarkPlugins = withExtraPlugins(
CHAT_MARKDOWN_REMARK_PLUGIN_SETS[
componentState.mathEnabled
? lineBreaks
Expand All @@ -1846,7 +1854,11 @@ function ChatMarkdown({
: lineBreaks
? "breaks"
: "plain"
];
],
hideWebCitations && text.includes(CODEX_WEB_CITATION_START)
? WEB_CITATION_REMARK_PLUGINS
: EMPTY_REMARK_PLUGINS,
);
// The incremental parser keeps per-instance state, so its output is not shared.
const uncachedPlugins = incrementalParsing;
const remarkPlugins = incrementalParsing
Expand Down
7 changes: 6 additions & 1 deletion apps/web/src/components/chat/MessagesTimeline.logic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
} from "@t3tools/client-runtime/user-message";
import type { HandoffTimelineRun } from "@t3tools/client-runtime/handoff";
import { renderCodexDirectivesForCopy } from "@t3tools/shared/codexMarkdownDirectives";
import { hideCodexWebCitations } from "@t3tools/shared/codexWebCitations";
import {
commandDisplayText,
commandProgramName,
Expand Down Expand Up @@ -690,7 +691,11 @@ export function resolveAssistantMessageCopyState({
const hasText = text !== null && text.trim().length > 0;
const visible = showCopyButton && hasText && !streaming;
return {
text: hasText ? (visible ? renderCodexDirectivesForCopy(text) : text) : null,
text: hasText
? visible
? hideCodexWebCitations(renderCodexDirectivesForCopy(text))
: text
: null,
visible,
};
}
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/components/chat/TimelineAssistantRows.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ export function AssistantTimelineRow({ row }: { row: Extract<TimelineRow, { kind
<ChatMarkdown
text={messageText}
renderInsights
hideWebCitations
cwd={ctx.markdownCwd}
threadRef={ctx.threadRef ?? undefined}
isStreaming={Boolean(row.message.streaming)}
Expand Down
4 changes: 4 additions & 0 deletions packages/shared/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,10 @@
"types": "./src/codexFileCitations.ts",
"import": "./src/codexFileCitations.ts"
},
"./codexWebCitations": {
"types": "./src/codexWebCitations.ts",
"import": "./src/codexWebCitations.ts"
},
"./codexMarkdownDirectives": {
"types": "./src/codexMarkdownDirectives.ts",
"import": "./src/codexMarkdownDirectives.ts"
Expand Down
135 changes: 135 additions & 0 deletions packages/shared/src/codexWebCitations.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import type { Root } from "mdast";
import remarkGfm from "remark-gfm";
import remarkParse from "remark-parse";
import { unified } from "unified";
import { describe, expect, it } from "vite-plus/test";

import { hideCodexWebCitations, remarkHideCodexWebCitations } from "./codexWebCitations.ts";
import { CHAT_MARKDOWN_REMARK_PLUGINS } from "./markdownPipeline.ts";

const CITE = "citeturn0search0";
const CITE_TWO = "citeturn0search0turn1news2";

interface TestNode {
readonly type: string;
readonly value?: string;
readonly url?: string;
readonly children?: ReadonlyArray<TestNode>;
}

// Positions and text node boundaries differ between a displayed tree and a reparsed copy, and a
// copy can carry an invisible word joiner; structure and visible text must not differ.
function shape(node: TestNode): TestNode {
const children: Array<TestNode> = [];
for (const child of (node.children ?? []).map(shape)) {
const previous = children.at(-1);
if (child.type === "text" && child.value === "") continue;
if (child.type === "text" && previous?.type === "text") {
children[children.length - 1] = { type: "text", value: `${previous.value}${child.value}` };
} else {
children.push(child);
}
}
return {
type: node.type,
...(node.value === undefined ? {} : { value: node.value.replaceAll("\u2060", "") }),
...(node.url === undefined ? {} : { url: node.url }),
...(node.children === undefined ? {} : { children }),
};
}

function parsed(markdown: string): TestNode {
return shape(unified().use(remarkParse).use(remarkGfm).parse(markdown));
}

function displayed(markdown: string): TestNode {
const processor = unified().use(remarkParse).use(remarkGfm).use(remarkHideCodexWebCitations);
return shape(processor.runSync(processor.parse(markdown)) as Root);
}

describe("Codex web-search citation markers", () => {
it.each([
["at the end of a sentence", `Node 24 is current. ${CITE}`, "Node 24 is current."],
["with several sources", `Node 24 is current. ${CITE_TWO}`, "Node 24 is current."],
["between sentences", `One. ${CITE} Two. ${CITE}${CITE_TWO} Three.`, "One. Two. Three."],
["before punctuation", `Node 24 ${CITE}, then 25 ${CITE}.`, "Node 24, then 25."],
["closing bold before a word", `**Result ${CITE}**next`, "**Result**next"],
[
"closing bold before a link",
`**Result ${CITE}**[next](https://a.dev)`,
"**Result**[next](https://a.dev)",
],
["closing bold after punctuation", `**Result. ${CITE}** next`, "**Result.** next"],
[
"closing bold between punctuation and a word",
`**Result. ${CITE}**next`,
"**Result.\u2060**next",
],
[
"closing bold between a link and a word",
`**[docs](https://a.dev) ${CITE}**next`,
"**[docs](https://a.dev)\u2060**next",
],
[
"opening bold between a word and punctuation",
`word**${CITE} (Result)** next`,
"word**\u2060(Result)** next",
],
["opening bold", `**${CITE} Result** next`, "**Result** next"],
["after inline code inside bold", `**Use \`npm\` ${CITE}** now`, "**Use `npm`** now"],
["as all of the bold text", `Done **${CITE}** now`, "Done now"],
["closing strikethrough", `~~old ${CITE}~~ new`, "~~old~~ new"],
["before bold", `See ${CITE}**this**`, "See **this**"],
["after a link", `[docs](https://a.dev)${CITE} say so`, "[docs](https://a.dev) say so"],
[
"after a bare URL",
`See https://nodejs.org${CITE} for more`,
"See https://nodejs.org for more",
],
[
"in a heading and a list",
`# News ${CITE}\n\n- item ${CITE}\n- other`,
"# News\n\n- item\n- other",
],
[
"in a table",
`| a | b |\n| - | - |\n| one ${CITE} | two |`,
"| a | b |\n| - | - |\n| one | two |",
],
["between code spans", `\`one\`${CITE}\`two\``, "`one`\u2060`two`"],
[
"in a shortcut reference link",
`[docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`,
`[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`,
],
[
"in a collapsed reference link",
`[docs ${CITE}][]\n\n[docs ${CITE}]: https://a.dev`,
`[docs][docs ${CITE}]\n\n[docs ${CITE}]: https://a.dev`,
],
["in a quote", `> first\n> ${CITE} second ${CITE}`, "> first\n> second"],
])("are hidden %s, in display and in copied text alike", (_name, markdown, expected) => {
expect(hideCodexWebCitations(markdown)).toBe(expected);
expect(displayed(markdown)).toEqual(parsed(expected));
});

it.each([
["inline code", `Literally \`${CITE}\` here`],
["a code fence", `\`\`\`\n${CITE}\n\`\`\``],
["an unfinished marker", "Still streaming citeturn0search0"],
["a link destination", `[docs](https://a.dev/${CITE})`],
["another private-use marker", "See fileciteturn0file0 here"],
])("are kept in %s", (_name, markdown) => {
expect(hideCodexWebCitations(markdown)).toBe(markdown);
expect(displayed(markdown)).toEqual(parsed(markdown));
});

it("stay in Markdown that does not opt in, such as a user's prompt", () => {
const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS);
const tree = processor.runSync(processor.parse(`Explain this marker: ${CITE}`)) as Root;

expect(shape(tree).children?.[0]?.children).toEqual([
{ type: "text", value: `Explain this marker: ${CITE}` },
]);
});
});
Loading
Loading