Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions apps/mobile/src/features/threads/ThreadFeed.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ import {
} from "@t3tools/client-runtime/markdown-images";
import { resolveViewedImageAsset } from "@t3tools/client-runtime/work-log/presentation";
import {
renderCodexInlineDirectivesAsMarkdown,
renderCodexDirectivesForCopy,
renderCodexMarkdownForNativeDisplay,
splitCodexArtifactTemplateMarkdown,
} from "@t3tools/shared/codexMarkdownDirectives";
import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList";
Expand Down Expand Up @@ -1208,7 +1209,7 @@ function AssistantMarkdownBlocks(props: {
return (
<SelectableMarkdownText
key={`markdown:${offset}`}
markdown={renderCodexInlineDirectivesAsMarkdown(segment.markdown)}
markdown={renderCodexMarkdownForNativeDisplay(segment.markdown)}
skills={props.skills}
models={props.models}
textStyle={props.markdownStyles.nativeTextStyle}
Expand Down Expand Up @@ -2246,7 +2247,7 @@ function renderFeedEntry(
<View className="mt-1 flex-row items-center gap-1">
<CopyTextButton
accessibilityLabel="Copy message"
text={renderedText}
text={renderCodexDirectivesForCopy(renderedText)}
tintColor={iconSubtleColor}
buttonSize={props.replyActionSize.button}
iconSize={props.replyActionSize.icon}
Expand Down
53 changes: 53 additions & 0 deletions packages/shared/src/codexMarkdownDirectives.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { describe, expect, it } from "vite-plus/test";
import remarkGfm from "remark-gfm";
import remarkParse from "remark-parse";
import { unified } from "unified";

import {
remarkCodexDirectives,
renderCodexDirectivesForCopy,
renderCodexInlineDirectivesAsMarkdown,
renderCodexMarkdownForNativeDisplay,
splitCodexArtifactTemplateMarkdown,
} from "./codexMarkdownDirectives.ts";
import { parseMarkdownFileLink } from "./markdownLinks.ts";
Expand All @@ -30,12 +32,20 @@ const FOLLOW_UP =
':codex-followup[Prepare print version]{prompt="Prepare the document for printing."}';
const ARTIFACT_TEMPLATE =
'::artifact-template{skill_name="artifact-template-hello-world" skill_directory="/Users/test/.codex/skills/artifact-template-hello-world" display_name="Hello World" artifact_kind="document"}';
// OpenAI web-search citation markers: U+E200 opens, U+E202 separates fields, U+E201 closes.
const WEB_CITATION_MARKDOWN =
"Prices rose 3%. \uE200cite\uE202turn816200view0\uE201 Rates held. \uE200cite\uE202turn0search0\uE202turn0search3\uE201";
const WEB_CITATION_MARKER = /[\uE200-\uE202]|turn\d+(?:view|search)\d+/u;

function parse(markdown: string): TestNode {
const processor = unified().use(remarkParse).use(remarkCodexDirectives);
return processor.runSync(processor.parse(markdown), { value: markdown }) as TestNode;
}

function textContent(node: TestNode): string {
return node.value ?? (node.children ?? []).map(textContent).join("");
}

function parseOrdinaryMarkdown(markdown: string): TestNode {
return unified().use(remarkParse).parse(markdown) as TestNode;
}
Expand Down Expand Up @@ -72,6 +82,24 @@ describe("remarkCodexDirectives", () => {
});
});

it("does not display web citation markers", () => {
const text = textContent(parse(WEB_CITATION_MARKDOWN));

expect(text).toContain("Prices rose 3%.");
expect(text).toContain("Rates held.");
expect(text).not.toMatch(WEB_CITATION_MARKER);
expect(textContent(parse("red \uE200cite\uE202turn0search0\uE201**blue**"))).toBe("red blue");
});

it("keeps web citation markers out of a bare URL they follow", () => {
const markdown = "See https://nodejs.orgciteturn0search0 for details.";
const processor = unified().use(remarkParse).use(remarkGfm).use(remarkCodexDirectives);
const tree = processor.runSync(processor.parse(markdown), { value: markdown }) as TestNode;
const link = tree.children?.[0]?.children?.find((node) => node.type === "link");

expect(link?.url).toBe("https://nodejs.org");
});

it.each([
"Meeting at 10:30",
":::note\ncontent\n:::",
Expand Down Expand Up @@ -108,6 +136,31 @@ describe.each([
});
});

describe.each([
{ name: "renderCodexDirectivesForCopy", render: renderCodexDirectivesForCopy },
{ name: "renderCodexMarkdownForNativeDisplay", render: renderCodexMarkdownForNativeDisplay },
])("$name web citation markers", ({ render }) => {
it.each([
[WEB_CITATION_MARKDOWN, "Prices rose 3%. Rates held."],
["red \uE200cite\uE202turn0search0\uE201blue", "red blue"],
["red \uE200cite\uE202turn0search0\uE201**blue**", "red **blue**"],
["red \uE200cite\uE202turn0search0\uE201[blue](https://a.dev)", "red [blue](https://a.dev)"],
["red \uE200cite\uE202turn0search0\uE201`blue`", "red `blue`"],
["**Result \uE200cite\uE202turn0search0\uE201**.", "**Result**."],
[
':codex-file-citation{path="src/foo.ts" note="\uE200cite\uE202turn0search0\uE201"} trailing',
"[foo.ts](<src/foo.ts>) trailing",
],
["[s](https://a.dev\uE200cite\uE202turn0search0\uE201)", "[s](https://a.dev)"],
["[s]\n\n[s]: https://a.dev\uE200cite\uE202turn0search0\uE201", "[s]\n\n[s]: https://a.dev"],
[':codex-followup[Read more citeturn0search0]{prompt="Read more"}', "Read more"],
["`citeturn0search0`", "`citeturn0search0`"],
[':codex-followup[Show `citeturn0search0`]{prompt="Show it"}', "Show `citeturn0search0`"],
])("leaves the markers out of the rendered Markdown, except in code", (input, expected) => {
expect(render(input)).toBe(expected);
});
});

describe("native Markdown adapters", () => {
it("renders follow-up suggestions as their label Markdown", () => {
expect(
Expand Down
93 changes: 82 additions & 11 deletions packages/shared/src/codexMarkdownDirectives.ts
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ const UNDERSCORE = 95;
const CODEX_FILE_CITATION_NAME = "codex-file-citation";
const CODEX_FOLLOW_UP_NAME = "codex-followup";
const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template";
const WEB_CITATION_OPEN = "\uE200";
// OpenAI web-search citations: U+E200 "cite" (U+E202 id)+ U+E201. T3 has no sources for the ids.
const WEB_CITATION_MARKER = /([ \t]?)\uE200cite\uE202[^\uE200\uE201]*\uE201/gu;
// Past any closing emphasis or code delimiters, nothing the space separates follows.
const DROPS_PRECEDING_SPACE = /[*_~`]*(?:\s|[.,;:!?)\]]|$)/uy;

export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [
"dataCodexArtifactTemplate",
Expand Down Expand Up @@ -247,7 +252,54 @@ function renderArtifactTemplate(node: MarkdownAstNode, source: string): void {
delete node.url;
}

function webCitationReplacement(space: string, text: string, markerEnd: number): string {
DROPS_PRECEDING_SPACE.lastIndex = markerEnd;
return DROPS_PRECEDING_SPACE.test(text) ? "" : space;
}

// A marker that ends `text` is judged by the source past `textEnd`.
function removeWebCitationMarkers(text: string, source = "", textEnd = 0): string {
return text.replace(WEB_CITATION_MARKER, (marker, space: string, offset: number) => {
const markerEnd = offset + marker.length;
return markerEnd < text.length
? webCitationReplacement(space, text, markerEnd)
: webCitationReplacement(space, source, textEnd);
});
}

function collectWebCitationMarkers(
source: string,
start: number,
end: number,
matches: DirectiveMatch[],
): void {
for (const marker of source.slice(start, end).matchAll(WEB_CITATION_MARKER)) {
const markerStart = start + marker.index;
const markerEnd = markerStart + marker[0].length;
matches.push({
start: markerStart,
end: markerEnd,
markdown: webCitationReplacement(marker[1] ?? "", source, markerEnd),
});
}
}

function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void {
if (node.type === "text" && node.value?.includes(WEB_CITATION_OPEN)) {
node.value = removeWebCitationMarkers(
node.value,
source,
node.position?.end.offset ?? source.length,
);
return;
}
// GFM autolinks take a marker that directly follows a bare URL into the href.
if (
(node.type === "link" || node.type === "definition") &&
node.url?.includes(WEB_CITATION_OPEN)
) {
node.url = removeWebCitationMarkers(node.url);
}
if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) {
renderFileCitation(node, source, insideLink);
return;
Expand Down Expand Up @@ -293,13 +345,20 @@ interface DirectiveMatch {
readonly end: number;
readonly markdown?: string;
readonly template?: CodexArtifactTemplate;
readonly labelRange?: readonly [start: number, end: number];
}

function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void {
function collectDirectiveMatches(
node: MarkdownAstNode,
source: string,
matches: DirectiveMatch[],
): void {
const start = node.position?.start.offset;
const end = node.position?.end.offset;
if (start !== undefined && end !== undefined) {
if (node.type === "text") {
collectWebCitationMarkers(source, start, end, matches);
return;
}
if (node.data?.codexFileCitationMarkdown !== undefined) {
matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown });
return;
Expand All @@ -308,27 +367,31 @@ function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[
matches.push({ start, end, template: node.data.codexArtifactTemplate });
return;
}
if (node.type === "link" || node.type === "definition") {
const destinationStart = node.children?.at(-1)?.position?.end.offset ?? start;
collectWebCitationMarkers(source, destinationStart, end, matches);
}
const labelStart = node.children?.[0]?.position?.start.offset;
const labelEnd = node.children?.at(-1)?.position?.end.offset;
if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) {
matches.push({ start, end, labelRange: [labelStart, labelEnd] });
return;
matches.push(
{ start, end: labelStart, markdown: "" },
{ start: labelEnd, end, markdown: "" },
);
}
}
for (const child of node.children ?? []) collectDirectiveMatches(child, matches);
for (const child of node.children ?? []) collectDirectiveMatches(child, source, matches);
}

function renderDirectiveMatches(
markdown: string,
replacementFor: (match: DirectiveMatch) => string | undefined,
): string {
const matches: DirectiveMatch[] = [];
collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches);
let rendered = markdown;
for (const match of matches.sort((left, right) => right.start - left.start)) {
const replacement = match.labelRange
? markdown.slice(...match.labelRange)
: replacementFor(match);
const replacement = replacementFor(match);
if (replacement !== undefined) {
rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end);
}
Expand All @@ -347,11 +410,19 @@ export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string
return renderDirectiveMatches(markdown, (match) => match.markdown);
}

// Kept apart from the inline renderer: its href-only callers run on every streamed delta.
export function renderCodexMarkdownForNativeDisplay(markdown: string): string {
return markdown.includes(WEB_CITATION_OPEN)
? renderDirectiveMatches(markdown, (match) => match.markdown)
: renderCodexInlineDirectivesAsMarkdown(markdown);
}

export function renderCodexDirectivesForCopy(markdown: string): string {
if (
!markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) &&
!markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) &&
!markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)
!markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) &&
!markdown.includes(WEB_CITATION_OPEN)
) {
return markdown;
}
Expand All @@ -371,7 +442,7 @@ export function splitCodexArtifactTemplateMarkdown(
}

const matches: DirectiveMatch[] = [];
collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
collectDirectiveMatches(parseCodexMarkdown(markdown), markdown, matches);
const templates = matches
.filter(
(match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } =>
Expand Down
Loading