diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 67638fc4e63ca..0754a1e62b67e 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -649,7 +649,7 @@ export default function ChatView(props: ChatViewProps) { const branchToolbarRef = useRef(null); const citeAssistantText = ( citation: AssistantCitation, - sourceAnchor: AssistantCitationSourceAnchor, + sourceAnchor: AssistantCitationSourceAnchor | undefined, ) => { const inserted = composerRef.current?.citeAssistantText(citation, sourceAnchor) ?? false; if (!inserted) { diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index d89f57068e9cf..87d311dee0d45 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -208,7 +208,7 @@ export type ComposerCitationCommentRequest = { previousValue: string; value: string; citationStart: number; - sourceAnchor: AssistantCitationSourceAnchor; + sourceAnchor: AssistantCitationSourceAnchor | undefined; insertedSpaces: CitationInsertedSpaces; }; @@ -216,7 +216,7 @@ type CitationInsertedSpaces = { before: boolean; after: boolean }; type OpenCitationComment = { key: string; - sourceAnchor?: AssistantCitationSourceAnchor; + sourceAnchor?: AssistantCitationSourceAnchor | undefined; removeOnCancel?: CitationInsertedSpaces; }; diff --git a/apps/web/src/components/chat/AssistantCitationSource.tsx b/apps/web/src/components/chat/AssistantCitationSource.tsx index cf2e950e96abd..322b52d59ca22 100644 --- a/apps/web/src/components/chat/AssistantCitationSource.tsx +++ b/apps/web/src/components/chat/AssistantCitationSource.tsx @@ -104,11 +104,13 @@ function observeAssistantCitationSource({ itemKey, request, list, + opaque, }: { root: HTMLElement; itemKey: string; request: AssistantCitationTarget; list: LegendListRef; + opaque: boolean; }) { const activation = request.activationRef.current; if (activation.dismissed) return; @@ -163,7 +165,7 @@ function observeAssistantCitationSource({ const state = list.getState(); const index = state.indexByKey(itemKey); if (index === undefined || !(state.sizeAtIndex(index) > 0)) return; - const range = resolveAssistantCitationRange(root, request.citation); + const range = opaque ? null : resolveAssistantCitationRange(root, request.citation); const rect = (range ?? root).getBoundingClientRect(); if (rect.height <= 0 || scrollNode.clientHeight <= 0) { clear(); @@ -204,7 +206,7 @@ function observeAssistantCitationSource({ if (rect.bottom <= scrollRect.top || rect.top >= scrollRect.bottom) return; activation.scrolled = true; request.onComplete(); - if (!range) { + if (!range && !opaque) { toastManager.add({ type: "warning", title: "The quoted text has changed", @@ -338,6 +340,7 @@ export function AssistantCitationSource({ itemKey, request, listRef, + opaque = false, children, }: { messageId: MessageId; @@ -345,6 +348,8 @@ export function AssistantCitationSource({ itemKey: string; request: AssistantCitationTarget | null; listRef: RefObject; + /** The source's text sits in a sandboxed frame, so navigation reveals the row without a range. */ + opaque?: boolean; children: ReactNode; }) { const rootRef = useRef(null); @@ -353,8 +358,8 @@ export function AssistantCitationSource({ const root = rootRef.current; const list = listRef.current; if (!root || !list || !request || request.citation.messageId !== messageId) return; - return observeAssistantCitationSource({ root, itemKey, request, list }); - }, [itemKey, listRef, messageId, request]); + return observeAssistantCitationSource({ root, itemKey, request, list, opaque }); + }, [itemKey, listRef, messageId, opaque, request]); return (
boolean; + onCite: ( + citation: AssistantCitation, + sourceAnchor: AssistantCitationSourceAnchor | undefined, + ) => boolean; }) { const [selection, setSelection] = useState<{ citation: AssistantCitation; position: SelectionActionPoint; - sourceAnchor: AssistantCitationSourceAnchor; + sourceAnchor: AssistantCitationSourceAnchor | undefined; } | null>(null); const toolbarRef = useRef(null); + const frameSelectionRef = useRef<{ + frame: HTMLIFrameElement; + selection: HtmlRenderSelection; + } | null>(null); const actionsRef = useRef | null>(null); const backgroundThread = use(BackgroundThreadContext); @@ -51,7 +61,53 @@ export function AssistantSelectionToolbar({ useEffect(() => { if (!viewport) return; const clear = () => setSelection(null); + // A sandboxed frame reports its own selection. The focused document owns the selection, + // so that report yields to the host's once focus leaves the frame. + const updateFromFrame = () => { + const reported = frameSelectionRef.current; + if (!reported) return false; + const { frame, selection: frameSelection } = reported; + const source = frame.closest("[data-assistant-citation-source]"); + const messageId = source?.dataset.assistantCitationSource; + if (document.activeElement !== frame || !source || !viewport.contains(source) || !messageId) { + frameSelectionRef.current = null; + return false; + } + const frameRect = frame.getBoundingClientRect(); + const inHost = (box: HtmlRenderBox) => ({ + left: frameRect.left + box.left, + top: frameRect.top + box.top, + right: frameRect.left + box.right, + bottom: frameRect.top + box.bottom, + }); + const selectionRect = inHost(frameSelection.rect); + const viewportRect = viewport.getBoundingClientRect(); + const selector = createAssistantTextSelector( + frameSelection.text, + 0, + frameSelection.text.length, + ); + if ( + !selector || + !isSelectionRectVisible(inHost(frameSelection.bounds), [frameRect, viewportRect]) + ) { + clear(); + return true; + } + setSelection({ + sourceAnchor: undefined, + citation: { version: 1, ...threadRef, messageId: MessageId.make(messageId), ...selector }, + position: resolveSelectionActionPosition({ + bounds: viewportRect, + selectionRect, + pointer: null, + viewport: { width: window.innerWidth, height: window.innerHeight }, + }), + }); + return true; + }; const update = (pointer: SelectionActionPoint | null) => { + if (updateFromFrame()) return; const nativeSelection = window.getSelection(); const captured = captureAssistantTextSelection(viewport, nativeSelection); const messageId = captured?.source.dataset.assistantCitationSource; @@ -145,6 +201,19 @@ export function AssistantSelectionToolbar({ onDismiss: clear, }); actionsRef.current = actions; + const onFrameSelection = (event: Event) => { + const frame = event.target; + if (!(frame instanceof HTMLIFrameElement)) return; + const reported = (event as CustomEvent).detail; + if (reported) { + frameSelectionRef.current = { frame, selection: reported }; + actions.selectionReported(); + } else if (frameSelectionRef.current?.frame === frame) { + frameSelectionRef.current = null; + clear(); + } + }; + viewport.addEventListener(HTML_RENDER_SELECTION_EVENT, onFrameSelection); const focusActions = (event: KeyboardEvent) => { const toolbar = toolbarRef.current; if ( @@ -169,10 +238,12 @@ export function AssistantSelectionToolbar({ if (!backgroundThread) document.addEventListener("keydown", focusActions, true); document.addEventListener("selectionchange", actions.selectionChanged); return () => { + viewport.removeEventListener(HTML_RENDER_SELECTION_EVENT, onFrameSelection); document.removeEventListener("keydown", focusActions, true); document.removeEventListener("selectionchange", actions.selectionChanged); actions.dispose(); actionsRef.current = null; + frameSelectionRef.current = null; }; }, [backgroundThread, threadRef, viewport]); @@ -180,6 +251,7 @@ export function AssistantSelectionToolbar({ const tooLong = selection.citation.text.length > ASSISTANT_CITATION_MAX_TEXT_LENGTH; const dismiss = () => { actionsRef.current?.cancel(); + frameSelectionRef.current = null; setSelection(null); }; const cite = () => { diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index ff14154dccb8f..3909e5aaf346d 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -304,7 +304,7 @@ export interface ChatComposerHandle { pasteTextAtEnd: (text: string, options?: { bypassAutoAttachment?: boolean }) => boolean; citeAssistantText: ( citation: AssistantCitation, - sourceAnchor: AssistantCitationSourceAnchor, + sourceAnchor: AssistantCitationSourceAnchor | undefined, ) => boolean; replyToResponse: (response: ResponseContextRecord) => boolean; openModelPicker: () => void; @@ -1718,7 +1718,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) focusEditorAfterReplace?: boolean; citationComment?: { start: number; - sourceAnchor: AssistantCitationSourceAnchor; + sourceAnchor: AssistantCitationSourceAnchor | undefined; insertedSpaces: ComposerCitationCommentRequest["insertedSpaces"]; }; }, @@ -2553,7 +2553,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) position: "cursor" | "end", options?: { ensureLeadingBoundary?: boolean; - citationCommentAnchor?: AssistantCitationSourceAnchor; + citationComment?: { sourceAnchor: AssistantCitationSourceAnchor | undefined }; clipboardData?: DataTransfer; }, ): boolean => { @@ -2562,7 +2562,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) activePendingIsResponding || (pendingUserInputs.length > 0 && !activePendingProgress?.activeQuestion) || projectSelectionRequired || - (options?.citationCommentAnchor && !composerEditorRef.current) + (options?.citationComment && !composerEditorRef.current) ) { return false; } @@ -2580,11 +2580,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) cursor, rangeEnd, needsLeadingSpace ? ` ${text}` : text, - options?.citationCommentAnchor + options?.citationComment ? { citationComment: { start: cursor + (needsLeadingSpace ? 1 : 0), - sourceAnchor: options.citationCommentAnchor, + sourceAnchor: options.citationComment.sourceAnchor, insertedSpaces: { before: needsLeadingSpace, after: rangeEnd === cursor }, }, focusEditorAfterReplace: false, @@ -3037,7 +3037,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) insertComposerTextStable( formatAssistantCitationForComposer(citation, citation.comment), "cursor", - { ensureLeadingBoundary: true, citationCommentAnchor: sourceAnchor }, + { ensureLeadingBoundary: true, citationComment: { sourceAnchor } }, ), replyToResponse: (response) => { // An open question owns the composer, and its answer would not carry the reply. diff --git a/apps/web/src/components/chat/HtmlRenderFrame.tsx b/apps/web/src/components/chat/HtmlRenderFrame.tsx index 767f4bc3f8f4f..5e577ccfd5b1c 100644 --- a/apps/web/src/components/chat/HtmlRenderFrame.tsx +++ b/apps/web/src/components/chat/HtmlRenderFrame.tsx @@ -117,7 +117,7 @@ export function HtmlRenderFrame(props: { ) : null} ) : failed ? ( -

+

Unable to load {title}

) : null} diff --git a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx index 448e6f90a8ed0..6da8a465a14c6 100644 --- a/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.browser.test.tsx @@ -2,6 +2,7 @@ import { injectHtmlRenderBootstrap } from "@t3tools/shared/htmlRender"; import { HtmlRenderDocument } from "../files/BrowserDocumentFrame"; import { flushSync } from "react-dom"; import { AudioPreview } from "../files/AudioPreview"; +import { HTML_RENDER_SELECTION_EVENT } from "../files/BrowserDocumentFrame"; import { cdp, page, userEvent } from "vite-plus/test/browser"; import { ComposerContextId, @@ -93,6 +94,14 @@ vi.mock(import("~/assets/assetUrls"), async (original) => { }; }); +vi.mock(import("~/assets/longLivedAssetUrls"), () => ({ + useLongLivedAssetUrl: () => ({ + _tag: "Success", + url: "about:blank", + expiresAt: Date.now() + 60 * 60_000, + }), +})); + const ENVIRONMENT_ID = EnvironmentId.make("environment-local"); const CREATED_AT = "2026-03-17T19:12:28.000Z"; @@ -1314,6 +1323,57 @@ describe("MessagesTimeline citations", () => { await expect.element(screen.getByText("the cited answer")).toBeVisible(); expect(toast).not.toHaveBeenCalled(); }); + + it("cites the selection an HTML render reports from its frame", async () => { + const onCiteAssistantText = vi.fn(() => true); + const listRef = createRef(); + const htmlRender: Entry = { + id: "render", + kind: "html-render", + createdAt: CREATED_AT, + runId: null, + htmlRender: { attachmentId: "page.html", title: "Report", height: 2000 }, + }; + await using screen = await renderTimeline( + [message("Show a report", {}, "question"), htmlRender], + { onCiteAssistantText, listRef, liveFollowEnabled: false }, + ); + const frame = (await screen.getByTitle("Report").findElement()) as HTMLIFrameElement; + const viewport = frame.closest("[data-assistant-citation-viewport]")!; + await listRef.current?.scrollToOffset({ offset: 0, animated: false }); + await vi.waitFor(() => + expect(frame.getBoundingClientRect().top).toBeGreaterThan( + viewport.getBoundingClientRect().top, + ), + ); + // The selection starts in view, but its last line sits below the chat viewport. + const below = viewport.getBoundingClientRect().bottom - frame.getBoundingClientRect().top + 100; + + frame.focus(); + frame.dispatchEvent( + new CustomEvent(HTML_RENDER_SELECTION_EVENT, { + bubbles: true, + detail: { + text: "Quoted words", + rect: { left: 8, top: below, right: 120, bottom: below + 20 }, + bounds: { left: 8, top: 8, right: 300, bottom: below + 20 }, + }, + }), + ); + await userEvent.click(page.getByRole("button", { name: "Cite selection in composer" })); + + expect(onCiteAssistantText).toHaveBeenCalledWith( + expect.objectContaining({ + messageId: "render", + text: "Quoted words", + start: 0, + end: "Quoted words".length, + prefix: "", + suffix: "", + }), + undefined, + ); + }); }); describe("MessagesTimeline automation prompts", () => { diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index b9653da6b315b..212de0ac6d0ce 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -228,7 +228,7 @@ interface MessagesTimelineProps { citationHistory?: CitedHistory; onCiteAssistantText?: ( citation: AssistantCitation, - sourceAnchor: AssistantCitationSourceAnchor, + sourceAnchor: AssistantCitationSourceAnchor | undefined, ) => boolean; onReplyToResponse?: TimelineRowSharedState["onReplyToResponse"]; isWorking: boolean; diff --git a/apps/web/src/components/chat/TimelineAssistantRows.tsx b/apps/web/src/components/chat/TimelineAssistantRows.tsx index b62f6f8e4b10f..3262dc5d5753a 100644 --- a/apps/web/src/components/chat/TimelineAssistantRows.tsx +++ b/apps/web/src/components/chat/TimelineAssistantRows.tsx @@ -375,13 +375,22 @@ export function HtmlRenderTimelineRow({ } return (
- + + +
); } diff --git a/apps/web/src/components/chat/citationProcedure.test.ts b/apps/web/src/components/chat/citationProcedure.test.ts index 8971363d8a7b9..84e357fb401ae 100644 --- a/apps/web/src/components/chat/citationProcedure.test.ts +++ b/apps/web/src/components/chat/citationProcedure.test.ts @@ -129,6 +129,30 @@ describe("citedThreadSnapshot", () => { title: "Could not load the cited response", }); }); + + it("finds a cited HTML render by its item id, but no other tool", () => { + const tool = (toolName: string) => + citedThreadSnapshot({ + ...unloaded, + thread: { + projection: { + visibleTurnItems: [ + { + item: { + type: "dynamic_tool", + id: "cited", + status: "completed", + toolName, + output: { htmlRender: { attachmentId: "page.html", title: "Page", height: 200 } }, + }, + }, + ], + }, + } as never, + }); + expect(tool("html_render")).toEqual({ message: "assistant", hasMoreHistory: false }); + expect(tool("shell")).toEqual({ message: "other", hasMoreHistory: false }); + }); }); describe("resolveCitationPhase", () => { @@ -198,6 +222,27 @@ describe("resolveCitationPhase", () => { ).toEqual({ _tag: "ready", rowId: messageId }); }); + it("is ready once a cited HTML render has a row", () => { + const htmlRender = { attachmentId: "page.html", title: "Page", height: 200 }; + const renderEntry: TimelineEntry = { + kind: "html-render", + id: messageId, + createdAt: at, + runId, + htmlRender, + }; + const renderRow = { + kind: "html-render", + id: messageId, + createdAt: at, + htmlRender, + } as MessagesTimelineRow; + expect(phase({ entries: [renderEntry], rows: [renderRow] })).toEqual({ + _tag: "ready", + rowId: messageId, + }); + }); + it("fails for a citation that names a user message", () => { const userEntry: TimelineEntry = { ...entry, diff --git a/apps/web/src/components/chat/citationProcedure.ts b/apps/web/src/components/chat/citationProcedure.ts index 2d77a468e67d7..47de58f7bd3a6 100644 --- a/apps/web/src/components/chat/citationProcedure.ts +++ b/apps/web/src/components/chat/citationProcedure.ts @@ -1,6 +1,6 @@ import { useAtomValue } from "@effect/atom-react"; import { type AtomCommandSuccess, runAtomCommand } from "@t3tools/client-runtime/state/runtime"; -import { sentMessageFromToolItem } from "@t3tools/shared/toolOutput"; +import { htmlRenderFromToolItem, sentMessageFromToolItem } from "@t3tools/shared/toolOutput"; import { Atom, type AtomRegistry } from "effect/reactivity"; import type { TimelineEntry } from "../../session-logic"; import { environmentThreadDetails, threadEnvironment } from "../../state/threads"; @@ -95,7 +95,8 @@ export function resolveCitationPhase(input: { const source = input.entries.find( (entry) => (entry.kind === "message" && entry.message.id === input.messageId) || - (entry.kind === "sent-message" && entry.id === input.messageId), + (entry.kind === "sent-message" && entry.id === input.messageId) || + (entry.kind === "html-render" && entry.id === input.messageId), ); // The history store can hold the message a render before the entries do. if (!source) return { _tag: "loading" }; @@ -103,7 +104,8 @@ export function resolveCitationPhase(input: { const row = input.rows.find( (candidate) => (candidate.kind === "message" && candidate.message.id === input.messageId) || - (candidate.kind === "sent-message" && candidate.id === input.messageId), + (candidate.kind === "sent-message" && candidate.id === input.messageId) || + (candidate.kind === "html-render" && candidate.id === input.messageId), ); if (row) return { _tag: "ready", rowId: row.id }; return { _tag: "expanding" }; @@ -130,18 +132,21 @@ export function citedThreadSnapshot(input: { if (input.status === "deleted") return { message: null, hasMoreHistory: false }; return input.error === null ? null : { message: null, hasMoreHistory: false, loadFailed: true }; } - // Older pages merge into the visible turn items, not the message list. - const item = input.thread.projection.visibleTurnItems.find( - ({ item }) => - ((item.type === "assistant_message" || item.type === "user_message") && - item.messageId === input.messageId) || - (item.type === "dynamic_tool" && - item.id === input.messageId && - item.status === "completed" && - sentMessageFromToolItem(item) !== undefined), + // Older pages merge into the visible turn items, not the message list. As in history + // paging, messages match by message id and other items, such as sent messages and + // HTML renders, by item id. + const item = input.thread.projection.visibleTurnItems.find(({ item }) => + item.type === "assistant_message" || item.type === "user_message" + ? item.messageId === input.messageId + : item.id === input.messageId, )?.item; + const cited = + item?.type === "assistant_message" || + (item?.type === "dynamic_tool" && + item.status === "completed" && + (htmlRenderFromToolItem(item) !== undefined || sentMessageFromToolItem(item) !== undefined)); return { - message: item === undefined ? null : item.type === "user_message" ? "other" : "assistant", + message: item === undefined ? null : cited ? "assistant" : "other", hasMoreHistory: input.history.hasMoreHistory, }; } diff --git a/apps/web/src/components/files/BrowserDocumentFrame.browser.test.tsx b/apps/web/src/components/files/BrowserDocumentFrame.browser.test.tsx index cc3640fc2a166..6dd9fea4061a0 100644 --- a/apps/web/src/components/files/BrowserDocumentFrame.browser.test.tsx +++ b/apps/web/src/components/files/BrowserDocumentFrame.browser.test.tsx @@ -1,11 +1,11 @@ -import { injectHtmlRenderBootstrap } from "@t3tools/shared/htmlRender"; +import { injectHtmlRenderBootstrap, type HtmlRenderSelection } from "@t3tools/shared/htmlRender"; import { describe, expect, it, vi } from "vite-plus/test"; -import { page } from "vite-plus/test/browser"; +import { page, userEvent } from "vite-plus/test/browser"; import "~/index.css"; import { render } from "../../../test/render"; -import { HtmlRenderDocument } from "./BrowserDocumentFrame"; +import { HTML_RENDER_SELECTION_EVENT, HtmlRenderDocument } from "./BrowserDocumentFrame"; const sizeChanged = (height: number) => ({ jsonrpc: "2.0", @@ -57,6 +57,104 @@ describe("HtmlRenderDocument", () => { await expect.element(viewer.getByRole("button", { name: "Next media" })).toBeVisible(); await expect.element(viewer.getByRole("button", { name: "Previous media" })).toBeVisible(); }); + + it("reports the page's text selection to the host only while the frame has focus", async () => { + const html = injectHtmlRenderBootstrap(`

Quoted words

+
News
+ `); + const src = URL.createObjectURL(new Blob([html], { type: "text/html" })); + using _url = { [Symbol.dispose]: () => URL.revokeObjectURL(src) }; + const selections: Array = []; + const listeners = new AbortController(); + using _listeners = { [Symbol.dispose]: () => listeners.abort() }; + document.addEventListener( + HTML_RENDER_SELECTION_EVENT, + (event) => selections.push((event as CustomEvent).detail), + { signal: listeners.signal }, + ); + const onContentHeight = vi.fn(); + await using screen = await render( + <> + + + , + ); + await vi.waitFor(() => expect(onContentHeight).toHaveBeenCalled()); + const frame = screen.getByTitle("Quote").element() as HTMLIFrameElement; + const post = (command: "select" | "collapse" | "scroll-ticker") => { + const settled = new Promise((resolve) => + window.addEventListener( + "message", + (event) => { + if ( + event.source === frame.contentWindow && + event.data === (command === "scroll-ticker" ? "ticker-scrolled" : "selection-settled") + ) { + resolve(); + } + }, + { signal: listeners.signal }, + ), + ); + frame.contentWindow?.postMessage(command, "*"); + return settled; + }; + const reported = { + text: "Quoted words", + rect: expect.objectContaining({ left: expect.any(Number), bottom: expect.any(Number) }), + bounds: expect.objectContaining({ left: expect.any(Number), bottom: expect.any(Number) }), + }; + + await post("select"); + expect(selections).toEqual([]); + // A press inside an existing selection would drag that text instead of selecting anew. + await post("collapse"); + + // The page reports a drag only once the button is released. + const quote = page.frameLocator(screen.getByTitle("Quote")).getByText("Quoted words"); + await userEvent.dragAndDrop(quote, quote, { + sourcePosition: { x: 0, y: 5 }, + targetPosition: { x: 250, y: 5 }, + }); + await vi.waitFor(() => expect(selections.at(-1)).toEqual(reported)); + for (const box of [selections.at(-1)!.rect, selections.at(-1)!.bounds]) { + expect(box.left).toBeGreaterThanOrEqual(0); + expect(box.top).toBeGreaterThanOrEqual(0); + expect(box.right).toBeGreaterThan(box.left); + expect(box.bottom).toBeGreaterThan(box.top); + expect(box.right).toBeLessThanOrEqual(frame.clientWidth); + expect(box.bottom).toBeLessThanOrEqual(frame.clientHeight); + } + + // Scrolling content that holds none of the selection keeps it. + await post("scroll-ticker"); + expect(selections.at(-1)).toEqual(reported); + + await post("collapse"); + expect(selections.at(-1)).toBeNull(); + + await post("select"); + expect(selections.at(-1)).toEqual(reported); + screen.getByRole("button", { name: "Outside" }).element().focus(); + await vi.waitFor(() => expect(selections.at(-1)).toBeNull()); + }); }); it("keeps the chat in place when a label far from its hidden radio is clicked", async () => { diff --git a/apps/web/src/components/files/BrowserDocumentFrame.tsx b/apps/web/src/components/files/BrowserDocumentFrame.tsx index 4dba7607f81c0..ace7adaec73eb 100644 --- a/apps/web/src/components/files/BrowserDocumentFrame.tsx +++ b/apps/web/src/components/files/BrowserDocumentFrame.tsx @@ -7,6 +7,7 @@ import { THREAD_JUMP_KEYBINDING_COMMANDS, } from "@t3tools/contracts"; import { + type HtmlRenderBox, type HtmlRenderShortcut, htmlRenderShortcutsMessage, htmlRenderThemeFragment, @@ -16,6 +17,7 @@ import { readHtmlRenderImageRequest, readHtmlRenderKeyDown, readHtmlRenderLinkRequest, + readHtmlRenderSelection, } from "@t3tools/shared/htmlRender"; import { isMacPlatform, @@ -69,6 +71,12 @@ const HTML_RENDER_SHORTCUT_COMMANDS: ReadonlySet = new Set([ ...MODEL_PICKER_JUMP_KEYBINDING_COMMANDS, ]); +/** + * Bubbles from an HTML render's frame with the page's selection, in the frame's + * viewport coordinates, or null once the page clears it. + */ +export const HTML_RENDER_SELECTION_EVENT = "t3:html-render-selection"; + export const isPdfPreviewFile = (path: string): boolean => /\.pdf$/i.test(path.split(/[?#]/, 1)[0] ?? ""); @@ -284,6 +292,26 @@ export function HtmlRenderDocument(props: { props.onContentHeight?.(height); return; } + // A selection only shows Cite, and nothing reaches the composer until the reader clicks it, + // so it skips the activation check: long drags and touch selection outlast that activation. + const selection = readHtmlRenderSelection(event.data); + if (selection !== undefined) { + if (selection !== null && document.activeElement !== frame) return; + const clamp = (value: number, max: number) => Math.min(Math.max(value, 0), max); + const inFrame = (box: HtmlRenderBox) => ({ + left: clamp(box.left, frame.clientWidth), + top: clamp(box.top, frame.clientHeight), + right: clamp(box.right, frame.clientWidth), + bottom: clamp(box.bottom, frame.clientHeight), + }); + const detail = selection && { + text: selection.text, + rect: inFrame(selection.rect), + bounds: inFrame(selection.bounds), + }; + frame.dispatchEvent(new CustomEvent(HTML_RENDER_SELECTION_EVENT, { bubbles: true, detail })); + return; + } if (document.activeElement !== frame || navigator.userActivation?.isActive === false) return; const linkRequest = readHtmlRenderLinkRequest(event.data); if (linkRequest !== undefined) { diff --git a/apps/web/src/lib/selectionActions.ts b/apps/web/src/lib/selectionActions.ts index 9303602097543..240ac5dbd0628 100644 --- a/apps/web/src/lib/selectionActions.ts +++ b/apps/web/src/lib/selectionActions.ts @@ -186,6 +186,12 @@ export function observeSelectionActions({ schedule(0); } }, + /** A framed document reports its own finished selection; focus moving into it blurred this window. */ + selectionReported() { + dismissed = false; + pointer = null; + schedule(0); + }, dispose() { cancel(); document.removeEventListener("pointerdown", onPointerDown, true); diff --git a/docs/internals/assistant-citations.md b/docs/internals/assistant-citations.md index ec830f75bc031..aadadc8fd6628 100644 --- a/docs/internals/assistant-citations.md +++ b/docs/internals/assistant-citations.md @@ -21,6 +21,11 @@ highlight. The [navigation hash](../../apps/web/src/lib/assistantCitationNavigat uses base64url because router normalization otherwise decodes whitespace and source IDs inside the citation URI. +An HTML render is cited by its turn item ID in the message slot. Its sandboxed +page reports selections over the [render bridge](../../packages/shared/src/htmlRender.ts), +and navigation stops at the render's row because the host cannot read or +highlight text inside the frame. + Expand citations with the [shared formatter](../../packages/shared/src/assistantCitations.ts) before dispatch to a provider adapter. Send the saved excerpt without looking up the source, distinguish quoted reference material from the user's comment, and diff --git a/docs/user/composer.md b/docs/user/composer.md index 89a96f120d4d0..ce0faf06d6012 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -128,9 +128,9 @@ Leaving reasoning level or service tier unset uses the provider's own configurat ## Quote an assistant response -On web and desktop, select text within one assistant response and choose -**Cite in composer**. You can add a comment about the quote and write instructions -around it. +On web and desktop, select text within one assistant response, or within an HTML render shown in +the thread (not its full-screen view), and choose **Cite in composer**. You can add a comment +about the quote and write instructions around it. Select the quote in a draft or sent message to return to its source. If the source is unavailable or has changed, the saved quote remains readable. diff --git a/packages/contracts/src/assistantCitations.ts b/packages/contracts/src/assistantCitations.ts index 96572a4bfa656..d1074b9320d35 100644 --- a/packages/contracts/src/assistantCitations.ts +++ b/packages/contracts/src/assistantCitations.ts @@ -12,6 +12,7 @@ export const AssistantCitation = Schema.Struct({ version: Schema.Literal(1), environmentId: EnvironmentId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)), threadId: ThreadId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)), + /** The cited assistant message, or the turn item id of a cited HTML render. */ messageId: MessageId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)), text: Schema.String.check( Schema.isNonEmpty(), diff --git a/packages/shared/src/htmlRender.test.ts b/packages/shared/src/htmlRender.test.ts index 9f6315541547f..73b9b0e08385e 100644 --- a/packages/shared/src/htmlRender.test.ts +++ b/packages/shared/src/htmlRender.test.ts @@ -4,6 +4,7 @@ import { injectHtmlRenderBootstrap, readHtmlRenderImageRequest, readHtmlRenderLinkRequest, + readHtmlRenderSelection, } from "./htmlRender.ts"; import { htmlRenderFromToolItem } from "./toolOutput.ts"; @@ -158,6 +159,41 @@ describe("readHtmlRenderImageRequest", () => { }); }); +describe("readHtmlRenderSelection", () => { + it("reads a selection with its bounds and last line box, a clear, and nothing else", () => { + const selection = (params: unknown, method = "t3/notifications/selection-changed") => ({ + jsonrpc: "2.0", + method, + params, + }); + const rect = { left: 8, top: 20, right: 120, bottom: 38 }; + const bounds = { left: 8, top: 0, right: 300, bottom: 38 }; + expect(readHtmlRenderSelection(selection({ text: " Quoted\nwords ", rect, bounds }))).toEqual({ + text: " Quoted\nwords ", + rect, + bounds, + }); + expect(readHtmlRenderSelection(selection({ text: "" }))).toBeNull(); + expect(readHtmlRenderSelection(selection({ text: " \n ", rect, bounds }))).toBeNull(); + for (const params of [ + { text: "Quoted", bounds }, + { text: "Quoted", rect }, + { text: "Quoted", rect: { ...rect, top: Number.NaN }, bounds }, + { text: "Quoted", rect: { ...rect, right: 0 }, bounds }, + { text: "Quoted", rect, bounds: { ...bounds, bottom: Number.POSITIVE_INFINITY } }, + { text: "Quoted", rect, bounds: { ...bounds, top: 40 } }, + { text: 1, rect, bounds }, + ]) { + expect(readHtmlRenderSelection(selection(params))).toBeUndefined(); + } + expect( + readHtmlRenderSelection( + selection({ text: "Quoted", rect, bounds }, "t3/notifications/open-image"), + ), + ).toBeUndefined(); + }); +}); + describe("htmlRenderFromToolItem", () => { const result = { htmlRender: reference, message: "Rendered above your reply." }; diff --git a/packages/shared/src/htmlRender.ts b/packages/shared/src/htmlRender.ts index 07ade4db11068..f128752abba34 100644 --- a/packages/shared/src/htmlRender.ts +++ b/packages/shared/src/htmlRender.ts @@ -244,6 +244,7 @@ const SIZE_CHANGED_METHOD = "ui/notifications/size-changed"; const SHORTCUTS_METHOD = "t3/notifications/shortcuts"; const KEY_DOWN_METHOD = "t3/notifications/keydown"; const OPEN_IMAGE_METHOD = "t3/notifications/open-image"; +const SELECTION_CHANGED_METHOD = "t3/notifications/selection-changed"; export function readHtmlRenderContentHeight(data: unknown): number | undefined { if (typeof data !== "object" || data === null) return undefined; @@ -323,6 +324,52 @@ export function readHtmlRenderImageRequest( : { images: [clicked], index: 0 }; } +export interface HtmlRenderBox { + readonly left: number; + readonly top: number; + readonly right: number; + readonly bottom: number; +} + +/** Boxes are in the frame's viewport coordinates. */ +export interface HtmlRenderSelection { + readonly text: string; + /** The last line box, where Cite sits. */ + readonly rect: HtmlRenderBox; + /** The whole selection, so a visible start keeps Cite when the last line is scrolled away. */ + readonly bounds: HtmlRenderBox; +} + +const isFiniteNumber = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + +function readHtmlRenderBox(value: unknown): HtmlRenderBox | undefined { + if (typeof value !== "object" || value === null) return undefined; + const { left, top, right, bottom } = value as Record; + return isFiniteNumber(left) && + isFiniteNumber(top) && + isFiniteNumber(right) && + isFiniteNumber(bottom) && + right >= left && + bottom >= top + ? { left, top, right, bottom } + : undefined; +} + +/** Null clears the page's reported selection. */ +export function readHtmlRenderSelection(data: unknown): HtmlRenderSelection | null | undefined { + if (typeof data !== "object" || data === null) return undefined; + const { jsonrpc, method, params } = data as Record; + if (jsonrpc !== "2.0" || method !== SELECTION_CHANGED_METHOD) return undefined; + if (typeof params !== "object" || params === null) return undefined; + const { text, rect, bounds } = params as Record; + if (typeof text !== "string") return undefined; + if (text.trim().length === 0) return null; + const line = readHtmlRenderBox(rect); + const box = readHtmlRenderBox(bounds); + return line && box ? { text, rect: line, bounds: box } : undefined; +} + export function readHtmlRenderKeyDown(data: unknown): | { readonly key: string; @@ -403,7 +450,9 @@ function rootRule(theme: HtmlRenderTheme): string { // Apply overridable theme CSS before paint and strip the fragment before routing; framed navigation uses the client. // Forward shortcuts after page handlers; text fields retain their keys. -const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);document.addEventListener("click",function(e){var t=e.composedPath()[0],l=t&&t.closest?t.closest("label,a[href],button,input,select,textarea"):null,c=l&&l.tagName==="LABEL"?l.control:null;if(e.isTrusted&&c&&document.activeElement!==c)c.focus({preventScroll:true});},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);window.addEventListener("click",function(e){var t=e.composedPath()[0],u;if(!e.isTrusted||e.defaultPrevented||e.button!==0||e.metaKey||e.ctrlKey||e.shiftKey||e.altKey||!t||t.tagName!=="IMG"||t.closest("a[href],button"))return;u=t.currentSrc||t.src;if(u)setTimeout(function(){var g=[],i=-1;if(e.defaultPrevented)return;Array.prototype.forEach.call(document.images,function(m){var v=m.currentSrc||m.src;if(!v||m.closest("a[href],button"))return;if(m===t)i=g.length;g.push({src:v,name:m.alt||m.title||""});});window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(OPEN_IMAGE_METHOD)},params:{src:u,name:t.alt||t.title||"",images:g,index:i}},"*");});});var K=[],B=[],C=${JSON.stringify(EVENT_CODE_SHORTCUT_KEYS)};window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(e.source===window.parent&&d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(SHORTCUTS_METHOD)}&&p&&Array.isArray(p.shortcuts)&&Array.isArray(p.blocked)){K=p.shortcuts;B=p.blocked;}});window.addEventListener("keydown",function(e){var t=e.composedPath()[0],c=e.code||"",l=e.key.toLowerCase(),x=/^[a-z]$/.test(l)?[l]:[l,/^Key[A-Z]$/.test(c)?c.slice(3).toLowerCase():"",l==="+"&&!e.shiftKey&&!e.altKey&&!/^Digit/.test(c)?"":C.hasOwnProperty(c)?C[c]:""],f,m=function(s){return s&&s.metaKey===e.metaKey&&s.ctrlKey===e.ctrlKey&&s.shiftKey===e.shiftKey&&s.altKey===e.altKey&&x.indexOf(s.key)>-1;};if(!e.isTrusted||e.isComposing||e.getModifierState("AltGraph")&&!/^[a-z0-9]$/i.test(e.key)||t&&(t.isContentEditable||/^(?:INPUT|TEXTAREA|SELECT)$/.test(t.tagName))||!K.some(m)||B.some(m))return;f=function(v){window.removeEventListener("keydown",f);if(v!==e||e.defaultPrevented)return;e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(KEY_DOWN_METHOD)},params:{key:e.key,code:e.code,metaKey:e.metaKey,ctrlKey:e.ctrlKey,shiftKey:e.shiftKey,altKey:e.altKey,repeat:e.repeat}},"*");};window.addEventListener("keydown",f);setTimeout(function(){window.removeEventListener("keydown",f);});},true);}})();`; +// Report a selection once the pointer is released. Clears go out at once, and so do focus leaving +// the page (the host sees no frame-to-frame focus move) and scrolls that move the selection. +const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);document.addEventListener("click",function(e){var t=e.composedPath()[0],l=t&&t.closest?t.closest("label,a[href],button,input,select,textarea"):null,c=l&&l.tagName==="LABEL"?l.control:null;if(e.isTrusted&&c&&document.activeElement!==c)c.focus({preventScroll:true});},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);window.addEventListener("click",function(e){var t=e.composedPath()[0],u;if(!e.isTrusted||e.defaultPrevented||e.button!==0||e.metaKey||e.ctrlKey||e.shiftKey||e.altKey||!t||t.tagName!=="IMG"||t.closest("a[href],button"))return;u=t.currentSrc||t.src;if(u)setTimeout(function(){var g=[],i=-1;if(e.defaultPrevented)return;Array.prototype.forEach.call(document.images,function(m){var v=m.currentSrc||m.src;if(!v||m.closest("a[href],button"))return;if(m===t)i=g.length;g.push({src:v,name:m.alt||m.title||""});});window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(OPEN_IMAGE_METHOD)},params:{src:u,name:t.alt||t.title||"",images:g,index:i}},"*");});});var K=[],B=[],C=${JSON.stringify(EVENT_CODE_SHORTCUT_KEYS)};window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(e.source===window.parent&&d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(SHORTCUTS_METHOD)}&&p&&Array.isArray(p.shortcuts)&&Array.isArray(p.blocked)){K=p.shortcuts;B=p.blocked;}});window.addEventListener("keydown",function(e){var t=e.composedPath()[0],c=e.code||"",l=e.key.toLowerCase(),x=/^[a-z]$/.test(l)?[l]:[l,/^Key[A-Z]$/.test(c)?c.slice(3).toLowerCase():"",l==="+"&&!e.shiftKey&&!e.altKey&&!/^Digit/.test(c)?"":C.hasOwnProperty(c)?C[c]:""],f,m=function(s){return s&&s.metaKey===e.metaKey&&s.ctrlKey===e.ctrlKey&&s.shiftKey===e.shiftKey&&s.altKey===e.altKey&&x.indexOf(s.key)>-1;};if(!e.isTrusted||e.isComposing||e.getModifierState("AltGraph")&&!/^[a-z0-9]$/i.test(e.key)||t&&(t.isContentEditable||/^(?:INPUT|TEXTAREA|SELECT)$/.test(t.tagName))||!K.some(m)||B.some(m))return;f=function(v){window.removeEventListener("keydown",f);if(v!==e||e.defaultPrevented)return;e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(KEY_DOWN_METHOD)},params:{key:e.key,code:e.code,metaKey:e.metaKey,ctrlKey:e.ctrlKey,shiftKey:e.shiftKey,altKey:e.altKey,repeat:e.repeat}},"*");};window.addEventListener("keydown",f);setTimeout(function(){window.removeEventListener("keydown",f);});},true);var D=0,P=0,Q=1,W=function(p){window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SELECTION_CHANGED_METHOD)},params:p},"*");},X=function(){if(!Q){Q=1;W({text:""});}},Y=function(){var g=getSelection(),e=document.activeElement,t=g&&g.rangeCount?g.toString():"",q,r,b;if(!t.trim()||e&&(e.isContentEditable||/^(?:INPUT|TEXTAREA)$/.test(e.tagName)))return X();if(D){P=1;return;}r=g.getRangeAt(g.rangeCount-1);q=[].filter.call(r.getClientRects(),function(l){return l.width>0&&l.height>0;});b=r.getBoundingClientRect();r=q[q.length-1]||b;Q=0;W({text:t,rect:{left:r.left,top:r.top,right:r.right,bottom:r.bottom},bounds:{left:b.left,top:b.top,right:b.right,bottom:b.bottom}});},U=function(e){if(e.isPrimary){D=0;if(P){P=0;Y();}}};document.addEventListener("selectionchange",function(){P=0;Y();});window.addEventListener("pointerdown",function(e){if(e.isPrimary&&e.button===0)D=1;},true);window.addEventListener("pointerup",U,true);window.addEventListener("pointercancel",U,true);window.addEventListener("blur",X);window.addEventListener("scroll",function(e){var g,t=e.target;if(Q)return;g=getSelection();if(g&&g.rangeCount&&t&&t.nodeType&&!g.getRangeAt(0).intersectsNode(t))return;X();},true);}})();`; function bootstrapMarkup(markup: string): string { const dark = htmlRenderTheme(T3_CODE_DARK_THEME_COLORS, "dark");