diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 94fbe68e29528..9e9d82a11fcbd 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -79,7 +79,7 @@ import { } from "@t3tools/shared/toolActivity"; import { isWindowsAbsolutePath } from "@t3tools/shared/path"; -function timelineEntryRunId(entry: TimelineEntry): RunId | null { +export function timelineEntryRunId(entry: TimelineEntry): RunId | null { if (entry.kind === "message") { return entry.message.role === "assistant" ? (entry.message.runId ?? null) : null; } @@ -917,7 +917,7 @@ function timelineEntryIsSteer(entry: TimelineEntry): boolean { ); } -function timelineEntryStartsResponse(entry: TimelineEntry): boolean { +export function timelineEntryStartsResponse(entry: TimelineEntry): boolean { return ( (entry.kind === "message" && entry.message.role === "user" && !timelineEntryIsSteer(entry)) || (entry.kind === "work" && entry.entry.itemType === "notification") diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 61d5c4d97e45e..a2b39ffd143f9 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -200,6 +200,7 @@ const TIMELINE_MAINTAIN_SCROLL_AT_END = { }, } as const satisfies MaintainScrollAtEndOptions; const EMPTY_RUN_IDS: ReadonlySet = new Set(); +const EMPTY_TIMELINE_ENTRIES: ReadonlyArray = []; const TIMELINE_MAINTAIN_SCROLL_AT_END_SMOOTH = { ...TIMELINE_MAINTAIN_SCROLL_AT_END, animated: true, @@ -552,6 +553,9 @@ const ConversationTimeline = memo(function ConversationTimeline({ } const preferredToolCallVisibility = useClientSettings((settings) => settings.toolCallVisibility); + const screenReaderAnnouncements = useClientSettings( + (settings) => settings.screenReaderAnnouncements, + ); const toolCallVisibility = findActive && preferredToolCallVisibility === "zen" ? "hideAll" : preferredToolCallVisibility; const revealedMessageId = citationRequest?.citation.messageId ?? null; @@ -1078,6 +1082,9 @@ const ConversationTimeline = memo(function ConversationTimeline({ runId={announcedRun?.runId ?? null} runStatus={announcedRun?.status ?? null} detailBehind={detailBehindShell} + level={screenReaderAnnouncements} + entries={screenReaderAnnouncements === "status" ? EMPTY_TIMELINE_ENTRIES : timelineEntries} + workspaceRoot={workspaceRoot} /> ); const listHeaderLeadingContent = diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx new file mode 100644 index 0000000000000..b9efe2dcd610e --- /dev/null +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -0,0 +1,157 @@ +import { MessageId, RunId } from "@t3tools/contracts"; +import { type ComponentProps } from "react"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { act, fakeTimers, render } from "../../../test/render"; +import type { TimelineEntry } from "../../session-logic"; +import { ThreadActivityAnnouncer } from "./ThreadActivityAnnouncer"; + +const run = RunId.make("run-1"); +const at = "2026-01-01T00:00:00.000Z"; + +const message = (id: string, role: "user" | "assistant", streaming = false): TimelineEntry => ({ + kind: "message", + id, + createdAt: at, + message: { + id: MessageId.make(id), + role, + text: id, + runId: role === "assistant" ? run : null, + streaming, + createdAt: at, + updatedAt: at, + }, +}); +const command = (program: string): TimelineEntry => ({ + kind: "work", + id: program, + createdAt: at, + entry: { + id: program, + createdAt: at, + runId: run, + label: program, + tone: "tool", + itemType: "command_execution", + command: program, + toolLifecycleStatus: "inProgress", + }, +}); +const prompt = message("prompt", "user"); + +const announcer = (props: Partial>) => ( + +); + +const regions = (container: HTMLElement) => { + const [detail, status] = container.querySelectorAll('[aria-live="polite"]'); + return { detail: detail!, status: status! }; +}; + +describe("ThreadActivityAnnouncer", () => { + it("drops a waiting step once the agent has moved on", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + { + // Restored before the assertions, which poll on real timers. + using _timers = fakeTimers(); + await screen.rerender(announcer({})); + await screen.rerender(announcer({ entries: [prompt, command("git")] })); + await screen.rerender( + announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }), + ); + await act(() => vi.advanceTimersByTime(5_000)); + } + + await expect.element(status).toHaveTextContent("Working"); + await expect.element(detail).toBeEmptyDOMElement(); + }); + + it("holds a waiting step while the thread's detail is behind, then reads it", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail } = regions(screen.container); + const entries = [prompt, command("git")]; + const heard: string[] = []; + { + // Restored before the assertion, which polls on real timers. + using _timers = fakeTimers(); + await screen.rerender(announcer({})); + await screen.rerender(announcer({ entries })); + await screen.rerender(announcer({ entries, detailBehind: true })); + await act(() => vi.advanceTimersByTime(5_000)); + heard.push(detail.textContent); + await screen.rerender(announcer({ entries })); + await act(() => vi.advanceTimersByTime(2_000)); + } + + expect(heard).toEqual([""]); + await expect.element(detail).toHaveTextContent("Running git"); + }); + + it("keeps a message beside the status that follows it, then reads the newest step", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + const entries = [prompt, message("Checking the repo", "assistant"), command("git")]; + const heard: Array<[status: string, detail: string]> = []; + using _timers = fakeTimers(); + for (const props of [ + {}, + { entries }, + { entries, status: "approval", requestId: "request-1" }, + { entries: [...entries, command("rg")] }, + ] satisfies ReadonlyArray>>) { + await screen.rerender(announcer(props)); + heard.push([status.textContent, detail.textContent]); + } + await act(() => vi.advanceTimersByTime(2_000)); + heard.push([status.textContent, detail.textContent]); + + expect(heard).toEqual([ + ["Working", ""], + ["Working", "T3 Code: Checking the repo"], + ["Needs approval", "T3 Code: Checking the repo"], + ["Needs approval", "T3 Code: Checking the repo"], + ["Needs approval", "Running rg"], + ]); + }); + + it("reads a status that lands with a message as the message's last sentence", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + await screen.rerender(announcer({})); + await screen.rerender( + announcer({ + entries: [prompt, message("Checking the repo", "assistant"), command("git")], + status: "approval", + requestId: "request-1", + }), + ); + + await expect.element(detail).toHaveTextContent("T3 Code: Checking the repo. Needs approval"); + await expect.element(status).toHaveTextContent("Working"); + }); +}); diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 998b9e1522ad5..0dc54f3f2df00 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -1,14 +1,46 @@ +import type { ScreenReaderAnnouncements } from "@t3tools/contracts"; import { memo, useEffect, useRef } from "react"; import { useLiveRegion } from "../../hooks/useLiveRegion"; +import type { TimelineEntry } from "../../session-logic"; import { nextThreadAnnouncement, type ThreadActivitySnapshot, type ThreadAnnouncementMemory, } from "./threadActivityAnnouncement"; +import { + nextTurnDetailAnnouncement, + NO_TURN_DETAIL, + turnDetail, + type TurnDetailMemory, +} from "./turnDetailAnnouncement"; + +// Long enough for a screen reader to finish a short label before the next one. +const ACTIVITY_GAP_MS = 2_000; +// A polite region queues what it can't say yet, so steps written while a +// message is still being read would all be heard late. They wait this long. +const MESSAGE_HOLD_MS_PER_CHARACTER = 50; +const MESSAGE_HOLD_MAX_MS = 15_000; + +function holdAfter(text: string) { + return Math.min( + MESSAGE_HOLD_MAX_MS, + Math.max(ACTIVITY_GAP_MS, text.length * MESSAGE_HOLD_MS_PER_CHARACTER), + ); +} + +/** One write, since a second write to a region replaces the first before it is heard. */ +function asOneAnnouncement(texts: ReadonlyArray) { + return texts + .map((text, index) => (index === texts.length - 1 || /[.!?:…]$/.test(text) ? text : `${text}.`)) + .join(" "); +} -/** One polite live region for the open thread. */ +/** + * The open thread's polite live regions. Status has its own, so "Needs + * approval" landing a moment after a message doesn't replace it unheard. + */ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ threadKey, status, @@ -16,35 +48,141 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ runId, runStatus, detailBehind, + level, + entries, + workspaceRoot, }: ThreadActivitySnapshot & { /** Null while the timeline shows another thread during a switch. */ readonly threadKey: string | null; /** See isDetailBehindShell. */ readonly detailBehind: boolean; + readonly level: ScreenReaderAnnouncements; + /** The thread's entries before any are hidden; unread at the "status" level. */ + readonly entries: ReadonlyArray; + readonly workspaceRoot: string | undefined; }) { const { regionRef, announce, clear } = useLiveRegion(); - const memoryRef = useRef<{ threadKey: string; memory: ThreadAnnouncementMemory } | null>(null); + const { + regionRef: detailRegionRef, + announce: announceDetail, + clear: clearDetail, + } = useLiveRegion(); + const memoryRef = useRef<{ + threadKey: string; + level: ScreenReaderAnnouncements; + memory: ThreadAnnouncementMemory; + detail: TurnDetailMemory; + } | null>(null); + // Messages are written at once; a step waits its turn, and a newer one replaces it. + const stepRef = useRef<{ + quietUntil: number; + waiting: { label: string; timer: ReturnType } | null; + }>({ quietUntil: 0, waiting: null }); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes thread status changes to the aria-live region after commit + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes the turn's status, messages and steps to the aria-live region after commit useEffect(() => { + const step = stepRef.current; + const write = (text: string, quietMs: number) => { + announceDetail(text); + step.quietUntil = Date.now() + quietMs; + }; + const takeWaiting = () => { + const label = step.waiting?.label ?? null; + if (step.waiting !== null) clearTimeout(step.waiting.timer); + step.waiting = null; + return label; + }; + if (threadKey === null) { + takeWaiting(); memoryRef.current = null; clear(); + clearDetail(); + return; + } + const waitingLabel = takeWaiting(); + if (detailBehind) { + // The dropped step counts as unsaid, so it is read once the detail catches up. + if (waitingLabel !== null && memoryRef.current !== null) { + const { detail } = memoryRef.current; + memoryRef.current = { ...memoryRef.current, detail: { ...detail, activity: null } }; + } return; } - if (detailBehind) return; - const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current.memory : null; - const { memory, announcement } = nextThreadAnnouncement(previous, { + const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current : null; + const turn = nextThreadAnnouncement(previous?.memory ?? null, { status, requestId, runId, runStatus, }); - memoryRef.current = { threadKey, memory }; - if (previous === null) clear(); - else if (announcement !== null) announce(announcement); - }, [threadKey, status, requestId, runId, runStatus, detailBehind, announce, clear]); + // A new level starts from what the turn already holds, so raising it mid-turn replays nothing. + const previousDetail = previous?.level === level ? previous.detail : null; + const detail = nextTurnDetailAnnouncement(previousDetail, { + ...(level === "status" + ? NO_TURN_DETAIL + : turnDetail(entries, workspaceRoot, previousDetail?.tailId ?? null)), + level, + status, + }); + memoryRef.current = { threadKey, level, memory: turn.memory, detail: detail.memory }; + if (previous === null) { + step.quietUntil = 0; + clear(); + clearDetail(); + return; + } + if (detail.messages.length > 0) { + // NVDA reads two regions written in the same frame in either order, so a + // status that lands with a message is read as its last sentence. + const text = asOneAnnouncement( + turn.announcement === null ? detail.messages : [...detail.messages, turn.announcement], + ); + write(text, holdAfter(text)); + } else if (turn.announcement !== null) { + announce(turn.announcement); + step.quietUntil = Math.max(step.quietUntil, Date.now() + ACTIVITY_GAP_MS); + } + // A waiting label is stale once its step is over or the level no longer reads steps. + const label = + level !== "everything" || detail.memory.activity === null + ? null + : (detail.activity ?? waitingLabel); + if (label === null) return; + const speakStep = () => { + step.waiting = null; + write(label, ACTIVITY_GAP_MS); + }; + const wait = step.quietUntil - Date.now(); + if (wait <= 0) { + speakStep(); + return; + } + const timer = setTimeout(speakStep, wait); + step.waiting = { label, timer }; + // The next run takes the label over; after an unmount nothing is left to speak it. + return () => clearTimeout(timer); + }, [ + threadKey, + status, + requestId, + runId, + runStatus, + detailBehind, + level, + entries, + workspaceRoot, + announce, + clear, + announceDetail, + clearDetail, + ]); // No role="status": Orca 50 maps it to a status bar in Chromium and drops its updates. - return
; + return ( + <> +
+
+ + ); }); diff --git a/apps/web/src/components/chat/timelineMinimapItems.ts b/apps/web/src/components/chat/timelineMinimapItems.ts index 3c730df680e54..386abbaa013a3 100644 --- a/apps/web/src/components/chat/timelineMinimapItems.ts +++ b/apps/web/src/components/chat/timelineMinimapItems.ts @@ -89,6 +89,11 @@ function markdownNodeText(node: Nodes): string { return node.type === "link" && parseThreadLinkHref(text) !== null ? "a thread" : text; } +/** Markdown as one line of the text it renders, without markup or link targets. */ +export function markdownPlainText(markdown: string): string { + return markdownNodeText(markdownPreviewParser.parse(markdown)).replace(/\s+/g, " ").trim(); +} + function compactMinimapPreview(text: string | null | undefined) { const compact = text?.replace(/\s+/g, " ").trim() ?? ""; return compact.length > 0 ? compact : null; diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts new file mode 100644 index 0000000000000..c5eaa080ff921 --- /dev/null +++ b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts @@ -0,0 +1,320 @@ +import { MessageId, RunAttemptId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import { describe, expect, it } from "vite-plus/test"; + +import type { TimelineAttempt, TimelineEntry, WorkLogEntry } from "../../session-logic"; +import { + nextTurnDetailAnnouncement, + turnDetail, + type TurnDetailMemory, + type TurnDetailSnapshot, + type TurnMessage, +} from "./turnDetailAnnouncement"; + +const run = RunId.make("run-1"); +const at = "2026-01-01T00:00:00.000Z"; + +function message( + id: string, + role: "user" | "assistant", + fields: Partial["message"]> = {}, +): TimelineEntry { + return { + kind: "message", + id, + createdAt: at, + message: { + id: MessageId.make(id), + role, + text: id, + runId: role === "assistant" ? run : null, + streaming: false, + createdAt: at, + updatedAt: at, + ...fields, + }, + }; +} + +function work(id: string, fields: Partial = {}): TimelineEntry { + return { + kind: "work", + id, + createdAt: at, + entry: { id, createdAt: at, runId: run, label: id, tone: "tool", ...fields }, + }; +} + +const prompt = message("prompt", "user"); +const command = (id: string, fields: Partial = {}) => + work(id, { + itemType: "command_execution", + command: "git status", + toolLifecycleStatus: "inProgress", + ...fields, + }); +const notice = work("notice", { itemType: "system_notice", tone: "info", label: "Heads up" }); +const steer = message("steer", "user", { inputIntent: "steer" }); +const time = (clock: string) => DateTime.makeUnsafe(`2026-01-01T${clock}.000Z`); +const inAttempt = ( + entry: TimelineEntry, + id: string, + status: TimelineAttempt["status"], + completed?: string, +): TimelineEntry => ({ + ...entry, + attempt: { + id: RunAttemptId.make(id), + runId: run, + attemptOrdinal: 1, + rootNodeId: `${id}-root` as TimelineAttempt["rootNodeId"], + status, + completedAt: completed === undefined ? null : time(completed), + }, +}); +const completedAt = (entry: TimelineEntry, clock: string) => + ({ + ...entry, + projectedItem: { + item: { type: "assistant_message", status: "completed", completedAt: time(clock) }, + }, + }) as TimelineEntry; + +describe("turnDetail", () => { + it.each<{ + name: string; + entries: ReadonlyArray; + activity: string | null; + }>([ + { name: "a running command", entries: [prompt, command("git")], activity: "Running git" }, + { + name: "reasoning, without its text", + entries: [ + prompt, + work("thought", { itemType: "reasoning", tone: "thinking", detail: "Private idea." }), + ], + activity: "Thinking", + }, + { + name: "a file change, with the verb its icon stands for", + entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["search.ts"] })], + activity: "Editing search.ts", + }, + { + name: "the step under a request and a notice", + entries: [prompt, command("git"), work("ask", { itemType: "approval_request" }), notice], + activity: "Running git", + }, + { + name: "nothing for a failed step", + entries: [prompt, command("git", { toolLifecycleStatus: "failed" })], + activity: null, + }, + { + name: "nothing once the agent is writing again", + entries: [prompt, command("git"), message("reply", "assistant", { streaming: true })], + activity: null, + }, + { + name: "nothing while the context is compacted", + entries: [ + prompt, + command("git"), + work("compact", { tone: "info", sourceActivityKind: "context-compaction" }), + ], + activity: null, + }, + { + name: "nothing from the turn before", + entries: [message("earlier", "user"), command("git"), prompt], + activity: null, + }, + ])("names $name", ({ entries, activity }) => { + expect(turnDetail(entries, undefined, null).activity).toBe(activity); + }); + + it.each<{ + name: string; + entries: ReadonlyArray; + finished: ReadonlyArray; + }>([ + { + name: "a message the agent kept working after, but not its last", + entries: [ + prompt, + message("first", "assistant"), + command("git"), + message("last", "assistant"), + ], + finished: ["first"], + }, + { + name: "a message once the next one starts", + entries: [ + prompt, + message("first", "assistant"), + message("second", "assistant", { streaming: true }), + ], + finished: ["first"], + }, + { + name: "no reply a steer cut off", + entries: [ + prompt, + inAttempt(message("cut off", "assistant"), "attempt-1", "superseded"), + steer, + inAttempt(command("git"), "attempt-2", "running"), + ], + finished: [], + }, + { + name: "a reply that completed before a steer replaced its attempt", + entries: [ + prompt, + completedAt( + inAttempt(message("complete", "assistant"), "attempt-1", "superseded", "00:00:10"), + "00:00:04", + ), + steer, + inAttempt(command("git"), "attempt-2", "running"), + ], + finished: ["complete"], + }, + { + name: "no last message that only a notice follows", + entries: [prompt, command("git"), message("last", "assistant"), notice], + finished: [], + }, + { + name: "no last message when another run picks the thread up", + entries: [ + prompt, + message("last", "assistant"), + command("git", { runId: RunId.make("run-2") }), + ], + finished: [], + }, + ])("counts $name as finished", ({ entries, finished }) => { + const { messages } = turnDetail(entries, undefined, null); + expect(messages.filter((spoken) => spoken.finished).map((spoken) => spoken.text)).toEqual( + finished, + ); + }); + + it.each<{ name: string; knownTailId: string | null; fresh: ReadonlyArray }>([ + { name: "after the entry last seen", knownTailId: "seen", fresh: [false, true] }, + { name: "after a prompt that was last seen", knownTailId: "prompt", fresh: [true, true] }, + { name: "not when the entry last seen is gone", knownTailId: "gone", fresh: [false, false] }, + ])("marks messages as fresh $name", ({ knownTailId, fresh }) => { + const entries = [ + prompt, + message("loaded", "assistant"), + command("seen"), + message("new", "assistant"), + command("git"), + ]; + expect( + turnDetail(entries, undefined, knownTailId).messages.map((spoken) => spoken.fresh), + ).toEqual(fresh); + }); +}); + +type Frame = Pick; +const said = (id: string, finished = true, fresh = false): TurnMessage => ({ + id, + text: id, + finished, + fresh, +}); +const frame = ( + activity: string | null, + messages: ReadonlyArray = [], + status: TurnDetailSnapshot["status"] = "working", +): Frame => ({ status, activity, messages }); + +function speak(level: ScreenReaderAnnouncements, frames: ReadonlyArray) { + let memory: TurnDetailMemory | null = null; + const spoken: string[] = []; + for (const next of frames) { + const result = nextTurnDetailAnnouncement(memory, { ...next, level, tailId: null }); + memory = result.memory; + spoken.push(...result.messages); + if (result.activity !== null) spoken.push(result.activity); + } + return spoken; +} + +describe("nextTurnDetailAnnouncement", () => { + const turn = [ + frame(null), + frame("Running git", [said("first", false)]), + frame("Running git", [said("first", false)]), + frame("Reading a.ts", [said("first"), said("second", false)]), + frame(null, [said("first"), said("second")]), + ]; + + it.each<{ level: ScreenReaderAnnouncements; spoken: ReadonlyArray }>([ + { level: "status", spoken: [] }, + { level: "messages", spoken: ["T3 Code: first", "T3 Code: second"] }, + { + level: "everything", + spoken: ["Running git", "T3 Code: first", "Reading a.ts", "T3 Code: second"], + }, + ])("says each message and step once at the $level level", ({ level, spoken }) => { + expect(speak(level, turn)).toEqual(spoken); + }); + + it("stays silent about what was there when the thread opened", () => { + expect( + speak("everything", [ + frame("Running git", [said("first")]), + frame("Running git", [said("first")]), + frame("Reading a.ts", [said("first"), said("second", true, true)]), + ]), + ).toEqual(["T3 Code: second", "Reading a.ts"]); + }); + + it("stays silent about history loaded above the messages it knows", () => { + expect( + speak("messages", [ + frame(null, [said("recent")]), + frame(null, [said("older"), said("recent")]), + frame(null, [said("older"), said("recent")]), + ]), + ).toEqual([]); + }); + + it("reads Markdown as the text it renders and skips a message with none", () => { + const first = { + ...said("first", true, true), + text: "**Tests** pass in `web`.\n\n- See [the docs](https://x.test/a(b)c?token=secret)
", + }; + expect( + speak("messages", [ + frame(null), + frame(null, [first, { ...said("empty", true, true), text: "" }]), + ]), + ).toEqual(["T3 Code: Tests pass in web. See the docs"]); + }); + + it("says a step again after waiting on the user", () => { + expect( + speak("everything", [ + frame(null), + frame("Running git"), + frame("Running git", [], "approval"), + frame("Running git"), + ]), + ).toEqual(["Running git", "Running git"]); + }); + + it("stays silent about a turn that already ended", () => { + expect( + speak("everything", [ + frame(null, [], "idle"), + frame("Running git", [said("first", true, true)], "idle"), + frame("Running git", [said("first")]), + ]), + ).toEqual(["Running git"]); + }); +}); diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.ts b/apps/web/src/components/chat/turnDetailAnnouncement.ts new file mode 100644 index 0000000000000..0728674ad8744 --- /dev/null +++ b/apps/web/src/components/chat/turnDetailAnnouncement.ts @@ -0,0 +1,202 @@ +import type { RunId, ScreenReaderAnnouncements } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; +import { + liveWorkEntryLabel, + timelineEntryRunId, + timelineEntryStartsResponse, +} from "./MessagesTimeline.logic"; +import { markdownPlainText } from "./timelineMinimapItems"; + +export interface TurnMessage { + readonly id: string; + /** Markdown. */ + readonly text: string; + /** The agent stopped writing it and went on working. A turn's closing reply never is. */ + readonly finished: boolean; + /** It arrived after the entry the caller last saw, unlike history loaded above it. */ + readonly fresh: boolean; +} + +export interface TurnDetail { + /** The step the agent is on, or null when its newest output is not a step. */ + readonly activity: string | null; + /** The agent's messages in the latest turn, oldest first. */ + readonly messages: ReadonlyArray; + /** The thread's newest entry. */ + readonly tailId: string | null; +} + +export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [], tailId: null }; + +const ACTIVITY_MAX_LENGTH = 120; +// Parsing is synchronous, and nobody listens to a longer message through a live region. +const MESSAGE_MAX_LENGTH = 2_000; + +// On screen these labels are a path or a query beside an icon. +function withVerb(work: WorkLogEntry, label: string) { + if (work.itemType === "file_change") return `Editing ${label}`; + if (work.itemType !== "file_search" && work.itemType !== "web_search") return label; + if (/^search(?:ed|ing)?\s/i.test(label)) return label; + return `${work.itemType === "web_search" ? "Searching the web for" : "Searching for"} ${label}`; +} + +type MessageEntry = Extract; + +// A steer starts a new attempt in the same run. The reply it cut off also stops streaming, +// so only one that completed before the steer finished; see completeSupersededReplyEntryIds. +function completedBeforeSuperseded(entry: MessageEntry) { + const item = entry.projectedItem?.item; + const supersededAt = entry.attempt?.completedAt ?? null; + return ( + item?.type === "assistant_message" && + item.status === "completed" && + item.completedAt !== null && + supersededAt !== null && + DateTime.isLessThan(item.completedAt, supersededAt) + ); +} + +function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): string | null { + // Reasoning text is never read out. + if (work.itemType === "reasoning") return "Thinking"; + const status = work.toolLifecycleStatus; + if (status === "failed" || status === "declined" || status === "stopped") return null; + const label = liveWorkEntryLabel(work, workspaceRoot, true).replace(/\s+/g, " ").trim(); + if (label === "") return null; + const spoken = withVerb(work, label); + return spoken.length > ACTIVITY_MAX_LENGTH ? `${spoken.slice(0, ACTIVITY_MAX_LENGTH)}…` : spoken; +} + +/** + * What a screen reader can be told about the latest turn beyond its status. + * It reads the thread's entries before "Tool calls in chat" hides any, so what + * is spoken follows the announcement setting alone. + */ +export function turnDetail( + entries: ReadonlyArray, + workspaceRoot: string | undefined, + knownTailId: string | null, +): TurnDetail { + const messages: TurnMessage[] = []; + // Runs and attempts the agent kept working in after the entry being looked at. + const continuedRunIds = new Set(); + const continuedAttemptIds = new Set(); + const markContinued = (entry: TimelineEntry, runId: RunId | null) => { + continuedRunIds.add(runId); + if (entry.attempt !== undefined) continuedAttemptIds.add(entry.attempt.id); + }; + let activity: string | null | undefined; + let fresh = true; + let index = entries.length - 1; + for (; index >= 0; index -= 1) { + const entry = entries[index]!; + if (timelineEntryStartsResponse(entry)) break; + if (entry.id === knownTailId) fresh = false; + if (entry.kind === "event") continue; + if (entry.kind === "work") { + const work = entry.entry; + if (work.sourceActivityKind === "context-compaction") { + activity ??= null; + continue; + } + // Notices, todo lists and checkpoints sit beside the turn's steps without being one. + if (work.tone !== "tool" && work.itemType !== "reasoning") continue; + markContinued(entry, work.runId ?? null); + // A request is spoken as the thread's status; once answered, the step it was for goes on. + if (work.itemType === "approval_request" || work.itemType === "user_input_request") continue; + if (activity === undefined) activity = stepLabel(work, workspaceRoot); + continue; + } + if (entry.kind === "message" && entry.message.role !== "assistant") continue; + // A reply, plan or rendered page is newer than any step before it. + activity ??= null; + const runId = timelineEntryRunId(entry); + if (entry.kind === "message") { + messages.push({ + id: entry.message.id, + text: entry.message.text, + finished: + !entry.message.streaming && + continuedRunIds.has(runId) && + (entry.attempt?.status !== "superseded" || + continuedAttemptIds.has(entry.attempt.id) || + completedBeforeSuperseded(entry)), + fresh, + }); + } + markContinued(entry, runId); + } + // Nothing is known to be new when the entry last seen is gone, or there was none. + let tailKnown = !fresh; + if (knownTailId !== null) { + for (; !tailKnown && index >= 0; index -= 1) tailKnown = entries[index]!.id === knownTailId; + } + return { + activity: activity ?? null, + messages: (tailKnown + ? messages + : messages.map((message) => ({ ...message, fresh: false })) + ).reverse(), + tailId: entries.at(-1)?.id ?? null, + }; +} + +export interface TurnDetailSnapshot extends TurnDetail { + readonly level: ScreenReaderAnnouncements; + readonly status: "working" | "approval" | "input" | "idle"; +} + +export interface TurnDetailMemory { + readonly activity: string | null; + /** Each message seen in the turn, and whether it was already finished then. */ + readonly messages: ReadonlyMap; + readonly tailId: string | null; +} + +/** The heading every agent message has in the timeline. */ +const MESSAGE_AUTHOR = "T3 Code"; + +/** + * Decides which of the turn's messages and steps a screen reader hears next. + * The first snapshot of a thread is a silent baseline, like its status. + */ +export function nextTurnDetailAnnouncement( + previous: TurnDetailMemory | null, + next: TurnDetailSnapshot, +): { + readonly memory: TurnDetailMemory; + readonly messages: ReadonlyArray; + readonly activity: string | null; +} { + const memory: TurnDetailMemory = { + // A step is spoken again after a pause for an approval or a question. + activity: next.status === "working" ? next.activity : null, + messages: new Map(next.messages.map((message) => [message.id, message.finished])), + tailId: next.tailId, + }; + // A turn that already ended is history, however late its entries arrive. + if (previous === null || next.level === "status" || next.status === "idle") { + return { memory, messages: [], activity: null }; + } + return { + memory, + messages: next.messages + .filter( + (message) => + message.finished && + previous.messages.get(message.id) !== true && + (message.fresh || previous.messages.has(message.id)), + ) + .map((message) => + message.text.length > MESSAGE_MAX_LENGTH + ? `${markdownPlainText(message.text.slice(0, MESSAGE_MAX_LENGTH))} More in the thread.` + : markdownPlainText(message.text), + ) + .filter((text) => text !== "") + .map((text) => `${MESSAGE_AUTHOR}: ${text}`), + activity: + next.level === "everything" && memory.activity !== previous.activity ? memory.activity : null, + }; +} diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index a37bf1581990e..3dc9566e8981d 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -22,6 +22,7 @@ import { type ResponseStreamingMode, type QuitConfirmationMode, SidebarProjectSortOrder, + ScreenReaderAnnouncements, ToolCallVisibility, } from "@t3tools/contracts/settings"; import { desktopUpdateChannelOf } from "@t3tools/shared/cliRelease"; @@ -539,6 +540,9 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.toolCallVisibility !== DEFAULT_UNIFIED_SETTINGS.toolCallVisibility ? ["Tool calls in chat"] : []), + ...(settings.screenReaderAnnouncements !== DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements + ? ["Screen reader announcements"] + : []), ...(settings.responseStreamingMode !== DEFAULT_UNIFIED_SETTINGS.responseStreamingMode ? ["Response streaming"] : []), @@ -654,6 +658,7 @@ export function useSettingsRestore(onRestored?: () => void) { sendShortcut: DEFAULT_UNIFIED_SETTINGS.sendShortcut, followUpBehavior: DEFAULT_UNIFIED_SETTINGS.followUpBehavior, toolCallVisibility: DEFAULT_UNIFIED_SETTINGS.toolCallVisibility, + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements, environmentIdentificationMode: DEFAULT_UNIFIED_SETTINGS.environmentIdentificationMode, glassOpacity: DEFAULT_UNIFIED_SETTINGS.glassOpacity, panelAnimationDurationMs: DEFAULT_UNIFIED_SETTINGS.panelAnimationDurationMs, @@ -1208,6 +1213,7 @@ export function ChatSettingsPanel() { + ); @@ -1262,6 +1268,65 @@ function ToolCallVisibilityRow() { ); } +const SCREEN_READER_ANNOUNCEMENTS_LABELS: Record = { + status: "Status only", + messages: "Status and messages", + everything: "Everything", +}; + +const isScreenReaderAnnouncements = Schema.is(ScreenReaderAnnouncements); + +function ScreenReaderAnnouncementsRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + + updateSettings({ + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements, + }) + } + /> + ) : null + } + control={ + + } + /> + ); +} + function DiffColorsSettingsRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index f7492035874da..4f61ba41c0d86 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -483,6 +483,14 @@ export const SETTINGS_SEARCH_ITEMS = [ "hide tool calls commands edits searches work log activity quiet chat zen final answers only", ], }, + { + id: "screen-reader-announcements", + title: "Screen reader announcements", + to: "/settings/chat", + searchTerms: [ + "accessibility a11y screen reader nvda jaws voiceover orca narrator speech announce read aloud messages activity steps", + ], + }, { id: "guest-links", title: "Guest links", diff --git a/docs/user/activity-log.md b/docs/user/activity-log.md index d4e6e7e32c940..a2a9278b80890 100644 --- a/docs/user/activity-log.md +++ b/docs/user/activity-log.md @@ -29,3 +29,15 @@ and the agent's last reply, which appears when the turn ends. If you steer while answering, the answer it finished before reading your steer stays too. On web and desktop, Find shows the in-between messages while it is open. Approvals, questions, and errors always show. The setting applies to this browser or device only. + +## Screen reader announcements + +On web and desktop, a screen reader hears the open thread's turn as it runs. **Screen reader +announcements** in **Settings → Chat** sets how much. **Status only** says when a turn starts, +needs your approval or input, and ends. **Status and messages** also reads each message the agent +writes and then keeps working after, once that message is complete. A very long message is cut +short. The reply that closes the turn is not read, so you can read it yourself when you hear +**Done**. **Everything** adds the step the agent is on, such as the command it is running. Steps +are spaced out, so a quick step can be skipped for a newer one. +The agent's reasoning text is never read. The setting follows the agent's real activity, whatever +**Tool calls in chat** hides, and applies to this browser or device only. diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd112d84bf9f4..346213992e4b2 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -346,6 +346,10 @@ export type EffortDisplay = typeof EffortDisplay.Type; export const ToolCallVisibility = Schema.Literals(["showAll", "hideSuccessful", "hideAll", "zen"]); export type ToolCallVisibility = typeof ToolCallVisibility.Type; +/** What the open thread reads to a screen reader: turn status, the agent's in-between messages too, or every step. */ +export const ScreenReaderAnnouncements = Schema.Literals(["status", "messages", "everything"]); +export type ScreenReaderAnnouncements = typeof ScreenReaderAnnouncements.Type; + export const ModelFavorite = Schema.Struct({ provider: ProviderInstanceId, model: TrimmedNonEmptyString, @@ -517,6 +521,9 @@ export const ClientSettingsSchema = Schema.Struct({ toolCallVisibility: ToolCallVisibility.pipe( Schema.withDecodingDefault(Effect.succeed("showAll" as const)), ), + screenReaderAnnouncements: ScreenReaderAnnouncements.pipe( + Schema.withDecodingDefault(Effect.succeed("status" as const)), + ), // Retired Labs toggle. Still read as the default for the Usage item in threadDetailsItems. threadDetailsUsageEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), // Sparse show/hide overrides for thread details card items. Missing ids use the client's @@ -2029,6 +2036,7 @@ export const ClientSettingsPatch = Schema.Struct({ boringSidebarEnabled: Schema.optionalKey(Schema.Boolean), boringSidebarMultilineRows: Schema.optionalKey(Schema.Boolean), toolCallVisibility: Schema.optionalKey(ToolCallVisibility), + screenReaderAnnouncements: Schema.optionalKey(ScreenReaderAnnouncements), threadDetailsUsageEnabled: Schema.optionalKey(Schema.Boolean), threadDetailsItems: Schema.optionalKey(Schema.Record(Schema.String, Schema.Boolean)), sidebarWorkingShelfEnabled: Schema.optionalKey(Schema.Boolean),