From ed5c20dc3562a14aef32c56c3269cee5d271badf Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:44:21 +0000 Subject: [PATCH 1/6] feat(web): screen readers can hear the agent's messages and steps during a turn A screen reader heard only a turn's status in the open thread: Working, Needs approval, Needs your input, Done. A new setting, Screen reader announcements in Settings > Chat, adds two more levels. "Status and messages" also reads each message the agent finishes before its last one. "Everything" adds the step the agent is on, at most one every two seconds, newest first. "Status only" is the default and is what the app did before. The announcer reads the thread's entries before "Tool calls in chat" hides any, so what is spoken follows this setting alone. Messages and steps get their own live region, so a status written right after a message does not replace it unheard. Co-Authored-By: Claude Fable 5.2 --- .../components/chat/MessagesTimeline.logic.ts | 4 +- .../src/components/chat/MessagesTimeline.tsx | 7 + .../ThreadActivityAnnouncer.browser.test.tsx | 114 ++++++++ .../chat/ThreadActivityAnnouncer.tsx | 148 +++++++++- .../chat/turnDetailAnnouncement.test.ts | 265 ++++++++++++++++++ .../components/chat/turnDetailAnnouncement.ts | 117 ++++++++ .../components/settings/SettingsPanels.tsx | 65 +++++ .../src/components/settings/settingsSearch.ts | 8 + docs/user/activity-log.md | 11 + .../src/work-log/presentation.ts | 11 +- packages/contracts/src/settings.ts | 8 + 11 files changed, 743 insertions(+), 15 deletions(-) create mode 100644 apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx create mode 100644 apps/web/src/components/chat/turnDetailAnnouncement.test.ts create mode 100644 apps/web/src/components/chat/turnDetailAnnouncement.ts diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index 94fbe68e29528..9e9d82a11fcbd 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -79,7 +79,7 @@ import { } from "@t3tools/shared/toolActivity"; import { isWindowsAbsolutePath } from "@t3tools/shared/path"; -function timelineEntryRunId(entry: TimelineEntry): RunId | null { +export function timelineEntryRunId(entry: TimelineEntry): RunId | null { if (entry.kind === "message") { return entry.message.role === "assistant" ? (entry.message.runId ?? null) : null; } @@ -917,7 +917,7 @@ function timelineEntryIsSteer(entry: TimelineEntry): boolean { ); } -function timelineEntryStartsResponse(entry: TimelineEntry): boolean { +export function timelineEntryStartsResponse(entry: TimelineEntry): boolean { return ( (entry.kind === "message" && entry.message.role === "user" && !timelineEntryIsSteer(entry)) || (entry.kind === "work" && entry.entry.itemType === "notification") diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 61d5c4d97e45e..a2b39ffd143f9 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -200,6 +200,7 @@ const TIMELINE_MAINTAIN_SCROLL_AT_END = { }, } as const satisfies MaintainScrollAtEndOptions; const EMPTY_RUN_IDS: ReadonlySet = new Set(); +const EMPTY_TIMELINE_ENTRIES: ReadonlyArray = []; const TIMELINE_MAINTAIN_SCROLL_AT_END_SMOOTH = { ...TIMELINE_MAINTAIN_SCROLL_AT_END, animated: true, @@ -552,6 +553,9 @@ const ConversationTimeline = memo(function ConversationTimeline({ } const preferredToolCallVisibility = useClientSettings((settings) => settings.toolCallVisibility); + const screenReaderAnnouncements = useClientSettings( + (settings) => settings.screenReaderAnnouncements, + ); const toolCallVisibility = findActive && preferredToolCallVisibility === "zen" ? "hideAll" : preferredToolCallVisibility; const revealedMessageId = citationRequest?.citation.messageId ?? null; @@ -1078,6 +1082,9 @@ const ConversationTimeline = memo(function ConversationTimeline({ runId={announcedRun?.runId ?? null} runStatus={announcedRun?.status ?? null} detailBehind={detailBehindShell} + level={screenReaderAnnouncements} + entries={screenReaderAnnouncements === "status" ? EMPTY_TIMELINE_ENTRIES : timelineEntries} + workspaceRoot={workspaceRoot} /> ); const listHeaderLeadingContent = diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx new file mode 100644 index 0000000000000..36bc3ea392454 --- /dev/null +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -0,0 +1,114 @@ +import { MessageId, RunId } from "@t3tools/contracts"; +import { type ComponentProps } from "react"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { act, fakeTimers, render } from "../../../test/render"; +import type { TimelineEntry } from "../../session-logic"; +import { ThreadActivityAnnouncer } from "./ThreadActivityAnnouncer"; + +const run = RunId.make("run-1"); +const at = "2026-01-01T00:00:00.000Z"; + +const message = (id: string, role: "user" | "assistant", streaming = false): TimelineEntry => ({ + kind: "message", + id, + createdAt: at, + message: { + id: MessageId.make(id), + role, + text: id, + runId: role === "assistant" ? run : null, + streaming, + createdAt: at, + updatedAt: at, + }, +}); +const command = (program: string): TimelineEntry => ({ + kind: "work", + id: program, + createdAt: at, + entry: { + id: program, + createdAt: at, + runId: run, + label: program, + tone: "tool", + itemType: "command_execution", + command: program, + toolLifecycleStatus: "inProgress", + }, +}); +const prompt = message("prompt", "user"); + +const announcer = (props: Partial>) => ( + +); + +const regions = (container: HTMLElement) => { + const [detail, status] = container.querySelectorAll('[aria-live="polite"]'); + return { detail: detail!, status: status! }; +}; + +describe("ThreadActivityAnnouncer", () => { + it("drops a waiting step once the agent has moved on", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + { + // Restored before the assertions, which poll on real timers. + using _timers = fakeTimers(); + await screen.rerender(announcer({})); + await screen.rerender(announcer({ entries: [prompt, command("git")] })); + await screen.rerender( + announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }), + ); + await act(() => vi.advanceTimersByTime(5_000)); + } + + await expect.element(status).toHaveTextContent("Working"); + await expect.element(detail).toBeEmptyDOMElement(); + }); + + it("keeps a message beside the status that follows it, then reads the newest step", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + const said = message("Checking the repo", "assistant"); + const heard: Array<[status: string, detail: string]> = []; + using _timers = fakeTimers(); + for (const props of [ + {}, + { entries: [prompt, said, command("git")], status: "approval", requestId: "request-1" }, + { entries: [prompt, said, command("git")] }, + { entries: [prompt, said, command("git"), command("rg")] }, + ] satisfies ReadonlyArray>>) { + await screen.rerender(announcer(props)); + heard.push([status.textContent, detail.textContent]); + } + await act(() => vi.advanceTimersByTime(2_000)); + heard.push([status.textContent, detail.textContent]); + + expect(heard).toEqual([ + ["Working", ""], + ["Needs approval", "T3 Code: Checking the repo"], + ["Needs approval", "T3 Code: Checking the repo"], + ["Needs approval", "T3 Code: Checking the repo"], + ["Needs approval", "Running rg"], + ]); + }); +}); diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 998b9e1522ad5..fea347aa0cc76 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -1,14 +1,46 @@ +import type { ScreenReaderAnnouncements } from "@t3tools/contracts"; import { memo, useEffect, useRef } from "react"; import { useLiveRegion } from "../../hooks/useLiveRegion"; +import type { TimelineEntry } from "../../session-logic"; import { nextThreadAnnouncement, type ThreadActivitySnapshot, type ThreadAnnouncementMemory, } from "./threadActivityAnnouncement"; +import { + nextTurnDetailAnnouncement, + NO_TURN_DETAIL, + turnDetail, + type TurnDetailMemory, +} from "./turnDetailAnnouncement"; + +// Long enough for a screen reader to finish a short label before the next one. +const ACTIVITY_GAP_MS = 2_000; +// A polite region queues what it can't say yet, so steps written while a +// message is still being read would all be heard late. They wait this long. +const MESSAGE_HOLD_MS_PER_CHARACTER = 50; +const MESSAGE_HOLD_MAX_MS = 15_000; + +function holdAfter(text: string) { + return Math.min( + MESSAGE_HOLD_MAX_MS, + Math.max(ACTIVITY_GAP_MS, text.length * MESSAGE_HOLD_MS_PER_CHARACTER), + ); +} + +/** One write, since a second write to a region replaces the first before it is heard. */ +function asOneAnnouncement(texts: ReadonlyArray) { + return texts + .map((text, index) => (index === texts.length - 1 || /[.!?:…]$/.test(text) ? text : `${text}.`)) + .join(" "); +} -/** One polite live region for the open thread. */ +/** + * The open thread's polite live regions. Status has its own, so "Needs + * approval" landing right after a message doesn't replace it unheard. + */ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ threadKey, status, @@ -16,35 +48,131 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ runId, runStatus, detailBehind, + level, + entries, + workspaceRoot, }: ThreadActivitySnapshot & { /** Null while the timeline shows another thread during a switch. */ readonly threadKey: string | null; /** See isDetailBehindShell. */ readonly detailBehind: boolean; + readonly level: ScreenReaderAnnouncements; + /** The thread's entries before any are hidden; unread at the "status" level. */ + readonly entries: ReadonlyArray; + readonly workspaceRoot: string | undefined; }) { const { regionRef, announce, clear } = useLiveRegion(); - const memoryRef = useRef<{ threadKey: string; memory: ThreadAnnouncementMemory } | null>(null); + const { + regionRef: detailRegionRef, + announce: announceDetail, + clear: clearDetail, + } = useLiveRegion(); + const memoryRef = useRef<{ + threadKey: string; + level: ScreenReaderAnnouncements; + memory: ThreadAnnouncementMemory; + detail: TurnDetailMemory; + } | null>(null); + // Messages are written at once; a step waits its turn, and a newer one replaces it. + const stepRef = useRef<{ + quietUntil: number; + waiting: { label: string; timer: ReturnType } | null; + }>({ quietUntil: 0, waiting: null }); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes thread status changes to the aria-live region after commit + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- writes the turn's status, messages and steps to the aria-live region after commit useEffect(() => { + const step = stepRef.current; + const write = (text: string, quietMs: number) => { + announceDetail(text); + step.quietUntil = Date.now() + quietMs; + }; + const takeWaiting = () => { + const label = step.waiting?.label ?? null; + if (step.waiting !== null) clearTimeout(step.waiting.timer); + step.waiting = null; + return label; + }; + if (threadKey === null) { + takeWaiting(); memoryRef.current = null; clear(); + clearDetail(); return; } if (detailBehind) return; - const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current.memory : null; - const { memory, announcement } = nextThreadAnnouncement(previous, { + const waitingLabel = takeWaiting(); + const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current : null; + const turn = nextThreadAnnouncement(previous?.memory ?? null, { status, requestId, runId, runStatus, }); - memoryRef.current = { threadKey, memory }; - if (previous === null) clear(); - else if (announcement !== null) announce(announcement); - }, [threadKey, status, requestId, runId, runStatus, detailBehind, announce, clear]); + // A new level starts from what the turn already holds, so raising it mid-turn replays nothing. + const previousDetail = previous?.level === level ? previous.detail : null; + const detail = nextTurnDetailAnnouncement(previousDetail, { + ...(level === "status" ? NO_TURN_DETAIL : turnDetail(entries, workspaceRoot)), + level, + status, + }); + memoryRef.current = { threadKey, level, memory: turn.memory, detail: detail.memory }; + if (previous === null) { + step.quietUntil = 0; + clear(); + clearDetail(); + return; + } + if (detail.messages.length > 0) { + const text = asOneAnnouncement(detail.messages); + write(text, holdAfter(text)); + } + if (turn.announcement !== null) { + announce(turn.announcement); + step.quietUntil = Math.max(step.quietUntil, Date.now() + ACTIVITY_GAP_MS); + } + // A waiting label is stale once its step is over or the level no longer reads steps. + const label = + level !== "everything" || detail.memory.activity === null + ? null + : (detail.activity ?? waitingLabel); + if (label === null) return; + const speakStep = () => { + step.waiting = null; + write(label, ACTIVITY_GAP_MS); + }; + const wait = step.quietUntil - Date.now(); + if (wait <= 0) speakStep(); + else step.waiting = { label, timer: setTimeout(speakStep, wait) }; + }, [ + threadKey, + status, + requestId, + runId, + runStatus, + detailBehind, + level, + entries, + workspaceRoot, + announce, + clear, + announceDetail, + clearDetail, + ]); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- drops a step still waiting to be spoken when the timeline unmounts + useEffect( + () => () => { + if (stepRef.current.waiting !== null) clearTimeout(stepRef.current.waiting.timer); + }, + [], + ); // No role="status": Orca 50 maps it to a status bar in Chromium and drops its updates. - return
; + return ( + <> +
+
+ + ); }); diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts new file mode 100644 index 0000000000000..18a46a39a7955 --- /dev/null +++ b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts @@ -0,0 +1,265 @@ +import { MessageId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; +import { + nextTurnDetailAnnouncement, + turnDetail, + type TurnDetail, + type TurnDetailMemory, + type TurnDetailSnapshot, +} from "./turnDetailAnnouncement"; + +const run = RunId.make("run-1"); +const at = "2026-01-01T00:00:00.000Z"; + +function message( + id: string, + role: "user" | "assistant", + fields: Partial["message"]> = {}, +): TimelineEntry { + return { + kind: "message", + id, + createdAt: at, + message: { + id: MessageId.make(id), + role, + text: id, + runId: role === "assistant" ? run : null, + streaming: false, + createdAt: at, + updatedAt: at, + ...fields, + }, + }; +} + +function work(id: string, fields: Partial = {}): TimelineEntry { + return { + kind: "work", + id, + createdAt: at, + entry: { id, createdAt: at, runId: run, label: id, tone: "tool", ...fields }, + }; +} + +const prompt = message("prompt", "user"); +const command = (id: string, fields: Partial = {}) => + work(id, { + itemType: "command_execution", + command: "git status", + toolLifecycleStatus: "inProgress", + ...fields, + }); +const notice = work("notice", { itemType: "system_notice", tone: "info", label: "Heads up" }); + +describe("turnDetail", () => { + it.each<{ + name: string; + entries: ReadonlyArray; + activity: string | null; + }>([ + { name: "a running command", entries: [prompt, command("git")], activity: "Running git" }, + { + name: "a command that already finished, until something follows it", + entries: [prompt, command("git", { toolLifecycleStatus: "completed" })], + activity: "Running git", + }, + { + name: "reasoning, without its text", + entries: [ + prompt, + work("thought", { itemType: "reasoning", tone: "thinking", detail: "Private idea." }), + ], + activity: "Thinking", + }, + { + name: "a file change, with the verb its icon stands for", + entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["src/a.ts"] })], + activity: "Editing src/a.ts", + }, + { + name: "the step under a notice", + entries: [prompt, command("git"), notice], + activity: "Running git", + }, + { + name: "nothing for a failed step", + entries: [prompt, command("git", { toolLifecycleStatus: "failed" })], + activity: null, + }, + { + name: "the step a request was about", + entries: [prompt, command("git"), work("ask", { itemType: "approval_request" })], + activity: "Running git", + }, + { + name: "nothing once the agent is writing again", + entries: [prompt, command("git"), message("reply", "assistant", { streaming: true })], + activity: null, + }, + { + name: "nothing from the turn before", + entries: [message("earlier", "user"), command("git"), prompt], + activity: null, + }, + ])("names $name", ({ entries, activity }) => { + expect(turnDetail(entries, undefined).activity).toBe(activity); + }); + + it.each<{ + name: string; + entries: ReadonlyArray; + messages: ReadonlyArray; + }>([ + { + name: "a message the agent kept working after, but not its last", + entries: [ + prompt, + message("first", "assistant"), + command("git"), + message("last", "assistant"), + ], + messages: ["first"], + }, + { + name: "a message once the next one starts", + entries: [ + prompt, + message("first", "assistant"), + message("second", "assistant", { streaming: true }), + ], + messages: ["first"], + }, + { + name: "a message across a steer", + entries: [ + prompt, + message("first", "assistant"), + message("steer", "user", { inputIntent: "steer" }), + command("git"), + ], + messages: ["first"], + }, + { + name: "nothing while the message is still streaming", + entries: [prompt, message("first", "assistant", { streaming: true }), command("git")], + messages: [], + }, + { + name: "nothing for a last message that only a notice follows", + entries: [prompt, command("git"), message("last", "assistant"), notice], + messages: [], + }, + { + name: "nothing for a last message when another run picks the thread up", + entries: [ + prompt, + message("last", "assistant"), + command("git", { runId: RunId.make("run-2") }), + ], + messages: [], + }, + { + name: "nothing from the turn before", + entries: [message("earlier", "user"), message("first", "assistant"), command("git"), prompt], + messages: [], + }, + ])("reads $name", ({ entries, messages }) => { + expect(turnDetail(entries, undefined).messages.map((spoken) => spoken.text)).toEqual(messages); + }); +}); + +const working = ( + activity: string | null, + messages: ReadonlyArray = [], + status: TurnDetailSnapshot["status"] = "working", +): TurnDetail & Pick => ({ + status, + activity, + messages: messages.map((id) => ({ id, text: id })), +}); + +function speak( + level: ScreenReaderAnnouncements, + snapshots: ReadonlyArray>, +) { + let memory: TurnDetailMemory | null = null; + const spoken: string[] = []; + for (const snapshot of snapshots) { + const result = nextTurnDetailAnnouncement(memory, { ...snapshot, level }); + memory = result.memory; + spoken.push(...result.messages); + if (result.activity !== null) spoken.push(result.activity); + } + return spoken; +} + +describe("nextTurnDetailAnnouncement", () => { + const turn = [ + working(null), + working("Running git"), + working("Running git"), + working("Reading a.ts", ["first"]), + working(null, ["first", "second"]), + ]; + + it.each<{ level: ScreenReaderAnnouncements; spoken: ReadonlyArray }>([ + { level: "status", spoken: [] }, + { level: "messages", spoken: ["T3 Code: first", "T3 Code: second"] }, + { + level: "everything", + spoken: ["Running git", "T3 Code: first", "Reading a.ts", "T3 Code: second"], + }, + ])("says each message and step once at the $level level", ({ level, spoken }) => { + expect(speak(level, turn)).toEqual(spoken); + }); + + it("stays silent about what was there when the thread opened", () => { + expect( + speak("everything", [ + working("Running git", ["first"]), + working("Running git", ["first"]), + working("Reading a.ts", ["first", "second"]), + ]), + ).toEqual(["T3 Code: second", "Reading a.ts"]); + }); + + it("reads Markdown as plain text and skips a message with no text", () => { + const message = (id: string, text: string) => ({ id, text }); + const first = message("first", "**Tests** pass in `web`.\n\n- See [docs](https://x)"); + expect( + nextTurnDetailAnnouncement( + { activity: null, spokenMessageIds: new Set() }, + { + level: "messages", + status: "working", + activity: null, + messages: [first, message("empty", "``")], + }, + ).messages, + ).toEqual(["T3 Code: Tests pass in web. See docs"]); + }); + + it("says a step again after waiting on the user", () => { + expect( + speak("everything", [ + working(null), + working("Running git"), + working("Running git", [], "approval"), + working("Running git"), + ]), + ).toEqual(["Running git", "Running git"]); + }); + + it("stays silent about a turn that already ended", () => { + expect( + speak("everything", [ + working(null, [], "idle"), + working("Running git", ["first"], "idle"), + working("Running git", ["first"]), + ]), + ).toEqual(["Running git"]); + }); +}); diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.ts b/apps/web/src/components/chat/turnDetailAnnouncement.ts new file mode 100644 index 0000000000000..38172187fb2a8 --- /dev/null +++ b/apps/web/src/components/chat/turnDetailAnnouncement.ts @@ -0,0 +1,117 @@ +import { markdownPlainText } from "@t3tools/client-runtime/work-log/presentation"; +import type { RunId, ScreenReaderAnnouncements } from "@t3tools/contracts"; + +import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; +import { + liveWorkEntryLabel, + timelineEntryRunId, + timelineEntryStartsResponse, +} from "./MessagesTimeline.logic"; + +export interface TurnDetail { + /** The step the agent is on, or null when its newest output is not a step. */ + readonly activity: string | null; + /** Messages the agent finished and then kept working after, oldest first, as Markdown. */ + readonly messages: ReadonlyArray<{ readonly id: string; readonly text: string }>; +} + +export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [] }; + +const ACTIVITY_MAX_LENGTH = 120; + +function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): string | null { + // Reasoning text is never read out. + if (work.itemType === "reasoning") return "Thinking"; + const status = work.toolLifecycleStatus; + if (status === "failed" || status === "declined" || status === "stopped") return null; + const label = liveWorkEntryLabel(work, workspaceRoot, true).replace(/\s+/g, " ").trim(); + if (label === "") return null; + // On screen a file change is its path beside an icon. + const spoken = work.itemType === "file_change" ? `Editing ${label}` : label; + return spoken.length > ACTIVITY_MAX_LENGTH ? `${spoken.slice(0, ACTIVITY_MAX_LENGTH)}…` : spoken; +} + +/** + * What a screen reader can be told about the latest turn beyond its status. + * It reads the thread's entries before "Tool calls in chat" hides any, so what + * is spoken follows the announcement setting alone. + */ +export function turnDetail( + entries: ReadonlyArray, + workspaceRoot: string | undefined, +): TurnDetail { + const messages: Array = []; + // Runs the agent kept working in after the entry being looked at. + const continuedRunIds = new Set(); + let activity: string | null | undefined; + for (let index = entries.length - 1; index >= 0; index -= 1) { + const entry = entries[index]!; + if (timelineEntryStartsResponse(entry)) break; + if (entry.kind === "event") continue; + if (entry.kind === "work") { + const work = entry.entry; + // Notices, todo lists and checkpoints sit beside the turn's steps without being one. + if (work.tone !== "tool" && work.itemType !== "reasoning") continue; + continuedRunIds.add(work.runId ?? null); + // A request is spoken as the thread's status; once answered, the step it was for goes on. + if (work.itemType === "approval_request" || work.itemType === "user_input_request") continue; + if (activity === undefined) activity = stepLabel(work, workspaceRoot); + continue; + } + if (entry.kind === "message" && entry.message.role !== "assistant") continue; + // A reply, plan or rendered page is newer than any step before it. + activity ??= null; + const runId = timelineEntryRunId(entry); + if (entry.kind === "message" && !entry.message.streaming && continuedRunIds.has(runId)) { + messages.push({ id: entry.message.id, text: entry.message.text }); + } + continuedRunIds.add(runId); + } + return { activity: activity ?? null, messages: messages.reverse() }; +} + +export interface TurnDetailSnapshot extends TurnDetail { + readonly level: ScreenReaderAnnouncements; + readonly status: "working" | "approval" | "input" | "idle"; +} + +export interface TurnDetailMemory { + readonly activity: string | null; + readonly spokenMessageIds: ReadonlySet; +} + +/** The heading every agent message has in the timeline. */ +const MESSAGE_AUTHOR = "T3 Code"; + +/** + * Decides which of the turn's messages and steps a screen reader hears next. + * The first snapshot of a thread is a silent baseline, like its status. + */ +export function nextTurnDetailAnnouncement( + previous: TurnDetailMemory | null, + next: TurnDetailSnapshot, +): { + readonly memory: TurnDetailMemory; + readonly messages: ReadonlyArray; + readonly activity: string | null; +} { + const memory: TurnDetailMemory = { + // A step is spoken again after a pause for an approval or a question. + activity: next.status === "working" ? next.activity : null, + spokenMessageIds: new Set(next.messages.map((message) => message.id)), + }; + // A turn that already ended is history, however late its entries arrive. + if (previous === null || next.level === "status" || next.status === "idle") { + return { memory, messages: [], activity: null }; + } + return { + memory, + messages: next.messages + .filter((message) => !previous.spokenMessageIds.has(message.id)) + .map((message) => markdownPlainText(message.text)) + .filter((text) => text !== "") + .map((text) => `${MESSAGE_AUTHOR}: ${text}`), + activity: + next.level === "everything" && memory.activity !== previous.activity ? memory.activity : null, + }; +} diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index a37bf1581990e..3dc9566e8981d 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -22,6 +22,7 @@ import { type ResponseStreamingMode, type QuitConfirmationMode, SidebarProjectSortOrder, + ScreenReaderAnnouncements, ToolCallVisibility, } from "@t3tools/contracts/settings"; import { desktopUpdateChannelOf } from "@t3tools/shared/cliRelease"; @@ -539,6 +540,9 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.toolCallVisibility !== DEFAULT_UNIFIED_SETTINGS.toolCallVisibility ? ["Tool calls in chat"] : []), + ...(settings.screenReaderAnnouncements !== DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements + ? ["Screen reader announcements"] + : []), ...(settings.responseStreamingMode !== DEFAULT_UNIFIED_SETTINGS.responseStreamingMode ? ["Response streaming"] : []), @@ -654,6 +658,7 @@ export function useSettingsRestore(onRestored?: () => void) { sendShortcut: DEFAULT_UNIFIED_SETTINGS.sendShortcut, followUpBehavior: DEFAULT_UNIFIED_SETTINGS.followUpBehavior, toolCallVisibility: DEFAULT_UNIFIED_SETTINGS.toolCallVisibility, + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements, environmentIdentificationMode: DEFAULT_UNIFIED_SETTINGS.environmentIdentificationMode, glassOpacity: DEFAULT_UNIFIED_SETTINGS.glassOpacity, panelAnimationDurationMs: DEFAULT_UNIFIED_SETTINGS.panelAnimationDurationMs, @@ -1208,6 +1213,7 @@ export function ChatSettingsPanel() { + ); @@ -1262,6 +1268,65 @@ function ToolCallVisibilityRow() { ); } +const SCREEN_READER_ANNOUNCEMENTS_LABELS: Record = { + status: "Status only", + messages: "Status and messages", + everything: "Everything", +}; + +const isScreenReaderAnnouncements = Schema.is(ScreenReaderAnnouncements); + +function ScreenReaderAnnouncementsRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + + updateSettings({ + screenReaderAnnouncements: DEFAULT_UNIFIED_SETTINGS.screenReaderAnnouncements, + }) + } + /> + ) : null + } + control={ + + } + /> + ); +} + function DiffColorsSettingsRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index f7492035874da..4f61ba41c0d86 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -483,6 +483,14 @@ export const SETTINGS_SEARCH_ITEMS = [ "hide tool calls commands edits searches work log activity quiet chat zen final answers only", ], }, + { + id: "screen-reader-announcements", + title: "Screen reader announcements", + to: "/settings/chat", + searchTerms: [ + "accessibility a11y screen reader nvda jaws voiceover orca narrator speech announce read aloud messages activity steps", + ], + }, { id: "guest-links", title: "Guest links", diff --git a/docs/user/activity-log.md b/docs/user/activity-log.md index d4e6e7e32c940..e3264898c1af6 100644 --- a/docs/user/activity-log.md +++ b/docs/user/activity-log.md @@ -29,3 +29,14 @@ and the agent's last reply, which appears when the turn ends. If you steer while answering, the answer it finished before reading your steer stays too. On web and desktop, Find shows the in-between messages while it is open. Approvals, questions, and errors always show. The setting applies to this browser or device only. + +## Screen reader announcements + +On web and desktop, a screen reader hears the open thread's turn as it runs. **Screen reader +announcements** in **Settings → Chat** sets how much. **Status only** says when a turn starts, +needs your approval or input, and ends. **Status and messages** also reads each message the agent +writes before its last one, once that message is complete. The last message is not read, so you +can read it yourself when you hear **Done**. **Everything** adds the step the agent is on, such as +the command it is running. Steps are spaced out, so a quick step can be skipped for a newer one. +The agent's reasoning text is never read. The setting follows the agent's real activity, whatever +**Tool calls in chat** hides, and applies to this browser or device only. diff --git a/packages/client-runtime/src/work-log/presentation.ts b/packages/client-runtime/src/work-log/presentation.ts index e2097ed4a46c4..8b44f88dce5b0 100644 --- a/packages/client-runtime/src/work-log/presentation.ts +++ b/packages/client-runtime/src/work-log/presentation.ts @@ -678,15 +678,20 @@ export function toolGroupSummaryKind( return fallbackKinds.size === 1 ? fallbackKinds.values().next().value! : "mixed"; } -export function liveThoughtLine(markdown: string): string { - const heading = /^\s*\*\*([^*\r\n]+)\*\*[ \t]*\r?(?:\n|$)/.exec(markdown)?.[1]; - const text = (heading ?? markdown) +/** Markdown as one line of plain text, for places that can't render it. */ +export function markdownPlainText(markdown: string): string { + return markdown .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") .replace(/^[ \t]*(?:#{1,6}|[-*+]|\d+\.)[ \t]+/gm, "") .replace(/`+|\*\*|~~/g, "") .replace(/(^|[^\w*])[*_]([^*_\n]+)[*_](?![\w*])/g, "$1$2") .replace(/\s+/g, " ") .trim(); +} + +export function liveThoughtLine(markdown: string): string { + const heading = /^\s*\*\*([^*\r\n]+)\*\*[ \t]*\r?(?:\n|$)/.exec(markdown)?.[1]; + const text = markdownPlainText(heading ?? markdown); if (heading !== undefined) return text; const end = /[.?!]["'”’)]?(?=\s)/.exec(text); return end ? text.slice(0, end.index + end[0].length) : text; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd112d84bf9f4..346213992e4b2 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -346,6 +346,10 @@ export type EffortDisplay = typeof EffortDisplay.Type; export const ToolCallVisibility = Schema.Literals(["showAll", "hideSuccessful", "hideAll", "zen"]); export type ToolCallVisibility = typeof ToolCallVisibility.Type; +/** What the open thread reads to a screen reader: turn status, the agent's in-between messages too, or every step. */ +export const ScreenReaderAnnouncements = Schema.Literals(["status", "messages", "everything"]); +export type ScreenReaderAnnouncements = typeof ScreenReaderAnnouncements.Type; + export const ModelFavorite = Schema.Struct({ provider: ProviderInstanceId, model: TrimmedNonEmptyString, @@ -517,6 +521,9 @@ export const ClientSettingsSchema = Schema.Struct({ toolCallVisibility: ToolCallVisibility.pipe( Schema.withDecodingDefault(Effect.succeed("showAll" as const)), ), + screenReaderAnnouncements: ScreenReaderAnnouncements.pipe( + Schema.withDecodingDefault(Effect.succeed("status" as const)), + ), // Retired Labs toggle. Still read as the default for the Usage item in threadDetailsItems. threadDetailsUsageEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), // Sparse show/hide overrides for thread details card items. Missing ids use the client's @@ -2029,6 +2036,7 @@ export const ClientSettingsPatch = Schema.Struct({ boringSidebarEnabled: Schema.optionalKey(Schema.Boolean), boringSidebarMultilineRows: Schema.optionalKey(Schema.Boolean), toolCallVisibility: Schema.optionalKey(ToolCallVisibility), + screenReaderAnnouncements: Schema.optionalKey(ScreenReaderAnnouncements), threadDetailsUsageEnabled: Schema.optionalKey(Schema.Boolean), threadDetailsItems: Schema.optionalKey(Schema.Record(Schema.String, Schema.Boolean)), sidebarWorkingShelfEnabled: Schema.optionalKey(Schema.Boolean), From ffa5fbf57342f6949678afe6613459952abc1ae9 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:00:19 +0000 Subject: [PATCH 2/6] fix(web): turn announcements skip history, cut-off replies and stale steps Review found cases where the new announcements said the wrong thing: - History loaded above a long turn was read as new messages. A message is now read only if it was seen unfinished before or arrived after the entry last seen. - A reply a steer cut off counted as finished. It counts only if its own attempt went on. - A step waiting to be spoken survived the thread's detail falling behind, and stayed current through a compaction. - A web or file search was spoken as its bare query. It gets a verb, like a file change. - Message text came from a regex that could leak part of a link target. It now comes from the Markdown parser the minimap preview already uses. Co-Authored-By: Claude Fable 5.2 --- .../ThreadActivityAnnouncer.browser.test.tsx | 15 +- .../chat/ThreadActivityAnnouncer.tsx | 6 +- .../components/chat/timelineMinimapItems.ts | 5 + .../chat/turnDetailAnnouncement.test.ts | 199 ++++++++++++------ .../components/chat/turnDetailAnnouncement.ts | 85 ++++++-- .../src/work-log/presentation.ts | 11 +- 6 files changed, 221 insertions(+), 100 deletions(-) diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx index 36bc3ea392454..fd68f3f10e014 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -61,7 +61,16 @@ const regions = (container: HTMLElement) => { }; describe("ThreadActivityAnnouncer", () => { - it("drops a waiting step once the agent has moved on", async () => { + it.each<{ name: string; props: Partial> }>([ + { + name: "the agent has moved on", + props: { entries: [prompt, command("git"), message("reply", "assistant", true)] }, + }, + { + name: "the thread's detail falls behind", + props: { entries: [prompt, command("git")], detailBehind: true }, + }, + ])("drops a waiting step once $name", async ({ props }) => { // The announcer reads no route. await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { router: false, @@ -72,9 +81,7 @@ describe("ThreadActivityAnnouncer", () => { using _timers = fakeTimers(); await screen.rerender(announcer({})); await screen.rerender(announcer({ entries: [prompt, command("git")] })); - await screen.rerender( - announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }), - ); + await screen.rerender(announcer(props)); await act(() => vi.advanceTimersByTime(5_000)); } diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index fea347aa0cc76..1f0d7458681b1 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -100,8 +100,8 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ clearDetail(); return; } - if (detailBehind) return; const waitingLabel = takeWaiting(); + if (detailBehind) return; const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current : null; const turn = nextThreadAnnouncement(previous?.memory ?? null, { status, @@ -112,7 +112,9 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ // A new level starts from what the turn already holds, so raising it mid-turn replays nothing. const previousDetail = previous?.level === level ? previous.detail : null; const detail = nextTurnDetailAnnouncement(previousDetail, { - ...(level === "status" ? NO_TURN_DETAIL : turnDetail(entries, workspaceRoot)), + ...(level === "status" + ? NO_TURN_DETAIL + : turnDetail(entries, workspaceRoot, previousDetail?.tailId ?? null)), level, status, }); diff --git a/apps/web/src/components/chat/timelineMinimapItems.ts b/apps/web/src/components/chat/timelineMinimapItems.ts index 3c730df680e54..386abbaa013a3 100644 --- a/apps/web/src/components/chat/timelineMinimapItems.ts +++ b/apps/web/src/components/chat/timelineMinimapItems.ts @@ -89,6 +89,11 @@ function markdownNodeText(node: Nodes): string { return node.type === "link" && parseThreadLinkHref(text) !== null ? "a thread" : text; } +/** Markdown as one line of the text it renders, without markup or link targets. */ +export function markdownPlainText(markdown: string): string { + return markdownNodeText(markdownPreviewParser.parse(markdown)).replace(/\s+/g, " ").trim(); +} + function compactMinimapPreview(text: string | null | undefined) { const compact = text?.replace(/\s+/g, " ").trim() ?? ""; return compact.length > 0 ? compact : null; diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts index 18a46a39a7955..9a4901f73c3ef 100644 --- a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts +++ b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts @@ -1,13 +1,13 @@ -import { MessageId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts"; +import { MessageId, RunAttemptId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; +import type { TimelineAttempt, TimelineEntry, WorkLogEntry } from "../../session-logic"; import { nextTurnDetailAnnouncement, turnDetail, - type TurnDetail, type TurnDetailMemory, type TurnDetailSnapshot, + type TurnMessage, } from "./turnDetailAnnouncement"; const run = RunId.make("run-1"); @@ -53,6 +53,22 @@ const command = (id: string, fields: Partial = {}) => ...fields, }); const notice = work("notice", { itemType: "system_notice", tone: "info", label: "Heads up" }); +const steer = message("steer", "user", { inputIntent: "steer" }); +const inAttempt = ( + entry: TimelineEntry, + id: string, + status: TimelineAttempt["status"], +): TimelineEntry => ({ + ...entry, + attempt: { + id: RunAttemptId.make(id), + runId: run, + attemptOrdinal: 1, + rootNodeId: `${id}-root` as TimelineAttempt["rootNodeId"], + status, + completedAt: null, + }, +}); describe("turnDetail", () => { it.each<{ @@ -79,39 +95,53 @@ describe("turnDetail", () => { entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["src/a.ts"] })], activity: "Editing src/a.ts", }, + { + name: "a web search, with the verb its icon stands for", + entries: [prompt, work("search", { itemType: "web_search", detail: "vitest fake timers" })], + activity: "Searching the web for vitest fake timers", + }, { name: "the step under a notice", entries: [prompt, command("git"), notice], activity: "Running git", }, - { - name: "nothing for a failed step", - entries: [prompt, command("git", { toolLifecycleStatus: "failed" })], - activity: null, - }, { name: "the step a request was about", entries: [prompt, command("git"), work("ask", { itemType: "approval_request" })], activity: "Running git", }, + { + name: "nothing for a failed step", + entries: [prompt, command("git", { toolLifecycleStatus: "failed" })], + activity: null, + }, { name: "nothing once the agent is writing again", entries: [prompt, command("git"), message("reply", "assistant", { streaming: true })], activity: null, }, + { + name: "nothing while the context is compacted", + entries: [ + prompt, + command("git"), + work("compact", { tone: "info", sourceActivityKind: "context-compaction" }), + ], + activity: null, + }, { name: "nothing from the turn before", entries: [message("earlier", "user"), command("git"), prompt], activity: null, }, ])("names $name", ({ entries, activity }) => { - expect(turnDetail(entries, undefined).activity).toBe(activity); + expect(turnDetail(entries, undefined, null).activity).toBe(activity); }); it.each<{ name: string; entries: ReadonlyArray; - messages: ReadonlyArray; + finished: ReadonlyArray; }>([ { name: "a message the agent kept working after, but not its last", @@ -121,7 +151,7 @@ describe("turnDetail", () => { command("git"), message("last", "assistant"), ], - messages: ["first"], + finished: ["first"], }, { name: "a message once the next one starts", @@ -130,65 +160,87 @@ describe("turnDetail", () => { message("first", "assistant"), message("second", "assistant", { streaming: true }), ], - messages: ["first"], + finished: ["first"], }, { name: "a message across a steer", - entries: [ - prompt, - message("first", "assistant"), - message("steer", "user", { inputIntent: "steer" }), - command("git"), - ], - messages: ["first"], + entries: [prompt, message("first", "assistant"), steer, command("git")], + finished: ["first"], }, { - name: "nothing while the message is still streaming", + name: "no message that is still streaming", entries: [prompt, message("first", "assistant", { streaming: true }), command("git")], - messages: [], + finished: [], + }, + { + name: "no reply a steer cut off", + entries: [ + prompt, + inAttempt(message("cut off", "assistant"), "attempt-1", "superseded"), + steer, + inAttempt(command("git"), "attempt-2", "running"), + ], + finished: [], }, { - name: "nothing for a last message that only a notice follows", + name: "no last message that only a notice follows", entries: [prompt, command("git"), message("last", "assistant"), notice], - messages: [], + finished: [], }, { - name: "nothing for a last message when another run picks the thread up", + name: "no last message when another run picks the thread up", entries: [ prompt, message("last", "assistant"), command("git", { runId: RunId.make("run-2") }), ], - messages: [], + finished: [], }, { - name: "nothing from the turn before", + name: "no message from the turn before", entries: [message("earlier", "user"), message("first", "assistant"), command("git"), prompt], - messages: [], + finished: [], }, - ])("reads $name", ({ entries, messages }) => { - expect(turnDetail(entries, undefined).messages.map((spoken) => spoken.text)).toEqual(messages); + ])("counts $name as finished", ({ entries, finished }) => { + const { messages } = turnDetail(entries, undefined, null); + expect(messages.filter((spoken) => spoken.finished).map((spoken) => spoken.text)).toEqual( + finished, + ); + }); + + it("marks only messages after the entry last seen as fresh", () => { + const entries = [ + prompt, + message("loaded", "assistant"), + command("seen"), + message("new", "assistant"), + command("git"), + ]; + expect(turnDetail(entries, undefined, "seen").messages.map((spoken) => spoken.fresh)).toEqual([ + false, + true, + ]); }); }); -const working = ( +type Frame = Pick; +const said = (id: string, finished = true, fresh = false): TurnMessage => ({ + id, + text: id, + finished, + fresh, +}); +const frame = ( activity: string | null, - messages: ReadonlyArray = [], + messages: ReadonlyArray = [], status: TurnDetailSnapshot["status"] = "working", -): TurnDetail & Pick => ({ - status, - activity, - messages: messages.map((id) => ({ id, text: id })), -}); +): Frame => ({ status, activity, messages }); -function speak( - level: ScreenReaderAnnouncements, - snapshots: ReadonlyArray>, -) { +function speak(level: ScreenReaderAnnouncements, frames: ReadonlyArray) { let memory: TurnDetailMemory | null = null; const spoken: string[] = []; - for (const snapshot of snapshots) { - const result = nextTurnDetailAnnouncement(memory, { ...snapshot, level }); + for (const next of frames) { + const result = nextTurnDetailAnnouncement(memory, { ...next, level, tailId: null }); memory = result.memory; spoken.push(...result.messages); if (result.activity !== null) spoken.push(result.activity); @@ -198,11 +250,11 @@ function speak( describe("nextTurnDetailAnnouncement", () => { const turn = [ - working(null), - working("Running git"), - working("Running git"), - working("Reading a.ts", ["first"]), - working(null, ["first", "second"]), + frame(null), + frame("Running git", [said("first", false)]), + frame("Running git", [said("first", false)]), + frame("Reading a.ts", [said("first"), said("second", false)]), + frame(null, [said("first"), said("second")]), ]; it.each<{ level: ScreenReaderAnnouncements; spoken: ReadonlyArray }>([ @@ -219,36 +271,43 @@ describe("nextTurnDetailAnnouncement", () => { it("stays silent about what was there when the thread opened", () => { expect( speak("everything", [ - working("Running git", ["first"]), - working("Running git", ["first"]), - working("Reading a.ts", ["first", "second"]), + frame("Running git", [said("first")]), + frame("Running git", [said("first")]), + frame("Reading a.ts", [said("first"), said("second", true, true)]), ]), ).toEqual(["T3 Code: second", "Reading a.ts"]); }); - it("reads Markdown as plain text and skips a message with no text", () => { - const message = (id: string, text: string) => ({ id, text }); - const first = message("first", "**Tests** pass in `web`.\n\n- See [docs](https://x)"); + it("stays silent about history loaded above the messages it knows", () => { expect( - nextTurnDetailAnnouncement( - { activity: null, spokenMessageIds: new Set() }, - { - level: "messages", - status: "working", - activity: null, - messages: [first, message("empty", "``")], - }, - ).messages, - ).toEqual(["T3 Code: Tests pass in web. See docs"]); + speak("messages", [ + frame(null, [said("recent")]), + frame(null, [said("older"), said("recent")]), + frame(null, [said("older"), said("recent")]), + ]), + ).toEqual([]); + }); + + it("reads Markdown as the text it renders and skips a message with none", () => { + const first = { + ...said("first", true, true), + text: "**Tests** pass in `web`.\n\n- See [the docs](https://x.test/a(b)c?token=secret)
", + }; + expect( + speak("messages", [ + frame(null), + frame(null, [first, { ...said("empty", true, true), text: "" }]), + ]), + ).toEqual(["T3 Code: Tests pass in web. See the docs"]); }); it("says a step again after waiting on the user", () => { expect( speak("everything", [ - working(null), - working("Running git"), - working("Running git", [], "approval"), - working("Running git"), + frame(null), + frame("Running git"), + frame("Running git", [], "approval"), + frame("Running git"), ]), ).toEqual(["Running git", "Running git"]); }); @@ -256,9 +315,9 @@ describe("nextTurnDetailAnnouncement", () => { it("stays silent about a turn that already ended", () => { expect( speak("everything", [ - working(null, [], "idle"), - working("Running git", ["first"], "idle"), - working("Running git", ["first"]), + frame(null, [], "idle"), + frame("Running git", [said("first", true, true)], "idle"), + frame("Running git", [said("first")]), ]), ).toEqual(["Running git"]); }); diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.ts b/apps/web/src/components/chat/turnDetailAnnouncement.ts index 38172187fb2a8..bf2cad8b1e14b 100644 --- a/apps/web/src/components/chat/turnDetailAnnouncement.ts +++ b/apps/web/src/components/chat/turnDetailAnnouncement.ts @@ -1,4 +1,3 @@ -import { markdownPlainText } from "@t3tools/client-runtime/work-log/presentation"; import type { RunId, ScreenReaderAnnouncements } from "@t3tools/contracts"; import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; @@ -7,18 +6,38 @@ import { timelineEntryRunId, timelineEntryStartsResponse, } from "./MessagesTimeline.logic"; +import { markdownPlainText } from "./timelineMinimapItems"; + +export interface TurnMessage { + readonly id: string; + /** Markdown. */ + readonly text: string; + /** The agent stopped writing it and went on working. A turn's closing reply never is. */ + readonly finished: boolean; + /** It arrived after the entry the caller last saw, unlike history loaded above it. */ + readonly fresh: boolean; +} export interface TurnDetail { /** The step the agent is on, or null when its newest output is not a step. */ readonly activity: string | null; - /** Messages the agent finished and then kept working after, oldest first, as Markdown. */ - readonly messages: ReadonlyArray<{ readonly id: string; readonly text: string }>; + /** The agent's messages in the latest turn, oldest first. */ + readonly messages: ReadonlyArray; + /** The thread's newest entry. */ + readonly tailId: string | null; } -export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [] }; +export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [], tailId: null }; const ACTIVITY_MAX_LENGTH = 120; +// On screen these labels are a path or a query beside an icon. +const SPOKEN_VERBS: Partial, string>> = { + file_change: "Editing", + file_search: "Searching for", + web_search: "Searching the web for", +}; + function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): string | null { // Reasoning text is never read out. if (work.itemType === "reasoning") return "Thinking"; @@ -26,8 +45,8 @@ function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): strin if (status === "failed" || status === "declined" || status === "stopped") return null; const label = liveWorkEntryLabel(work, workspaceRoot, true).replace(/\s+/g, " ").trim(); if (label === "") return null; - // On screen a file change is its path beside an icon. - const spoken = work.itemType === "file_change" ? `Editing ${label}` : label; + const verb = work.itemType === undefined ? undefined : SPOKEN_VERBS[work.itemType]; + const spoken = verb === undefined || /^search/i.test(label) ? label : `${verb} ${label}`; return spoken.length > ACTIVITY_MAX_LENGTH ? `${spoken.slice(0, ACTIVITY_MAX_LENGTH)}…` : spoken; } @@ -39,20 +58,32 @@ function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): strin export function turnDetail( entries: ReadonlyArray, workspaceRoot: string | undefined, + knownTailId: string | null, ): TurnDetail { - const messages: Array = []; - // Runs the agent kept working in after the entry being looked at. + const messages: TurnMessage[] = []; + // Runs and attempts the agent kept working in after the entry being looked at. const continuedRunIds = new Set(); + const continuedAttemptIds = new Set(); + const markContinued = (entry: TimelineEntry, runId: RunId | null) => { + continuedRunIds.add(runId); + if (entry.attempt !== undefined) continuedAttemptIds.add(entry.attempt.id); + }; let activity: string | null | undefined; + let fresh = true; for (let index = entries.length - 1; index >= 0; index -= 1) { const entry = entries[index]!; if (timelineEntryStartsResponse(entry)) break; + if (entry.id === knownTailId) fresh = false; if (entry.kind === "event") continue; if (entry.kind === "work") { const work = entry.entry; + if (work.sourceActivityKind === "context-compaction") { + activity ??= null; + continue; + } // Notices, todo lists and checkpoints sit beside the turn's steps without being one. if (work.tone !== "tool" && work.itemType !== "reasoning") continue; - continuedRunIds.add(work.runId ?? null); + markContinued(entry, work.runId ?? null); // A request is spoken as the thread's status; once answered, the step it was for goes on. if (work.itemType === "approval_request" || work.itemType === "user_input_request") continue; if (activity === undefined) activity = stepLabel(work, workspaceRoot); @@ -62,12 +93,26 @@ export function turnDetail( // A reply, plan or rendered page is newer than any step before it. activity ??= null; const runId = timelineEntryRunId(entry); - if (entry.kind === "message" && !entry.message.streaming && continuedRunIds.has(runId)) { - messages.push({ id: entry.message.id, text: entry.message.text }); + if (entry.kind === "message") { + messages.push({ + id: entry.message.id, + text: entry.message.text, + finished: + !entry.message.streaming && + // A steer starts a new attempt in the same run; the reply it cut off never finished. + (entry.attempt?.status === "superseded" + ? continuedAttemptIds.has(entry.attempt.id) + : continuedRunIds.has(runId)), + fresh, + }); } - continuedRunIds.add(runId); + markContinued(entry, runId); } - return { activity: activity ?? null, messages: messages.reverse() }; + return { + activity: activity ?? null, + messages: messages.reverse(), + tailId: entries.at(-1)?.id ?? null, + }; } export interface TurnDetailSnapshot extends TurnDetail { @@ -77,7 +122,9 @@ export interface TurnDetailSnapshot extends TurnDetail { export interface TurnDetailMemory { readonly activity: string | null; - readonly spokenMessageIds: ReadonlySet; + /** Each message seen in the turn, and whether it was already finished then. */ + readonly messages: ReadonlyMap; + readonly tailId: string | null; } /** The heading every agent message has in the timeline. */ @@ -98,7 +145,8 @@ export function nextTurnDetailAnnouncement( const memory: TurnDetailMemory = { // A step is spoken again after a pause for an approval or a question. activity: next.status === "working" ? next.activity : null, - spokenMessageIds: new Set(next.messages.map((message) => message.id)), + messages: new Map(next.messages.map((message) => [message.id, message.finished])), + tailId: next.tailId, }; // A turn that already ended is history, however late its entries arrive. if (previous === null || next.level === "status" || next.status === "idle") { @@ -107,7 +155,12 @@ export function nextTurnDetailAnnouncement( return { memory, messages: next.messages - .filter((message) => !previous.spokenMessageIds.has(message.id)) + .filter( + (message) => + message.finished && + previous.messages.get(message.id) !== true && + (message.fresh || previous.messages.has(message.id)), + ) .map((message) => markdownPlainText(message.text)) .filter((text) => text !== "") .map((text) => `${MESSAGE_AUTHOR}: ${text}`), diff --git a/packages/client-runtime/src/work-log/presentation.ts b/packages/client-runtime/src/work-log/presentation.ts index 8b44f88dce5b0..e2097ed4a46c4 100644 --- a/packages/client-runtime/src/work-log/presentation.ts +++ b/packages/client-runtime/src/work-log/presentation.ts @@ -678,20 +678,15 @@ export function toolGroupSummaryKind( return fallbackKinds.size === 1 ? fallbackKinds.values().next().value! : "mixed"; } -/** Markdown as one line of plain text, for places that can't render it. */ -export function markdownPlainText(markdown: string): string { - return markdown +export function liveThoughtLine(markdown: string): string { + const heading = /^\s*\*\*([^*\r\n]+)\*\*[ \t]*\r?(?:\n|$)/.exec(markdown)?.[1]; + const text = (heading ?? markdown) .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") .replace(/^[ \t]*(?:#{1,6}|[-*+]|\d+\.)[ \t]+/gm, "") .replace(/`+|\*\*|~~/g, "") .replace(/(^|[^\w*])[*_]([^*_\n]+)[*_](?![\w*])/g, "$1$2") .replace(/\s+/g, " ") .trim(); -} - -export function liveThoughtLine(markdown: string): string { - const heading = /^\s*\*\*([^*\r\n]+)\*\*[ \t]*\r?(?:\n|$)/.exec(markdown)?.[1]; - const text = markdownPlainText(heading ?? markdown); if (heading !== undefined) return text; const end = /[.?!]["'”’)]?(?=\s)/.exec(text); return end ? text.slice(0, end.index + end[0].length) : text; From 9333ea6976dc0cd2f16c663ceeb837d861267d4a Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:13:52 +0000 Subject: [PATCH 3/6] fix(web): turn announcements handle steered replies, missing history anchors and long messages Second review of the announcement rules: - A reply that completed before a steer replaced its attempt was never read. It counts as finished, by the same completion test the timeline uses for superseded replies. - When the entry last seen was gone, loaded history counted as new. Nothing is new unless that entry is still in the thread. - A step dropped while the thread's detail was behind was never read once it caught up. - "Editing search.ts" lost its verb, and a query starting with "search" kept none. - A message over 2,000 characters is cut short before it is parsed, so a huge one can't stall the page. Co-Authored-By: Claude Fable 5.2 --- .../ThreadActivityAnnouncer.browser.test.tsx | 20 +++++++ .../chat/ThreadActivityAnnouncer.tsx | 9 ++- .../chat/turnDetailAnnouncement.test.ts | 59 +++++++++++++++--- .../components/chat/turnDetailAnnouncement.ts | 60 ++++++++++++++----- docs/user/activity-log.md | 7 ++- 5 files changed, 129 insertions(+), 26 deletions(-) diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx index fd68f3f10e014..aeef3629e7c8c 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -89,6 +89,26 @@ describe("ThreadActivityAnnouncer", () => { await expect.element(detail).toBeEmptyDOMElement(); }); + it("reads a waiting step once the thread's detail catches up", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail } = regions(screen.container); + { + // Restored before the assertion, which polls on real timers. + using _timers = fakeTimers(); + await screen.rerender(announcer({})); + const entries = [prompt, command("git")]; + await screen.rerender(announcer({ entries })); + await screen.rerender(announcer({ entries, detailBehind: true })); + await screen.rerender(announcer({ entries })); + await act(() => vi.advanceTimersByTime(2_000)); + } + + await expect.element(detail).toHaveTextContent("Running git"); + }); + it("keeps a message beside the status that follows it, then reads the newest step", async () => { // The announcer reads no route. await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 1f0d7458681b1..822be6c69670e 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -101,7 +101,14 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ return; } const waitingLabel = takeWaiting(); - if (detailBehind) return; + if (detailBehind) { + // The dropped step counts as unsaid, so it is read once the detail catches up. + if (waitingLabel !== null && memoryRef.current !== null) { + const { detail } = memoryRef.current; + memoryRef.current = { ...memoryRef.current, detail: { ...detail, activity: null } }; + } + return; + } const previous = memoryRef.current?.threadKey === threadKey ? memoryRef.current : null; const turn = nextThreadAnnouncement(previous?.memory ?? null, { status, diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts index 9a4901f73c3ef..50f6adf940cdb 100644 --- a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts +++ b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts @@ -1,4 +1,5 @@ import { MessageId, RunAttemptId, RunId, type ScreenReaderAnnouncements } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import { describe, expect, it } from "vite-plus/test"; import type { TimelineAttempt, TimelineEntry, WorkLogEntry } from "../../session-logic"; @@ -54,10 +55,12 @@ const command = (id: string, fields: Partial = {}) => }); const notice = work("notice", { itemType: "system_notice", tone: "info", label: "Heads up" }); const steer = message("steer", "user", { inputIntent: "steer" }); +const time = (clock: string) => DateTime.makeUnsafe(`2026-01-01T${clock}.000Z`); const inAttempt = ( entry: TimelineEntry, id: string, status: TimelineAttempt["status"], + completed?: string, ): TimelineEntry => ({ ...entry, attempt: { @@ -66,9 +69,16 @@ const inAttempt = ( attemptOrdinal: 1, rootNodeId: `${id}-root` as TimelineAttempt["rootNodeId"], status, - completedAt: null, + completedAt: completed === undefined ? null : time(completed), }, }); +const completedAt = (entry: TimelineEntry, clock: string) => + ({ + ...entry, + projectedItem: { + item: { type: "assistant_message", status: "completed", completedAt: time(clock) }, + }, + }) as TimelineEntry; describe("turnDetail", () => { it.each<{ @@ -95,10 +105,20 @@ describe("turnDetail", () => { entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["src/a.ts"] })], activity: "Editing src/a.ts", }, + { + name: "a file change to a file called search", + entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["search.ts"] })], + activity: "Editing search.ts", + }, { name: "a web search, with the verb its icon stands for", - entries: [prompt, work("search", { itemType: "web_search", detail: "vitest fake timers" })], - activity: "Searching the web for vitest fake timers", + entries: [prompt, work("search", { itemType: "web_search", detail: "searchParams usage" })], + activity: "Searching the web for searchParams usage", + }, + { + name: "a file search that already says so", + entries: [prompt, work("find", { itemType: "file_search", label: "Searched files *.ts" })], + activity: "Searched files *.ts", }, { name: "the step under a notice", @@ -182,6 +202,19 @@ describe("turnDetail", () => { ], finished: [], }, + { + name: "a reply that completed before a steer replaced its attempt", + entries: [ + prompt, + completedAt( + inAttempt(message("complete", "assistant"), "attempt-1", "superseded", "00:00:10"), + "00:00:04", + ), + steer, + inAttempt(command("git"), "attempt-2", "running"), + ], + finished: ["complete"], + }, { name: "no last message that only a notice follows", entries: [prompt, command("git"), message("last", "assistant"), notice], @@ -208,7 +241,12 @@ describe("turnDetail", () => { ); }); - it("marks only messages after the entry last seen as fresh", () => { + it.each<{ name: string; knownTailId: string | null; fresh: ReadonlyArray }>([ + { name: "after the entry last seen", knownTailId: "seen", fresh: [false, true] }, + { name: "after a prompt that was last seen", knownTailId: "prompt", fresh: [true, true] }, + { name: "not when the entry last seen is gone", knownTailId: "gone", fresh: [false, false] }, + { name: "not when no entry was seen", knownTailId: null, fresh: [false, false] }, + ])("marks messages as fresh $name", ({ knownTailId, fresh }) => { const entries = [ prompt, message("loaded", "assistant"), @@ -216,10 +254,9 @@ describe("turnDetail", () => { message("new", "assistant"), command("git"), ]; - expect(turnDetail(entries, undefined, "seen").messages.map((spoken) => spoken.fresh)).toEqual([ - false, - true, - ]); + expect( + turnDetail(entries, undefined, knownTailId).messages.map((spoken) => spoken.fresh), + ).toEqual(fresh); }); }); @@ -301,6 +338,12 @@ describe("nextTurnDetailAnnouncement", () => { ).toEqual(["T3 Code: Tests pass in web. See the docs"]); }); + it("cuts a very long message short", () => { + const long = { ...said("long", true, true), text: "word ".repeat(1_000) }; + const [spoken] = speak("messages", [frame(null), frame(null, [long])]); + expect(spoken).toMatch(/^T3 Code: (?:word ){399}word More in the thread\.$/); + }); + it("says a step again after waiting on the user", () => { expect( speak("everything", [ diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.ts b/apps/web/src/components/chat/turnDetailAnnouncement.ts index bf2cad8b1e14b..0728674ad8744 100644 --- a/apps/web/src/components/chat/turnDetailAnnouncement.ts +++ b/apps/web/src/components/chat/turnDetailAnnouncement.ts @@ -1,4 +1,5 @@ import type { RunId, ScreenReaderAnnouncements } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import type { TimelineEntry, WorkLogEntry } from "../../session-logic"; import { @@ -30,13 +31,32 @@ export interface TurnDetail { export const NO_TURN_DETAIL: TurnDetail = { activity: null, messages: [], tailId: null }; const ACTIVITY_MAX_LENGTH = 120; +// Parsing is synchronous, and nobody listens to a longer message through a live region. +const MESSAGE_MAX_LENGTH = 2_000; // On screen these labels are a path or a query beside an icon. -const SPOKEN_VERBS: Partial, string>> = { - file_change: "Editing", - file_search: "Searching for", - web_search: "Searching the web for", -}; +function withVerb(work: WorkLogEntry, label: string) { + if (work.itemType === "file_change") return `Editing ${label}`; + if (work.itemType !== "file_search" && work.itemType !== "web_search") return label; + if (/^search(?:ed|ing)?\s/i.test(label)) return label; + return `${work.itemType === "web_search" ? "Searching the web for" : "Searching for"} ${label}`; +} + +type MessageEntry = Extract; + +// A steer starts a new attempt in the same run. The reply it cut off also stops streaming, +// so only one that completed before the steer finished; see completeSupersededReplyEntryIds. +function completedBeforeSuperseded(entry: MessageEntry) { + const item = entry.projectedItem?.item; + const supersededAt = entry.attempt?.completedAt ?? null; + return ( + item?.type === "assistant_message" && + item.status === "completed" && + item.completedAt !== null && + supersededAt !== null && + DateTime.isLessThan(item.completedAt, supersededAt) + ); +} function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): string | null { // Reasoning text is never read out. @@ -45,8 +65,7 @@ function stepLabel(work: WorkLogEntry, workspaceRoot: string | undefined): strin if (status === "failed" || status === "declined" || status === "stopped") return null; const label = liveWorkEntryLabel(work, workspaceRoot, true).replace(/\s+/g, " ").trim(); if (label === "") return null; - const verb = work.itemType === undefined ? undefined : SPOKEN_VERBS[work.itemType]; - const spoken = verb === undefined || /^search/i.test(label) ? label : `${verb} ${label}`; + const spoken = withVerb(work, label); return spoken.length > ACTIVITY_MAX_LENGTH ? `${spoken.slice(0, ACTIVITY_MAX_LENGTH)}…` : spoken; } @@ -70,7 +89,8 @@ export function turnDetail( }; let activity: string | null | undefined; let fresh = true; - for (let index = entries.length - 1; index >= 0; index -= 1) { + let index = entries.length - 1; + for (; index >= 0; index -= 1) { const entry = entries[index]!; if (timelineEntryStartsResponse(entry)) break; if (entry.id === knownTailId) fresh = false; @@ -99,18 +119,26 @@ export function turnDetail( text: entry.message.text, finished: !entry.message.streaming && - // A steer starts a new attempt in the same run; the reply it cut off never finished. - (entry.attempt?.status === "superseded" - ? continuedAttemptIds.has(entry.attempt.id) - : continuedRunIds.has(runId)), + continuedRunIds.has(runId) && + (entry.attempt?.status !== "superseded" || + continuedAttemptIds.has(entry.attempt.id) || + completedBeforeSuperseded(entry)), fresh, }); } markContinued(entry, runId); } + // Nothing is known to be new when the entry last seen is gone, or there was none. + let tailKnown = !fresh; + if (knownTailId !== null) { + for (; !tailKnown && index >= 0; index -= 1) tailKnown = entries[index]!.id === knownTailId; + } return { activity: activity ?? null, - messages: messages.reverse(), + messages: (tailKnown + ? messages + : messages.map((message) => ({ ...message, fresh: false })) + ).reverse(), tailId: entries.at(-1)?.id ?? null, }; } @@ -161,7 +189,11 @@ export function nextTurnDetailAnnouncement( previous.messages.get(message.id) !== true && (message.fresh || previous.messages.has(message.id)), ) - .map((message) => markdownPlainText(message.text)) + .map((message) => + message.text.length > MESSAGE_MAX_LENGTH + ? `${markdownPlainText(message.text.slice(0, MESSAGE_MAX_LENGTH))} More in the thread.` + : markdownPlainText(message.text), + ) .filter((text) => text !== "") .map((text) => `${MESSAGE_AUTHOR}: ${text}`), activity: diff --git a/docs/user/activity-log.md b/docs/user/activity-log.md index e3264898c1af6..a2a9278b80890 100644 --- a/docs/user/activity-log.md +++ b/docs/user/activity-log.md @@ -35,8 +35,9 @@ setting applies to this browser or device only. On web and desktop, a screen reader hears the open thread's turn as it runs. **Screen reader announcements** in **Settings → Chat** sets how much. **Status only** says when a turn starts, needs your approval or input, and ends. **Status and messages** also reads each message the agent -writes before its last one, once that message is complete. The last message is not read, so you -can read it yourself when you hear **Done**. **Everything** adds the step the agent is on, such as -the command it is running. Steps are spaced out, so a quick step can be skipped for a newer one. +writes and then keeps working after, once that message is complete. A very long message is cut +short. The reply that closes the turn is not read, so you can read it yourself when you hear +**Done**. **Everything** adds the step the agent is on, such as the command it is running. Steps +are spaced out, so a quick step can be skipped for a newer one. The agent's reasoning text is never read. The setting follows the agent's real activity, whatever **Tool calls in chat** hides, and applies to this browser or device only. From d454494b9f2fb19c095db0d43ee6fa249e59e83f Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:57:58 +0000 Subject: [PATCH 4/6] fix(web): a status that lands with a message is read after it, not before NVDA's log from a Windows VM showed every write to the two live regions being picked up once, but when the agent's message and "Needs approval" were written in the same frame, it read them in either order. They are now one write, the status as the message's last sentence. Written in separate frames they keep their own regions, where NVDA kept their order. Co-Authored-By: Claude Fable 5.2 --- .../ThreadActivityAnnouncer.browser.test.tsx | 29 +++++++++++++++---- .../chat/ThreadActivityAnnouncer.tsx | 11 ++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx index aeef3629e7c8c..9ac1657c441a5 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -115,14 +115,14 @@ describe("ThreadActivityAnnouncer", () => { router: false, }); const { detail, status } = regions(screen.container); - const said = message("Checking the repo", "assistant"); + const entries = [prompt, message("Checking the repo", "assistant"), command("git")]; const heard: Array<[status: string, detail: string]> = []; using _timers = fakeTimers(); for (const props of [ {}, - { entries: [prompt, said, command("git")], status: "approval", requestId: "request-1" }, - { entries: [prompt, said, command("git")] }, - { entries: [prompt, said, command("git"), command("rg")] }, + { entries }, + { entries, status: "approval", requestId: "request-1" }, + { entries: [...entries, command("rg")] }, ] satisfies ReadonlyArray>>) { await screen.rerender(announcer(props)); heard.push([status.textContent, detail.textContent]); @@ -132,10 +132,29 @@ describe("ThreadActivityAnnouncer", () => { expect(heard).toEqual([ ["Working", ""], - ["Needs approval", "T3 Code: Checking the repo"], + ["Working", "T3 Code: Checking the repo"], ["Needs approval", "T3 Code: Checking the repo"], ["Needs approval", "T3 Code: Checking the repo"], ["Needs approval", "Running rg"], ]); }); + + it("reads a status that lands with a message as the message's last sentence", async () => { + // The announcer reads no route. + await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { + router: false, + }); + const { detail, status } = regions(screen.container); + await screen.rerender(announcer({})); + await screen.rerender( + announcer({ + entries: [prompt, message("Checking the repo", "assistant"), command("git")], + status: "approval", + requestId: "request-1", + }), + ); + + await expect.element(detail).toHaveTextContent("T3 Code: Checking the repo. Needs approval"); + await expect.element(status).toHaveTextContent("Working"); + }); }); diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 822be6c69670e..5da0d311321b7 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -39,7 +39,7 @@ function asOneAnnouncement(texts: ReadonlyArray) { /** * The open thread's polite live regions. Status has its own, so "Needs - * approval" landing right after a message doesn't replace it unheard. + * approval" landing a moment after a message doesn't replace it unheard. */ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ threadKey, @@ -133,10 +133,13 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ return; } if (detail.messages.length > 0) { - const text = asOneAnnouncement(detail.messages); + // NVDA reads two regions written in the same frame in either order, so a + // status that lands with a message is read as its last sentence. + const text = asOneAnnouncement( + turn.announcement === null ? detail.messages : [...detail.messages, turn.announcement], + ); write(text, holdAfter(text)); - } - if (turn.announcement !== null) { + } else if (turn.announcement !== null) { announce(turn.announcement); step.quietUntil = Math.max(step.quietUntil, Date.now() + ACTIVITY_GAP_MS); } From 5909cc3bb4bd641747b94328de905386e1f2dac1 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:10:35 +0000 Subject: [PATCH 5/6] refactor(web): the announcer's waiting step is cancelled by its own effect The timer for a step waiting to be spoken was cleared on unmount by a second effect with its own lint suppression. The effect that sets the timer now returns its cleanup, so the suppression goes away. Co-Authored-By: Claude Fable 5.2 --- .../chat/ThreadActivityAnnouncer.tsx | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 5da0d311321b7..0dc54f3f2df00 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -154,8 +154,14 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ write(label, ACTIVITY_GAP_MS); }; const wait = step.quietUntil - Date.now(); - if (wait <= 0) speakStep(); - else step.waiting = { label, timer: setTimeout(speakStep, wait) }; + if (wait <= 0) { + speakStep(); + return; + } + const timer = setTimeout(speakStep, wait); + step.waiting = { label, timer }; + // The next run takes the label over; after an unmount nothing is left to speak it. + return () => clearTimeout(timer); }, [ threadKey, status, @@ -172,14 +178,6 @@ export const ThreadActivityAnnouncer = memo(function ThreadActivityAnnouncer({ clearDetail, ]); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- drops a step still waiting to be spoken when the timeline unmounts - useEffect( - () => () => { - if (stepRef.current.waiting !== null) clearTimeout(stepRef.current.waiting.timer); - }, - [], - ); - // No role="status": Orca 50 maps it to a status bar in Chromium and drops its updates. return ( <> From 124342cd1ac6b9bc21282668101d8e60f534a670 Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 11 Nov 2026 05:52:34 +0000 Subject: [PATCH 6/6] test(web): turn announcement tests keep one case per behaviour Drops rows that only restated a label or repeated a boundary another row covers, and folds the two detail-behind browser tests into one. Co-Authored-By: Claude Fable 5.2 --- .../ThreadActivityAnnouncer.browser.test.tsx | 23 ++++----- .../chat/turnDetailAnnouncement.test.ts | 51 +------------------ 2 files changed, 12 insertions(+), 62 deletions(-) diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx index 9ac1657c441a5..b9efe2dcd610e 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.browser.test.tsx @@ -61,16 +61,7 @@ const regions = (container: HTMLElement) => { }; describe("ThreadActivityAnnouncer", () => { - it.each<{ name: string; props: Partial> }>([ - { - name: "the agent has moved on", - props: { entries: [prompt, command("git"), message("reply", "assistant", true)] }, - }, - { - name: "the thread's detail falls behind", - props: { entries: [prompt, command("git")], detailBehind: true }, - }, - ])("drops a waiting step once $name", async ({ props }) => { + it("drops a waiting step once the agent has moved on", async () => { // The announcer reads no route. await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { router: false, @@ -81,7 +72,9 @@ describe("ThreadActivityAnnouncer", () => { using _timers = fakeTimers(); await screen.rerender(announcer({})); await screen.rerender(announcer({ entries: [prompt, command("git")] })); - await screen.rerender(announcer(props)); + await screen.rerender( + announcer({ entries: [prompt, command("git"), message("reply", "assistant", true)] }), + ); await act(() => vi.advanceTimersByTime(5_000)); } @@ -89,23 +82,27 @@ describe("ThreadActivityAnnouncer", () => { await expect.element(detail).toBeEmptyDOMElement(); }); - it("reads a waiting step once the thread's detail catches up", async () => { + it("holds a waiting step while the thread's detail is behind, then reads it", async () => { // The announcer reads no route. await using screen = await render(announcer({ status: "idle", runStatus: "completed" }), { router: false, }); const { detail } = regions(screen.container); + const entries = [prompt, command("git")]; + const heard: string[] = []; { // Restored before the assertion, which polls on real timers. using _timers = fakeTimers(); await screen.rerender(announcer({})); - const entries = [prompt, command("git")]; await screen.rerender(announcer({ entries })); await screen.rerender(announcer({ entries, detailBehind: true })); + await act(() => vi.advanceTimersByTime(5_000)); + heard.push(detail.textContent); await screen.rerender(announcer({ entries })); await act(() => vi.advanceTimersByTime(2_000)); } + expect(heard).toEqual([""]); await expect.element(detail).toHaveTextContent("Running git"); }); diff --git a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts index 50f6adf940cdb..c5eaa080ff921 100644 --- a/apps/web/src/components/chat/turnDetailAnnouncement.test.ts +++ b/apps/web/src/components/chat/turnDetailAnnouncement.test.ts @@ -87,11 +87,6 @@ describe("turnDetail", () => { activity: string | null; }>([ { name: "a running command", entries: [prompt, command("git")], activity: "Running git" }, - { - name: "a command that already finished, until something follows it", - entries: [prompt, command("git", { toolLifecycleStatus: "completed" })], - activity: "Running git", - }, { name: "reasoning, without its text", entries: [ @@ -102,32 +97,12 @@ describe("turnDetail", () => { }, { name: "a file change, with the verb its icon stands for", - entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["src/a.ts"] })], - activity: "Editing src/a.ts", - }, - { - name: "a file change to a file called search", entries: [prompt, work("edit", { itemType: "file_change", changedFiles: ["search.ts"] })], activity: "Editing search.ts", }, { - name: "a web search, with the verb its icon stands for", - entries: [prompt, work("search", { itemType: "web_search", detail: "searchParams usage" })], - activity: "Searching the web for searchParams usage", - }, - { - name: "a file search that already says so", - entries: [prompt, work("find", { itemType: "file_search", label: "Searched files *.ts" })], - activity: "Searched files *.ts", - }, - { - name: "the step under a notice", - entries: [prompt, command("git"), notice], - activity: "Running git", - }, - { - name: "the step a request was about", - entries: [prompt, command("git"), work("ask", { itemType: "approval_request" })], + name: "the step under a request and a notice", + entries: [prompt, command("git"), work("ask", { itemType: "approval_request" }), notice], activity: "Running git", }, { @@ -182,16 +157,6 @@ describe("turnDetail", () => { ], finished: ["first"], }, - { - name: "a message across a steer", - entries: [prompt, message("first", "assistant"), steer, command("git")], - finished: ["first"], - }, - { - name: "no message that is still streaming", - entries: [prompt, message("first", "assistant", { streaming: true }), command("git")], - finished: [], - }, { name: "no reply a steer cut off", entries: [ @@ -229,11 +194,6 @@ describe("turnDetail", () => { ], finished: [], }, - { - name: "no message from the turn before", - entries: [message("earlier", "user"), message("first", "assistant"), command("git"), prompt], - finished: [], - }, ])("counts $name as finished", ({ entries, finished }) => { const { messages } = turnDetail(entries, undefined, null); expect(messages.filter((spoken) => spoken.finished).map((spoken) => spoken.text)).toEqual( @@ -245,7 +205,6 @@ describe("turnDetail", () => { { name: "after the entry last seen", knownTailId: "seen", fresh: [false, true] }, { name: "after a prompt that was last seen", knownTailId: "prompt", fresh: [true, true] }, { name: "not when the entry last seen is gone", knownTailId: "gone", fresh: [false, false] }, - { name: "not when no entry was seen", knownTailId: null, fresh: [false, false] }, ])("marks messages as fresh $name", ({ knownTailId, fresh }) => { const entries = [ prompt, @@ -338,12 +297,6 @@ describe("nextTurnDetailAnnouncement", () => { ).toEqual(["T3 Code: Tests pass in web. See the docs"]); }); - it("cuts a very long message short", () => { - const long = { ...said("long", true, true), text: "word ".repeat(1_000) }; - const [spoken] = speak("messages", [frame(null), frame(null, [long])]); - expect(spoken).toMatch(/^T3 Code: (?:word ){399}word More in the thread\.$/); - }); - it("says a step again after waiting on the user", () => { expect( speak("everything", [