diff --git a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx index 96c10b127b48e..6205fd1281797 100644 --- a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx @@ -28,6 +28,7 @@ import { AuthTerminalReadScope, DEFAULT_SERVER_SETTINGS, EnvironmentId, + isProviderNativeSubagentThread, MessageId, ThreadId, type ProjectScript, @@ -122,6 +123,7 @@ import { type ThreadInspectorPresentation, } from "./thread-inspector-preview"; import { useStartSideChat } from "./useStartSideChat"; +import { useThreadStatusAnnouncements } from "./useThreadStatusAnnouncements"; import type { ThreadInspectorMode, ThreadInspectorTarget } from "./thread-inspector-navigation"; import { useSelectWorkspaceDetail } from "../layout/workspace/useSelectWorkspaceDetail"; @@ -377,6 +379,24 @@ function ThreadRouteContent( const gitState = useSelectedThreadGitState(); const gitActions = useSelectedThreadGitActions(); const requests = useSelectedThreadRequests(); + useThreadStatusAnnouncements({ + threadKey: + selectedThread === null || isProviderNativeSubagentThread(selectedThread.source) + ? null + : scopedThreadKey(selectedThread.environmentId, selectedThread.id), + // A preview beside its host names its thread, so the two threads' statuses can't be confused. + subject: + props.inspectorPresentation !== undefined && "onMaximize" in props.inspectorPresentation + ? (selectedThread?.title ?? null) + : null, + approvalRequestId: requests.activePendingApproval?.requestId ?? null, + userInputRequestId: requests.activePendingUserInput?.requestId ?? null, + shellRequestId: selectedThread?.source.pendingRuntimeRequest?.id ?? null, + shellPendingUserInput: selectedThread?.hasPendingUserInput === true, + working: composer.activeWorkStartedAt !== null || composer.isCompacting, + run: composer.selectedThreadAnnouncedRun, + detailBehind: composer.selectedThreadDetailBehindShell, + }); const interruptThreadTurn = useAtomCommand(threadEnvironment.interruptTurn, "thread interrupt"); const resumeThreadQueue = useAtomCommand(threadEnvironment.resumeThreadQueue, { label: "resume queue", diff --git a/apps/mobile/src/features/threads/threadStatusAnnouncement.test.ts b/apps/mobile/src/features/threads/threadStatusAnnouncement.test.ts new file mode 100644 index 0000000000000..fa8814602cec7 --- /dev/null +++ b/apps/mobile/src/features/threads/threadStatusAnnouncement.test.ts @@ -0,0 +1,82 @@ +import { + nextThreadAnnouncement, + type ThreadAnnouncementMemory, +} from "@t3tools/client-runtime/state/thread-activity-announcement"; +import { RunId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + nextThreadActivitySnapshot, + type ThreadActivityInput, + type ThreadActivityMemory, +} from "./threadStatusAnnouncement"; + +const run = RunId.make("run-1"); + +/** A frame of a working turn: what the question card shows and what the shell holds. */ +function frame(card: string | null, shell: string | null): ThreadActivityInput { + return { + approvalRequestId: null, + userInputRequestId: card, + shellRequestId: shell, + shellPendingUserInput: shell !== null, + working: true, + run: { runId: run, status: "running" }, + }; +} + +const idle: ThreadActivityInput = { ...frame(null, null), working: false, run: null }; +const done: ThreadActivityInput = { + ...frame(null, null), + working: false, + run: { runId: run, status: "completed" }, +}; + +function announcements(frames: ReadonlyArray): ReadonlyArray { + let activity: ThreadActivityMemory | null = null; + let memory: ThreadAnnouncementMemory | null = null; + return frames.map((input) => { + activity = nextThreadActivitySnapshot(activity, input); + const next = nextThreadAnnouncement(memory, activity.snapshot); + memory = next.memory; + return next.announcement; + }); +} + +describe("nextThreadActivitySnapshot", () => { + it("says each question once while the card and the shell catch up at different times", () => { + expect( + announcements([ + idle, + frame(null, null), + // The card shows question A before the shell has it. + frame("a", null), + frame("a", "a"), + // Answered: the card hides A at once, the shell still holds it. + frame(null, "a"), + // Question B reaches the card while the shell still holds A, and is answered before the shell has B. + frame("b", "a"), + frame(null, "a"), + frame(null, "b"), + frame(null, null), + // A secret request shows only in the shell. + frame(null, "secret"), + frame(null, null), + done, + ]), + ).toEqual([ + null, + "Working", + "Needs your input", + null, + null, + "Needs your input", + null, + null, + null, + "Needs your input", + null, + "Done", + ]); + }); +}); diff --git a/apps/mobile/src/features/threads/threadStatusAnnouncement.ts b/apps/mobile/src/features/threads/threadStatusAnnouncement.ts new file mode 100644 index 0000000000000..ddaabb08d6c05 --- /dev/null +++ b/apps/mobile/src/features/threads/threadStatusAnnouncement.ts @@ -0,0 +1,53 @@ +import type { ThreadActivitySnapshot } from "@t3tools/client-runtime/state/thread-activity-announcement"; +import type { RunId, OrchestrationV2RunStatus } from "@t3tools/contracts"; + +export interface ThreadActivityInput { + readonly approvalRequestId: string | null; + readonly userInputRequestId: string | null; + /** The thread shell's pending request, which secret and link cards have only there. */ + readonly shellRequestId: string | null; + readonly shellPendingUserInput: boolean; + readonly working: boolean; + readonly run: { readonly runId: RunId; readonly status: OrchestrationV2RunStatus } | null; +} + +export interface ThreadActivityMemory { + readonly snapshot: ThreadActivitySnapshot; + /** Requests the thread screen has shown as a card. */ + readonly cardRequestIds: ReadonlySet; +} + +/** What the open thread's screen reader status follows, from the same sources the thread screen shows. */ +export function nextThreadActivitySnapshot( + previous: ThreadActivityMemory | null, + input: ThreadActivityInput, +): ThreadActivityMemory { + const { approvalRequestId, userInputRequestId, shellRequestId } = input; + const cardRequestId = approvalRequestId ?? userInputRequestId; + const seen = previous?.cardRequestIds ?? new Set(); + // A card leaves as soon as it is answered, before the shell; a shell request the card + // already showed is that answered one, so the request stays the one last announced. + // Secret and link requests never show as a card and come from the shell alone. + const requestId = + cardRequestId ?? + (shellRequestId !== null && seen.has(shellRequestId) + ? (previous?.snapshot.requestId ?? null) + : shellRequestId); + return { + snapshot: { + status: + approvalRequestId !== null + ? "approval" + : userInputRequestId !== null || input.shellPendingUserInput + ? "input" + : input.working + ? "working" + : "idle", + requestId, + runId: input.run?.runId ?? null, + runStatus: input.run?.status ?? null, + }, + cardRequestIds: + cardRequestId === null || seen.has(cardRequestId) ? seen : new Set([...seen, cardRequestId]), + }; +} diff --git a/apps/mobile/src/features/threads/useThreadStatusAnnouncements.ts b/apps/mobile/src/features/threads/useThreadStatusAnnouncements.ts new file mode 100644 index 0000000000000..c3e14809b39f9 --- /dev/null +++ b/apps/mobile/src/features/threads/useThreadStatusAnnouncements.ts @@ -0,0 +1,85 @@ +import { useIsFocused } from "@react-navigation/native"; +import { + nextThreadAnnouncement, + type ThreadAnnouncementMemory, +} from "@t3tools/client-runtime/state/thread-activity-announcement"; +import { useEffect, useRef } from "react"; +import { AccessibilityInfo } from "react-native"; + +import { useAppActive } from "../../lib/appState"; +import { + nextThreadActivitySnapshot, + type ThreadActivityInput, + type ThreadActivityMemory, +} from "./threadStatusAnnouncement"; + +/** Tells VoiceOver and TalkBack when the open thread starts working, needs the user, or finishes. */ +export function useThreadStatusAnnouncements({ + threadKey, + subject, + detailBehind, + approvalRequestId, + userInputRequestId, + shellRequestId, + shellPendingUserInput, + working, + run, +}: ThreadActivityInput & { + /** Null when nothing should be announced, such as a provider subagent's thread. */ + readonly threadKey: string | null; + /** The thread's title, said after each status when another thread is on screen beside it. */ + readonly subject: string | null; + /** See isDetailBehindShell. */ + readonly detailBehind: boolean; +}) { + // A retained route stays mounted under the screen above it; it speaks only while shown, + // and starts from a silent baseline when it is shown again. + const focused = useIsFocused(); + const appActive = useAppActive(); + const activeKey = focused && appActive ? threadKey : null; + const memoryRef = useRef<{ + threadKey: string; + activity: ThreadActivityMemory; + announcement: ThreadAnnouncementMemory; + } | null>(null); + const runId = run?.runId ?? null; + const runStatus = run?.status ?? null; + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- speaks committed status changes through the platform screen reader + useEffect(() => { + if (activeKey === null) { + memoryRef.current = null; + return; + } + if (detailBehind) return; + const previous = memoryRef.current?.threadKey === activeKey ? memoryRef.current : null; + const activity = nextThreadActivitySnapshot(previous?.activity ?? null, { + approvalRequestId, + userInputRequestId, + shellRequestId, + shellPendingUserInput, + working, + run: runId === null || runStatus === null ? null : { runId, status: runStatus }, + }); + const next = nextThreadAnnouncement(previous?.announcement ?? null, activity.snapshot); + memoryRef.current = { threadKey: activeKey, activity, announcement: next.memory }; + // Queued, like the web's polite region, so VoiceOver doesn't cut off what it is reading. + if (next.announcement !== null) { + AccessibilityInfo.announceForAccessibilityWithOptions( + subject === null ? next.announcement : `${next.announcement}: ${subject}`, + { queue: true }, + ); + } + }, [ + activeKey, + subject, + detailBehind, + approvalRequestId, + userInputRequestId, + shellRequestId, + shellPendingUserInput, + working, + runId, + runStatus, + ]); +} diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 8a762e88c97c6..8b88e489620ef 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -8,10 +8,15 @@ import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/shell"; import type { ThreadCheckpointSummary } from "@t3tools/client-runtime/state/thread-checkpoints"; import { threadSteersDuringCompaction } from "@t3tools/client-runtime/state/thread-workflows"; import { resolvePendingHandoff } from "@t3tools/client-runtime/handoff"; +import { + announcedRun, + isDetailBehindShell, +} from "@t3tools/client-runtime/state/thread-activity-announcement"; import { deriveProviderSubagentStatus, deriveProviderExtensionStatuses, deriveResumableRunId, + deriveLatestThreadRun, deriveRunlessWorkStartedAt, deriveThreadActivityRun, deriveThreadRuntime, @@ -335,6 +340,14 @@ export function useThreadComposerState(input: { const selectedThreadActivityRun = selectedThreadProjection ? deriveThreadActivityRun(selectedThreadProjection.projection) : (selectedThreadShell?.latestRun ?? null); + const selectedThreadAnnouncedRun = announcedRun( + selectedThreadActivityRun, + selectedThreadProjection?.projection.runs ?? [], + ); + const selectedThreadDetailBehindShell = isDetailBehindShell( + selectedThreadProjection ? deriveLatestThreadRun(selectedThreadProjection.projection) : null, + selectedThreadShell?.latestRun ?? null, + ); const activeRunId = selectedThreadRuntime?.activeRunId && threadRuntimeIsActive(selectedThreadRuntime) @@ -881,6 +894,8 @@ export function useThreadComposerState(input: { dismissFeedback, selectedThreadFeed, selectedThreadActivityRun, + selectedThreadAnnouncedRun, + selectedThreadDetailBehindShell, selectedThreadQueueCount, selectedThreadQueuedMessages, dispatchingQueuedMessageId, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 156c2f8f94327..d3f3f9125df5e 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -7,7 +7,10 @@ import { ScrollToEndButton, WorkspaceFileDropOverlay } from "./chat/TimelineOver import { usePasteToFocusComposer } from "./chat/usePasteToFocusComposer"; import { useFocusComposerOnThreadOpen } from "./chat/useFocusComposerOnThreadOpen"; import { configuredHostedAppUrl } from "../hostedPairing"; -import { announcedRun, isDetailBehindShell } from "./chat/threadActivityAnnouncement"; +import { + announcedRun, + isDetailBehindShell, +} from "@t3tools/client-runtime/state/thread-activity-announcement"; import { deriveAvailableRevertTurnCounts, deriveThreadPendingBackgroundTasks, diff --git a/apps/web/src/components/backgroundThreadAnnouncement.ts b/apps/web/src/components/backgroundThreadAnnouncement.ts index 10ca422d0f3cb..e7ad9b0c08103 100644 --- a/apps/web/src/components/backgroundThreadAnnouncement.ts +++ b/apps/web/src/components/backgroundThreadAnnouncement.ts @@ -1,7 +1,10 @@ import type { OrchestrationV2RunStatus, RunId } from "@t3tools/contracts"; import type { SidebarThreadSummary } from "../types"; -import { runOutcomeText, THREAD_STATUS_TEXT } from "./chat/threadActivityAnnouncement"; +import { + runOutcomeText, + THREAD_STATUS_TEXT, +} from "@t3tools/client-runtime/state/thread-activity-announcement"; export interface BackgroundThreadSnapshot { readonly key: string; diff --git a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx index 0dc54f3f2df00..5d367cf851d75 100644 --- a/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx +++ b/apps/web/src/components/chat/ThreadActivityAnnouncer.tsx @@ -8,7 +8,7 @@ import { nextThreadAnnouncement, type ThreadActivitySnapshot, type ThreadAnnouncementMemory, -} from "./threadActivityAnnouncement"; +} from "@t3tools/client-runtime/state/thread-activity-announcement"; import { nextTurnDetailAnnouncement, NO_TURN_DETAIL, diff --git a/apps/web/src/components/chat/usePromptSuggestion.ts b/apps/web/src/components/chat/usePromptSuggestion.ts index b3f52b2a7dc78..595841629c023 100644 --- a/apps/web/src/components/chat/usePromptSuggestion.ts +++ b/apps/web/src/components/chat/usePromptSuggestion.ts @@ -6,7 +6,7 @@ import { useMediaQuery } from "../../hooks/useMediaQuery"; import { useServerConfigs } from "../../state/entities"; import { orchestrationEnvironment } from "../../state/orchestration"; import { useEnvironmentQuery } from "../../state/query"; -import { announcedRun } from "./threadActivityAnnouncement"; +import { announcedRun } from "@t3tools/client-runtime/state/thread-activity-announcement"; // Dismissed suggestions, keyed by environment and run, shared by every mounted composer and kept // across remounts. Run ids derive from thread ids, which can repeat across environments. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 7c5b8ef1cb27b..d1e74151e53e3 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -335,6 +335,10 @@ "types": "./src/state/threads.ts", "default": "./src/state/threads.ts" }, + "./state/thread-activity-announcement": { + "types": "./src/state/threadActivityAnnouncement.ts", + "default": "./src/state/threadActivityAnnouncement.ts" + }, "./state/subagent-display": { "types": "./src/state/subagentDisplay.ts", "default": "./src/state/subagentDisplay.ts" diff --git a/apps/web/src/components/chat/threadActivityAnnouncement.test.ts b/packages/client-runtime/src/state/threadActivityAnnouncement.test.ts similarity index 99% rename from apps/web/src/components/chat/threadActivityAnnouncement.test.ts rename to packages/client-runtime/src/state/threadActivityAnnouncement.test.ts index b47895426306e..f40613b9910af 100644 --- a/apps/web/src/components/chat/threadActivityAnnouncement.test.ts +++ b/packages/client-runtime/src/state/threadActivityAnnouncement.test.ts @@ -13,7 +13,7 @@ import { nextThreadAnnouncement, type ThreadActivitySnapshot, type ThreadAnnouncementMemory, -} from "./threadActivityAnnouncement"; +} from "./threadActivityAnnouncement.ts"; const firstRun = RunId.make("run-1"); const secondRun = RunId.make("run-2"); diff --git a/apps/web/src/components/chat/threadActivityAnnouncement.ts b/packages/client-runtime/src/state/threadActivityAnnouncement.ts similarity index 100% rename from apps/web/src/components/chat/threadActivityAnnouncement.ts rename to packages/client-runtime/src/state/threadActivityAnnouncement.ts