fix(mobile): screen readers hear when the open thread needs you or finishes

apps/mobile/src/features/threads/ThreadRouteScreen.tsx

@@ -28,6 +28,7 @@ import {
   AuthTerminalReadScope,
   DEFAULT_SERVER_SETTINGS,
   EnvironmentId,
+  isProviderNativeSubagentThread,
   MessageId,
   ThreadId,
   type ProjectScript,
@@ -122,6 +123,7 @@ import {
   type ThreadInspectorPresentation,
 } from "./thread-inspector-preview";
 import { useStartSideChat } from "./useStartSideChat";
+import { useThreadStatusAnnouncements } from "./useThreadStatusAnnouncements";
 import type { ThreadInspectorMode, ThreadInspectorTarget } from "./thread-inspector-navigation";
 import { useSelectWorkspaceDetail } from "../layout/workspace/useSelectWorkspaceDetail";
 
@@ -377,6 +379,24 @@ function ThreadRouteContent(
   const gitState = useSelectedThreadGitState();
   const gitActions = useSelectedThreadGitActions();
   const requests = useSelectedThreadRequests();
+  useThreadStatusAnnouncements({
+    threadKey:
+      selectedThread === null || isProviderNativeSubagentThread(selectedThread.source)
+        ? null
+        : scopedThreadKey(selectedThread.environmentId, selectedThread.id),
+    // A preview beside its host names its thread, so the two threads' statuses can't be confused.
+    subject:
+      props.inspectorPresentation !== undefined && "onMaximize" in props.inspectorPresentation
+        ? (selectedThread?.title ?? null)
+        : null,
+    approvalRequestId: requests.activePendingApproval?.requestId ?? null,
+    userInputRequestId: requests.activePendingUserInput?.requestId ?? null,
+    shellRequestId: selectedThread?.source.pendingRuntimeRequest?.id ?? null,
+    shellPendingUserInput: selectedThread?.hasPendingUserInput === true,
+    working: composer.activeWorkStartedAt !== null || composer.isCompacting,
+    run: composer.selectedThreadAnnouncedRun,
+    detailBehind: composer.selectedThreadDetailBehindShell,
+  });
   const interruptThreadTurn = useAtomCommand(threadEnvironment.interruptTurn, "thread interrupt");
   const resumeThreadQueue = useAtomCommand(threadEnvironment.resumeThreadQueue, {
     label: "resume queue",

apps/mobile/src/features/threads/threadStatusAnnouncement.test.ts

@@ -0,0 +1,82 @@
+import {
+  nextThreadAnnouncement,
+  type ThreadAnnouncementMemory,
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
+import { RunId } from "@t3tools/contracts";
+import { describe, expect, it } from "vite-plus/test";
+
+import {
+  nextThreadActivitySnapshot,
+  type ThreadActivityInput,
+  type ThreadActivityMemory,
+} from "./threadStatusAnnouncement";
+
+const run = RunId.make("run-1");
+
+/** A frame of a working turn: what the question card shows and what the shell holds. */
+function frame(card: string | null, shell: string | null): ThreadActivityInput {
+  return {
+    approvalRequestId: null,
+    userInputRequestId: card,
+    shellRequestId: shell,
+    shellPendingUserInput: shell !== null,
+    working: true,
+    run: { runId: run, status: "running" },
+  };
+}
+
+const idle: ThreadActivityInput = { ...frame(null, null), working: false, run: null };
+const done: ThreadActivityInput = {
+  ...frame(null, null),
+  working: false,
+  run: { runId: run, status: "completed" },
+};
+
+function announcements(frames: ReadonlyArray<ThreadActivityInput>): ReadonlyArray<string | null> {
+  let activity: ThreadActivityMemory | null = null;
+  let memory: ThreadAnnouncementMemory | null = null;
+  return frames.map((input) => {
+    activity = nextThreadActivitySnapshot(activity, input);
+    const next = nextThreadAnnouncement(memory, activity.snapshot);
+    memory = next.memory;
+    return next.announcement;
+  });
+}
+
+describe("nextThreadActivitySnapshot", () => {
+  it("says each question once while the card and the shell catch up at different times", () => {
+    expect(
+      announcements([
+        idle,
+        frame(null, null),
+        // The card shows question A before the shell has it.
+        frame("a", null),
+        frame("a", "a"),
+        // Answered: the card hides A at once, the shell still holds it.
+        frame(null, "a"),
+        // Question B reaches the card while the shell still holds A, and is answered before the shell has B.
+        frame("b", "a"),
+        frame(null, "a"),
+        frame(null, "b"),
+        frame(null, null),
+        // A secret request shows only in the shell.
+        frame(null, "secret"),
+        frame(null, null),
+        done,
+      ]),
+    ).toEqual([
+      null,
+      "Working",
+      "Needs your input",
+      null,
+      null,
+      "Needs your input",
+      null,
+      null,
+      null,
+      "Needs your input",
+      null,
+      "Done",
+    ]);
+  });
+});

apps/mobile/src/features/threads/threadStatusAnnouncement.ts

@@ -0,0 +1,53 @@
+import type { ThreadActivitySnapshot } from "@t3tools/client-runtime/state/thread-activity-announcement";
+import type { RunId, OrchestrationV2RunStatus } from "@t3tools/contracts";
+
+export interface ThreadActivityInput {
+  readonly approvalRequestId: string | null;
+  readonly userInputRequestId: string | null;
+  /** The thread shell's pending request, which secret and link cards have only there. */
+  readonly shellRequestId: string | null;
+  readonly shellPendingUserInput: boolean;
+  readonly working: boolean;
+  readonly run: { readonly runId: RunId; readonly status: OrchestrationV2RunStatus } | null;
+}
+
+export interface ThreadActivityMemory {
+  readonly snapshot: ThreadActivitySnapshot;
+  /** Requests the thread screen has shown as a card. */
+  readonly cardRequestIds: ReadonlySet<string>;
+}
+
+/** What the open thread's screen reader status follows, from the same sources the thread screen shows. */
+export function nextThreadActivitySnapshot(
+  previous: ThreadActivityMemory | null,
+  input: ThreadActivityInput,
+): ThreadActivityMemory {
+  const { approvalRequestId, userInputRequestId, shellRequestId } = input;
+  const cardRequestId = approvalRequestId ?? userInputRequestId;
+  const seen = previous?.cardRequestIds ?? new Set<string>();
+  // A card leaves as soon as it is answered, before the shell; a shell request the card
+  // already showed is that answered one, so the request stays the one last announced.
+  // Secret and link requests never show as a card and come from the shell alone.
+  const requestId =
+    cardRequestId ??
+    (shellRequestId !== null && seen.has(shellRequestId)
+      ? (previous?.snapshot.requestId ?? null)
+      : shellRequestId);
+  return {
+    snapshot: {
+      status:
+        approvalRequestId !== null
+          ? "approval"
+          : userInputRequestId !== null || input.shellPendingUserInput
+            ? "input"
+            : input.working
+              ? "working"
+              : "idle",
+      requestId,
+      runId: input.run?.runId ?? null,
+      runStatus: input.run?.status ?? null,
+    },
+    cardRequestIds:
+      cardRequestId === null || seen.has(cardRequestId) ? seen : new Set([...seen, cardRequestId]),
+  };
+}

apps/mobile/src/features/threads/useThreadStatusAnnouncements.ts

@@ -0,0 +1,85 @@
+import { useIsFocused } from "@react-navigation/native";
+import {
+  nextThreadAnnouncement,
+  type ThreadAnnouncementMemory,
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
+import { useEffect, useRef } from "react";
+import { AccessibilityInfo } from "react-native";
+
+import { useAppActive } from "../../lib/appState";
+import {
+  nextThreadActivitySnapshot,
+  type ThreadActivityInput,
+  type ThreadActivityMemory,
+} from "./threadStatusAnnouncement";
+
+/** Tells VoiceOver and TalkBack when the open thread starts working, needs the user, or finishes. */
+export function useThreadStatusAnnouncements({
+  threadKey,
+  subject,
+  detailBehind,
+  approvalRequestId,
+  userInputRequestId,
+  shellRequestId,
+  shellPendingUserInput,
+  working,
+  run,
+}: ThreadActivityInput & {
+  /** Null when nothing should be announced, such as a provider subagent's thread. */
+  readonly threadKey: string | null;
+  /** The thread's title, said after each status when another thread is on screen beside it. */
+  readonly subject: string | null;
+  /** See isDetailBehindShell. */
+  readonly detailBehind: boolean;
+}) {
+  // A retained route stays mounted under the screen above it; it speaks only while shown,
+  // and starts from a silent baseline when it is shown again.
+  const focused = useIsFocused();
+  const appActive = useAppActive();
+  const activeKey = focused && appActive ? threadKey : null;
+  const memoryRef = useRef<{
+    threadKey: string;
+    activity: ThreadActivityMemory;
+    announcement: ThreadAnnouncementMemory;
+  } | null>(null);
+  const runId = run?.runId ?? null;
+  const runStatus = run?.status ?? null;
+
+  // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- speaks committed status changes through the platform screen reader
+  useEffect(() => {
+    if (activeKey === null) {
+      memoryRef.current = null;
+      return;
+    }
+    if (detailBehind) return;
+    const previous = memoryRef.current?.threadKey === activeKey ? memoryRef.current : null;
+    const activity = nextThreadActivitySnapshot(previous?.activity ?? null, {
+      approvalRequestId,
+      userInputRequestId,
+      shellRequestId,
+      shellPendingUserInput,
+      working,
+      run: runId === null || runStatus === null ? null : { runId, status: runStatus },
+    });
+    const next = nextThreadAnnouncement(previous?.announcement ?? null, activity.snapshot);
+    memoryRef.current = { threadKey: activeKey, activity, announcement: next.memory };
+    // Queued, like the web's polite region, so VoiceOver doesn't cut off what it is reading.
+    if (next.announcement !== null) {
+      AccessibilityInfo.announceForAccessibilityWithOptions(
+        subject === null ? next.announcement : `${next.announcement}: ${subject}`,
+        { queue: true },
+      );
+    }
+  }, [
+    activeKey,
+    subject,
+    detailBehind,
+    approvalRequestId,
+    userInputRequestId,
+    shellRequestId,
+    shellPendingUserInput,
+    working,
+    runId,
+    runStatus,
+  ]);
+}

apps/mobile/src/state/use-thread-composer-state.ts

@@ -8,10 +8,15 @@ import { threadRuntimeIsActive } from "@t3tools/client-runtime/state/shell";
 import type { ThreadCheckpointSummary } from "@t3tools/client-runtime/state/thread-checkpoints";
 import { threadSteersDuringCompaction } from "@t3tools/client-runtime/state/thread-workflows";
 import { resolvePendingHandoff } from "@t3tools/client-runtime/handoff";
+import {
+  announcedRun,
+  isDetailBehindShell,
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
 import {
   deriveProviderSubagentStatus,
   deriveProviderExtensionStatuses,
   deriveResumableRunId,
+  deriveLatestThreadRun,
   deriveRunlessWorkStartedAt,
   deriveThreadActivityRun,
   deriveThreadRuntime,
@@ -335,6 +340,14 @@ export function useThreadComposerState(input: {
   const selectedThreadActivityRun = selectedThreadProjection
     ? deriveThreadActivityRun(selectedThreadProjection.projection)
     : (selectedThreadShell?.latestRun ?? null);
+  const selectedThreadAnnouncedRun = announcedRun(
+    selectedThreadActivityRun,
+    selectedThreadProjection?.projection.runs ?? [],
+  );
+  const selectedThreadDetailBehindShell = isDetailBehindShell(
+    selectedThreadProjection ? deriveLatestThreadRun(selectedThreadProjection.projection) : null,
+    selectedThreadShell?.latestRun ?? null,
+  );
 
   const activeRunId =
     selectedThreadRuntime?.activeRunId && threadRuntimeIsActive(selectedThreadRuntime)
@@ -881,6 +894,8 @@ export function useThreadComposerState(input: {
     dismissFeedback,
     selectedThreadFeed,
     selectedThreadActivityRun,
+    selectedThreadAnnouncedRun,
+    selectedThreadDetailBehindShell,
     selectedThreadQueueCount,
     selectedThreadQueuedMessages,
     dispatchingQueuedMessageId,

apps/web/src/components/ChatView.tsx

@@ -7,7 +7,10 @@ import { ScrollToEndButton, WorkspaceFileDropOverlay } from "./chat/TimelineOver
 import { usePasteToFocusComposer } from "./chat/usePasteToFocusComposer";
 import { useFocusComposerOnThreadOpen } from "./chat/useFocusComposerOnThreadOpen";
 import { configuredHostedAppUrl } from "../hostedPairing";
-import { announcedRun, isDetailBehindShell } from "./chat/threadActivityAnnouncement";
+import {
+  announcedRun,
+  isDetailBehindShell,
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
 import {
   deriveAvailableRevertTurnCounts,
   deriveThreadPendingBackgroundTasks,

apps/web/src/components/backgroundThreadAnnouncement.ts

@@ -1,7 +1,10 @@
 import type { OrchestrationV2RunStatus, RunId } from "@t3tools/contracts";
 
 import type { SidebarThreadSummary } from "../types";
-import { runOutcomeText, THREAD_STATUS_TEXT } from "./chat/threadActivityAnnouncement";
+import {
+  runOutcomeText,
+  THREAD_STATUS_TEXT,
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
 
 export interface BackgroundThreadSnapshot {
   readonly key: string;

apps/web/src/components/chat/ThreadActivityAnnouncer.tsx

@@ -8,7 +8,7 @@ import {
   nextThreadAnnouncement,
   type ThreadActivitySnapshot,
   type ThreadAnnouncementMemory,
-} from "./threadActivityAnnouncement";
+} from "@t3tools/client-runtime/state/thread-activity-announcement";
 import {
   nextTurnDetailAnnouncement,
   NO_TURN_DETAIL,

apps/web/src/components/chat/usePromptSuggestion.ts

@@ -6,7 +6,7 @@ import { useMediaQuery } from "../../hooks/useMediaQuery";
 import { useServerConfigs } from "../../state/entities";
 import { orchestrationEnvironment } from "../../state/orchestration";
 import { useEnvironmentQuery } from "../../state/query";
-import { announcedRun } from "./threadActivityAnnouncement";
+import { announcedRun } from "@t3tools/client-runtime/state/thread-activity-announcement";
 
 // Dismissed suggestions, keyed by environment and run, shared by every mounted composer and kept
 // across remounts. Run ids derive from thread ids, which can repeat across environments.

packages/client-runtime/package.json

@@ -335,6 +335,10 @@
       "types": "./src/state/threads.ts",
       "default": "./src/state/threads.ts"
     },
+    "./state/thread-activity-announcement": {
+      "types": "./src/state/threadActivityAnnouncement.ts",
+      "default": "./src/state/threadActivityAnnouncement.ts"
+    },
     "./state/subagent-display": {
       "types": "./src/state/subagentDisplay.ts",
       "default": "./src/state/subagentDisplay.ts"

apps/web/src/components/chat/threadActivityAnnouncement.test.ts → packages/client-runtime/src/state/threadActivityAnnouncement.test.ts

@@ -13,7 +13,7 @@ import {
   nextThreadAnnouncement,
   type ThreadActivitySnapshot,
   type ThreadAnnouncementMemory,
-} from "./threadActivityAnnouncement";
+} from "./threadActivityAnnouncement.ts";
 
 const firstRun = RunId.make("run-1");
 const secondRun = RunId.make("run-2");