diff --git a/apps/web/src/components/ChatView.dictation.browser.test.tsx b/apps/web/src/components/ChatView.dictation.browser.test.tsx new file mode 100644 index 0000000000000..8af6bcbe9143f --- /dev/null +++ b/apps/web/src/components/ChatView.dictation.browser.test.tsx @@ -0,0 +1,235 @@ +import { createMemoryHistory } from "@tanstack/react-router"; +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { MessageId, TurnItemId } from "@t3tools/contracts"; +import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts/settings"; +import type * as Context from "effect/Context"; +import { expect, it, vi } from "vite-plus/test"; +import { page } from "vite-plus/test/browser"; + +import { AppRoot } from "~/AppRoot"; +import "~/components/ChatView"; +import { useComposerDraftStore } from "~/composerDraftStore"; +import { __setClientSettingsForTests } from "~/hooks/useSettings"; +import { responseContextRecord } from "~/lib/composerContextRecords"; +import { getRouter } from "~/router"; +import { chooseAudioDestination, chooseTranscriptCleanup } from "~/speech/voiceConsent"; +import { + ENVIRONMENT_ID, + dispatchedCommands, + installFakeEnvironment, + threadWithHistory, + type FakeDriver, +} from "../../test/fakeEnvironment"; +import { act, render } from "../../test/render"; + +vi.mock("../../../../packages/client-runtime/src/connection/driver.ts", async (importOriginal) => { + const [actual, { fakeDriver }, Layer] = await Promise.all([ + importOriginal<{ readonly ConnectionDriver: Context.Key }>(), + import("../../test/fakeEnvironment"), + import("effect/Layer"), + ]); + return { ...actual, layer: Layer.succeed(actual.ConnectionDriver, fakeDriver) }; +}); +vi.mock(import("~/rpc/requestLatencyState"), async (original) => ({ + ...(await original()), + trackRpcRequestSent: () => {}, +})); + +it.each([ + ["transcription", "navigation"], + ["cleanup", "navigation"], + ["transcription", "queued-different"], + ["transcription", "queued-equal"], + ["transcription", "upload"], + ["transcription", "attachment-change"], + ["transcription", "prompt-change"], +])("keeps a confirmed voice submission owned by its draft: %s / %s", async (stage, scenario) => { + const normal = threadWithHistory("dictation-a", 1); + const b = threadWithHistory("dictation-b", 1); + const template = threadWithHistory("dictation-queued", 1); + const queued = { + ...template, + runs: template.runs.map((run) => ({ + ...run, + status: "queued" as const, + startedAt: null, + completedAt: null, + })), + }; + const editing = scenario.startsWith("queued-"); + const a = editing ? queued : normal; + using _environment = installFakeEnvironment([normal, b, queued], { acceptCommands: true }); + __setClientSettingsForTests({ + ...DEFAULT_CLIENT_SETTINGS, + onboardingCompletedAt: "2026-01-01T00:00:00.000Z", + }); + chooseAudioDestination({ kind: "environment", environmentId: ENVIRONMENT_ID }); + chooseTranscriptCleanup(stage === "cleanup"); + const aRef = scopeThreadRef(ENVIRONMENT_ID, a.id); + const bRef = scopeThreadRef(ENVIRONMENT_ID, b.id); + const mainPrompt = scenario === "queued-equal" ? "Queued message" : "Fix"; + useComposerDraftStore.getState().setPrompt(aRef, mainPrompt); + if (editing) + useComposerDraftStore.getState().setReplyTo( + aRef, + responseContextRecord({ + citation: { + version: 1, + environmentId: ENVIRONMENT_ID, + threadId: a.id, + messageId: MessageId.make("dictation-reply"), + text: "Normal draft reply", + start: 0, + end: 18, + prefix: "", + suffix: "", + }, + itemId: TurnItemId.make("dictation-reply"), + subagents: [], + }), + ); + const file = { + type: "file" as const, + id: `dictation-file-${scenario}`, + name: "report.txt", + mimeType: "text/plain", + sizeBytes: 1, + file: null, + uploadedAttachmentId: `uploaded-${scenario}`, + uploadEnvironmentId: ENVIRONMENT_ID, + }; + const fileScenario = ["upload", "attachment-change", "prompt-change"].includes(scenario); + if (fileScenario) useComposerDraftStore.getState().addFiles(aRef, [file]); + useComposerDraftStore.getState().setPrompt(bRef, "Do not send"); + let resolveResponse!: (response: Response) => void; + const response = new Promise((resolve) => { + resolveResponse = resolve; + }); + await using _response = { + [Symbol.asyncDispose]: () => + act(async () => { + resolveResponse(Response.json({ text: "the login bug" })); + }), + }; + const fetch = globalThis.fetch; + using _fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input), window.location.href); + if (url.pathname === "/api/voice/status") + return Response.json({ + supported: true, + state: "ready", + modelId: "test-model", + model: "Test", + size: 1, + supportsStreaming: false, + supportsTranslation: false, + acceleration: "auto", + modelUnloadTimeout: "min_15", + gpuDevices: [], + }); + if (url.pathname === "/api/voice/transcribe") + return stage === "transcription" ? response : Response.json({ text: "the login bug" }); + if (url.pathname === "/api/voice/post-process") return response; + return fetch(input, init); + }); + const router = getRouter(createMemoryHistory({ initialEntries: [`/${ENVIRONMENT_ID}/${a.id}`] })); + await router.load(); + await using _app = await render(, { + // AppRoot supplies its own router. + router: false, + }); + const audio = new AudioContext(); + const oscillator = audio.createOscillator(); + const destination = audio.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + using _audio = { + [Symbol.dispose]: () => { + oscillator.stop(); + void audio.close(); + }, + }; + using _microphone = vi + .spyOn(navigator.mediaDevices, "getUserMedia") + .mockResolvedValue(destination.stream); + if (editing) { + await page.getByRole("button", { name: "Edit queued message", exact: true }).click(); + await expect.element(page.getByTestId("composer-editor")).toHaveTextContent("Queued message"); + } + await page.getByRole("button", { name: "Start voice input", exact: true }).click(); + await expect + .element(page.getByRole("button", { name: "Cancel voice input", exact: true })) + .toBeVisible(); + await page + .getByRole("button", { + name: editing ? "Update queued message" : "Submit message", + exact: true, + }) + .click(); + await vi.waitFor(() => + expect( + _fetch.mock.calls.some(([input]) => + String(input instanceof Request ? input.url : input).includes( + stage === "transcription" ? "/api/voice/transcribe" : "/api/voice/post-process", + ), + ), + ).toBe(true), + ); + if (!editing) { + await act(() => + router.navigate({ + to: "/$environmentId/$threadId", + params: { environmentId: ENVIRONMENT_ID, threadId: b.id }, + }), + ); + await expect.element(page.getByTestId("composer-editor")).toHaveTextContent("Do not send"); + } + if (fileScenario) { + const before = useComposerDraftStore.getState().getComposerDraft(aRef); + await act(() => { + if (scenario === "attachment-change") { + useComposerDraftStore.getState().removeFile(aRef, file.id); + useComposerDraftStore + .getState() + .addFiles(aRef, [{ ...file, id: `${file.id}-replacement` }]); + } else if (scenario === "prompt-change") + useComposerDraftStore.getState().setPrompt(aRef, "Later text"); + else + useComposerDraftStore + .getState() + .setFileUpload(aRef, file.id, ENVIRONMENT_ID, "completed-upload"); + }); + expect(useComposerDraftStore.getState().getComposerDraft(aRef)).not.toBe(before); + } + resolveResponse(Response.json({ text: "the login bug" })); + if (scenario === "attachment-change" || scenario === "prompt-change") { + await expect + .element(page.getByText("Voice message was not sent", { exact: true })) + .toBeVisible(); + expect(dispatchedCommands().filter((command) => command.type === "message.dispatch")).toEqual( + [], + ); + expect(useComposerDraftStore.getState().getComposerDraft(aRef)?.prompt).toBe( + scenario === "prompt-change" ? "Later text" : "Fix", + ); + } else if (editing) { + await vi.waitFor(() => + expect(dispatchedCommands().filter((command) => command.type === "queued-run.edit")).toEqual([ + expect.objectContaining({ threadId: a.id, text: "Queued message the login bug" }), + ]), + ); + expect(dispatchedCommands().filter((command) => command.type === "message.dispatch")).toEqual( + [], + ); + expect(useComposerDraftStore.getState().getComposerDraft(aRef)?.prompt).toBe(mainPrompt); + } else { + await vi.waitFor(() => + expect(dispatchedCommands().filter((command) => command.type === "message.dispatch")).toEqual( + [expect.objectContaining({ threadId: a.id, text: "Fix the login bug" })], + ), + ); + } + expect(useComposerDraftStore.getState().getComposerDraft(bRef)?.prompt).toBe("Do not send"); + if (!editing) + await expect.element(page.getByTestId("composer-editor")).toHaveTextContent("Do not send"); +}); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6616af7f3fbd5..40d1a7d2ee4d4 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -2978,7 +2978,14 @@ export default function ChatView(props: ChatViewProps) { }); }; const onWorktreeSetupWorkLocallyStable = useStableCallback(onWorktreeSetupWorkLocally); - const onComposerSendStable = useStableCallback(onSend); + const onComposerSendStable = useStableCallback( + ( + event?: { preventDefault: () => void }, + mode?: Parameters[1], + intent?: Parameters[2], + dictation?: Promise, + ) => onSend(event, mode, intent, undefined, dictation), + ); const sendPreviewAnnotation = ( annotation: PreviewAnnotationPayload, image: ComposerImageAttachment | null, diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index eb01b8242e2df..cad812ec78961 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -461,6 +461,7 @@ export interface ChatComposerProps { e?: { preventDefault: () => void }, dispatchMode?: ComposerDispatchMode, submissionIntent?: ComposerSubmissionIntent, + dictation?: Promise, ) => void; onResume: () => void; onInterrupt: () => void; @@ -1809,8 +1810,35 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) environmentId, projectId: activeProjectId, projectName, - ownerKey: questionAttachmentTarget ?? composerDraftTargetKey, + ownerKey: + questionAttachmentTarget ?? + (isEditingQueuedMessage ? composerDraftTargetKey : composerTargetKey(routeThreadRef)), disabled: dictationDisabled, + captureSubmissionDraft: + questionAttachmentTarget || isEditingQueuedMessage + ? undefined + : () => { + const target = routeThreadRef; + const captured = getComposerDraft(target); + return { + readDraft: () => { + const current = getComposerDraft(target); + if ( + (current?.prompt ?? "") !== (captured?.prompt ?? "") || + (current?.images.length ?? 0) !== (captured?.images.length ?? 0) || + current?.images.some( + (image, index) => image.id !== captured?.images[index]?.id, + ) || + (current?.files.length ?? 0) !== (captured?.files.length ?? 0) || + current?.files.some((file, index) => file.id !== captured?.files[index]?.id) + ) + return null; + const text = current?.prompt ?? ""; + return { text, selection: { start: text.length, end: text.length } }; + }, + commitDraft: (text) => setComposerDraftPrompt(target, text), + }; + }, readDraft: () => { const snapshot = readComposerSnapshot(); return { @@ -2102,7 +2130,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) if (activePendingProgress) { return activePendingProgress.isLastQuestion && Boolean(activePendingResolvedAnswers); } - return composerSendState.hasSendableContent; + return composerSendState.hasSendableContent || voiceInputActive; }; const submitComposerRef = useRef< @@ -2113,11 +2141,6 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) dispatchMode?: ComposerDispatchMode, submissionIntent?: ComposerSubmissionIntent, ) => { - if (voiceInputActive) { - event?.preventDefault(); - submitAfterDictation(() => submitComposerRef.current(dispatchMode, submissionIntent)); - return false; - } if (submissionBlocked || !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) { event?.preventDefault(); return false; @@ -2143,6 +2166,41 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }); return false; } + if (voiceInputActive) { + event?.preventDefault(); + if (activePendingProgress) { + submitAfterDictation(() => submitComposerRef.current(dispatchMode, submissionIntent)); + return false; + } + const transcript = new Promise((resolve) => { + submitAfterDictation( + (text) => { + resolve(text); + if ( + composerDraftTargetKeyRef.current === composerDraftTargetKey && + composerFormRef.current?.isConnected + ) { + onTypingGuardSend(); + if (shouldBlurMobileComposerOnSubmit()) blurMobileComposerAfterSend(); + } + }, + () => resolve(null), + ); + }); + onSend( + undefined, + dispatchMode ?? + resolveComposerDispatchMode({ + running: phase === "running" || phase === "connecting", + alternateModifier: false, + activeTurnDefault: followUpBehavior, + canSteer, + }), + submissionIntent, + transcript, + ); + return false; + } // An empty draft has nothing to send; Enter does what the Resume thread button does. if (showResumeAction && !activePendingProgress) { event?.preventDefault(); @@ -2179,7 +2237,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) return true; }; const submitComposerStable = useStableCallback(submitComposer); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed submission handler to the asynchronous dictation completion + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- question submissions need the answers from the render that committed their transcript useLayoutEffect(() => { submitComposerRef.current = (mode, intent) => submitComposerStable(undefined, mode, intent); }, [submitComposerStable]); diff --git a/apps/web/src/components/chat/useComposerSend.ts b/apps/web/src/components/chat/useComposerSend.ts index 648b2fd62bf05..df27b0770c2dc 100644 --- a/apps/web/src/components/chat/useComposerSend.ts +++ b/apps/web/src/components/chat/useComposerSend.ts @@ -1,3 +1,4 @@ +import { getComposerSubmissionValidationMessage } from "./composerSubmission"; import { isClearComposerCommand, parseSideChatCommand } from "@t3tools/shared/composerTrigger"; import { armFreshSession, disarmFreshSession, readFreshSessionArm } from "./freshSessionArms"; import { @@ -276,8 +277,6 @@ export function useComposerSend(input: ComposerSendInput) { activeProject, activeProjectDefaultModelSelection, activeMessageCount, - isServerThread, - isLocalDraftThread, isGitRepo, phase, runtimeMode, @@ -381,20 +380,88 @@ export function useComposerSend(input: ComposerSendInput) { annotation: PreviewAnnotationPayload; image: ComposerImageAttachment | null; }, + dictation?: Promise, ) => { e?.preventDefault(); + const stillHere = () => currentRouteThreadKeyRef.current === routeThreadKey; + const capturedHandle = input.composerRef.current; + const capturedContext = dictation ? capturedHandle?.getSendContext() : undefined; + const pendingEnvironmentAtSend = readPendingEnvironmentChange(); + const pendingAttachmentsAtSend = dictation && capturedHandle?.hasPendingAttachments(); + let dictatedPrompt = input.promptRef.current; + const composerRef = dictation + ? { + current: capturedHandle && { + ...capturedHandle, + getSendContext: () => capturedContext!, + hasPendingAttachments: () => Boolean(pendingAttachmentsAtSend), + validateProviderInput: (providerInput: string) => { + const message = getComposerSubmissionValidationMessage({ + prompt: dictatedPrompt, + providerInput, + submissionTarget: "provider-turn", + }); + if (message) + toastManager.add({ + type: "error", + title: "Voice message was not sent", + description: message, + }); + return message === null; + }, + resetCursorState: (...args: Parameters) => { + if (stillHere()) input.composerRef.current?.resetCursorState(...args); + }, + }, + } + : input.composerRef; + if (dictation) { + const text = await dictation; + if (text === null) return; + dictatedPrompt = text; + if ( + !stillHere() && + (queuedRunEdit.isEditing || parseSideChatCommand(text) || isUsageLimitsCommand(text)) + ) { + toastManager.add({ + type: "info", + title: "Voice message kept in its draft", + description: "Return to the thread to finish this action.", + }); + return; + } + } + const composerDraftTarget = + dictation && activeThread && !queuedRunEdit.isEditing + ? scopeThreadRef(environmentId, activeThread.id) + : input.composerDraftTarget; + const promoted = + dictation && draftId && useComposerDraftStore.getState().getPromotedDraftRedirect(draftId); + if ( + promoted && + (promoted.environmentId !== environmentId || promoted.threadId !== activeThread?.id) + ) { + toastManager.add({ + type: "info", + title: "Voice message was not sent", + description: "The draft's destination changed. Return to its draft to send again.", + }); + return; + } + const isLocalDraftThread = input.isLocalDraftThread && !promoted; + const isServerThread = input.isServerThread || Boolean(promoted); // What the user pressed send on; uploads below can take a while, and a redirect must not // decline approvals for a draft that changed meanwhile. - const promptAtSend = promptRef.current; + const promptAtSend = dictatedPrompt; if ( usageLimitsOffered && usageLimitsKey !== null && !directAnnotation && !draftHasNonPromptContent(composerDraftTarget) && - isUsageLimitsCommand(promptRef.current) + isUsageLimitsCommand(promptAtSend) ) { if (openUsageLimits()) { - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; setComposerDraftPrompt(composerDraftTarget, ""); composerRef.current?.resetCursorState(); } @@ -436,7 +503,8 @@ export function useComposerSend(input: ComposerSendInput) { isRevertingCheckpoint || !clientSettingsHydrated || threadDetailLoading || - readPendingEnvironmentChange() !== null || + pendingEnvironmentAtSend !== null || + (stillHere() && readPendingEnvironmentChange() !== null) || feedbackUploadsInFlightRef.current.has(routeThreadKey) ) { notifyDirectAnnotationAttached(); @@ -522,7 +590,7 @@ export function useComposerSend(input: ComposerSendInput) { currentRouteThreadKeyRef.current === routeThreadKey && promptRef.current === promptAtSend ) { - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; setComposerDraftPrompt(composerDraftTarget, ""); composerRef.current?.resetCursorState(); } @@ -596,10 +664,10 @@ export function useComposerSend(input: ComposerSendInput) { // A direct "send annotation" writes the draft and sends in the same tick; the reference // must be in the text now, not after the next render. const draftPrompt = directAnnotation - ? ensureInlineContextReferences(promptRef.current, [ + ? ensureInlineContextReferences(promptAtSend, [ previewAnnotationContextReference(directAnnotation.annotation), ]) - : promptRef.current; + : promptAtSend; const replyTo = useComposerDraftStore.getState().getComposerDraft(composerDraftTarget)?.replyTo ?? null; // The reply leads the message, so its card sits above the text it is about. Text that @@ -691,7 +759,7 @@ export function useComposerSend(input: ComposerSendInput) { createdAt: new Date().toISOString(), }, clearDraft: () => { - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); }, @@ -729,7 +797,7 @@ export function useComposerSend(input: ComposerSendInput) { replyTo === null; if (freshSessionAvailable && composerHoldsOnlyText && isClearComposerCommand(trimmed)) { armFreshSession(routeThreadKey); - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); return; @@ -899,6 +967,7 @@ export function useComposerSend(input: ComposerSendInput) { if ( multipleModelSelections === null && + stillHere() && shouldDockDraftHeroForSubmission({ isDraftHeroState, activeThreadKey, submissionIntent }) && activeThreadKey ) { @@ -988,8 +1057,10 @@ export function useComposerSend(input: ComposerSendInput) { const attachmentIds = sendContext ? [...sendContext.images, ...sendContext.files].map((attachment) => attachment.id) : attachmentIdsAtSend; - const promptUnchanged = - promptRef.current === promptAtSend || promptRef.current === promptForSend; + const currentPrompt = dictation + ? useComposerDraftStore.getState().getComposerDraft(composerDraftTarget)?.prompt + : promptRef.current; + const promptUnchanged = currentPrompt === promptAtSend || currentPrompt === promptForSend; const attachmentsUnchanged = sameIds(attachmentIds, attachmentIdsAtSend) || (annotationImageId !== undefined && @@ -1039,7 +1110,7 @@ export function useComposerSend(input: ComposerSendInput) { useComposerDraftStore.getState().getComposerDraft(composerDraftTarget) === draftBeforeDecline ) { - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); } @@ -1089,7 +1160,7 @@ export function useComposerSend(input: ComposerSendInput) { } if (sendQueuesBehindSetup) { setSetupQueueSendThreadKey(activeThreadKey); - } else { + } else if (stillHere()) { beginLocalDispatch({ preparingWorktree: multipleModelSelections !== null || Boolean(baseBranchForWorktree), submissionIntent: @@ -1156,7 +1227,7 @@ export function useComposerSend(input: ComposerSendInput) { } else { optimisticBlobPreviews.release(optimisticMessage); } - setOptimisticUserMessages((existing) => [...existing, optimisticMessage]); + if (stillHere()) setOptimisticUserMessages((existing) => [...existing, optimisticMessage]); setThreadError(threadIdForSend, null); if (expiredTerminalContextCount > 0) { const toastCopy = buildExpiredTerminalContextToastCopy( @@ -1176,7 +1247,7 @@ export function useComposerSend(input: ComposerSendInput) { : null; // Keep new-thread flows from reusing this draft while attachments and worktree setup settle. if (sentDraftThreadRef) markPromotedDraftThreadByRef(sentDraftThreadRef); - promptRef.current = ""; + if (stillHere()) promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); @@ -1275,7 +1346,7 @@ export function useComposerSend(input: ComposerSendInput) { } : undefined; const backgroundThreadRef = - submissionIntent === "background" && isLocalDraftThread + submissionIntent === "background" && isLocalDraftThread && stillHere() ? scopeThreadRef(environmentId, threadIdForSend) : null; // Read the live session, not the render snapshot: the draft is finalized away once the @@ -1484,7 +1555,7 @@ export function useComposerSend(input: ComposerSendInput) { setDockedDraftHeroThreadKey((currentThreadKey) => currentThreadKey === activeThreadKey ? null : currentThreadKey, ); - if (!sendQueuesBehindSetup) resetLocalDispatch(); + if (!sendQueuesBehindSetup && stillHere()) resetLocalDispatch(); } }; diff --git a/apps/web/src/components/chat/useMultipleModelSend.ts b/apps/web/src/components/chat/useMultipleModelSend.ts index ccea2378968f5..185c1745b0015 100644 --- a/apps/web/src/components/chat/useMultipleModelSend.ts +++ b/apps/web/src/components/chat/useMultipleModelSend.ts @@ -133,9 +133,11 @@ export function useMultipleModelSend(input: MultipleModelSendInput) { text: messageText, attachments: attachmentsSnapshot, }); - promptRef.current = ""; + if (currentRouteThreadKeyRef.current === routeThreadKey) { + promptRef.current = ""; + composerRef.current?.resetCursorState(); + } clearComposerDraftContent(composerDraftTarget); - composerRef.current?.resetCursorState(); clearedDraft = true; const clearedDraftSnapshot = useComposerDraftStore .getState() diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index b5418bcb409da..ca6efd41236c4 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -4,6 +4,7 @@ import { page, userEvent } from "vite-plus/test/browser"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; +import { AsyncResult, Atom } from "effect/reactivity"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input"; import { @@ -17,7 +18,7 @@ import { type SpeechTranscriptionOptions, } from "@t3tools/contracts"; -import { act, render, stubGlobal } from "../../test/render"; +import { act, fakeTimers, render, stubGlobal } from "../../test/render"; import { VoiceInputSetup } from "../components/chat/VoiceInputSetup"; import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput"; @@ -28,6 +29,7 @@ import { pendingCloudAudioChoice, } from "./voiceConsent"; import { writeBrowserClientSettings } from "../clientPersistenceStorage"; +import { AppAtomRegistryProvider } from "../rpc/atomRegistry"; const mocks = vi.hoisted(() => ({ voice: undefined as unknown as ReturnType, @@ -36,6 +38,7 @@ const mocks = vi.hoisted(() => ({ voiceCapable: new Set(), cloudVoiceCapable: new Set(), canDictate: new Set(), + sessionAtoms: new Map>>(), prepared: new Map(), primaryEnvironmentId: "primary-env", statusRequests: [] as string[], @@ -51,27 +54,44 @@ const mocks = vi.hoisted(() => ({ microphoneRequests: [] as string[], transcribeCalls: [] as string[], cleanupCalls: [] as string[], + cleanupResult: undefined as Promise | undefined, cleanupDraft: vi.fn(), cleanupOptions: [] as SpeechPostProcessingOptions[], recordingOptions: [] as SpeechTranscriptionOptions[], projectWords: {} as Record, - transcripts: [] as Array, + transcripts: [] as Array>, committed: vi.fn(), draft: "", + drafts: {} as Record, })); -vi.mock("../state/session", () => ({ - useEnvironmentScope: (id: string) => mocks.canDictate.has(id), - usePreparedConnection: (environmentId: string | null) => { - if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); - let prepared = mocks.prepared.get(environmentId); - if (!prepared) { - prepared = { environmentId, label: environmentId }; - mocks.prepared.set(environmentId, prepared); +vi.mock("../state/session", async () => { + const { useAtomValue } = await import("@effect/atom-react"); + const { AsyncResult, Atom } = await import("effect/reactivity"); + const sessionStateAtom = (id: string) => { + let atom = mocks.sessionAtoms.get(id); + if (!atom) { + atom = Atom.make(AsyncResult.success(true)); + mocks.sessionAtoms.set(id, atom); } - return Option.some(prepared); - }, -})); + return atom; + }; + return { + environmentSession: { sessionStateAtom }, + useEnvironmentScope: (id: string) => + AsyncResult.getOrElse(useAtomValue(sessionStateAtom(id)), () => false) && + mocks.canDictate.has(id), + usePreparedConnection: (environmentId: string | null) => { + if (!environmentId || !mocks.connected.has(environmentId)) return Option.none(); + let prepared = mocks.prepared.get(environmentId); + if (!prepared) { + prepared = { environmentId, label: environmentId }; + mocks.prepared.set(environmentId, prepared); + } + return Option.some(prepared); + }, + }; +}); vi.mock("../state/environments", () => ({ usePrimaryEnvironmentId: () => mocks.primaryEnvironmentId, useEnvironment: (environmentId: string | null) => @@ -157,11 +177,11 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ draft: unknown, options: SpeechPostProcessingOptions, ) => - Effect.sync(() => { + Effect.promise(async () => { mocks.cleanupCalls.push(prepared.environmentId); mocks.cleanupDraft(draft); mocks.cleanupOptions.push(options); - return { text: `Cleaned ${transcript}` }; + return { text: mocks.cleanupResult ? await mocks.cleanupResult : `Cleaned ${transcript}` }; }), })); vi.mock("./browserVoiceInput", () => ({ @@ -215,6 +235,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); + mocks.sessionAtoms.clear(); mocks.prepared.clear(); mocks.statusRequests = []; mocks.statusUnavailable = false; @@ -229,6 +250,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.microphoneRequests = []; mocks.transcribeCalls = []; mocks.cleanupCalls = []; + mocks.cleanupResult = undefined; mocks.cleanupDraft.mockClear(); mocks.cleanupOptions = []; mocks.recordingOptions = []; @@ -363,6 +385,216 @@ it.each([ expect(mocks.committed).not.toHaveBeenCalled(); }); +function SubmissionProbe(props: Parameters[0]) { + return ( + + + + ); +} + +function SubmissionProbeContent({ + owner, + environmentId = THREAD_ENV, +}: { + owner: string; + environmentId?: string; +}) { + const text = mocks.drafts[owner] ?? ""; + const readDraft = () => ({ + text: mocks.drafts[owner] ?? "", + selection: { start: text.length, end: text.length }, + }); + const value = useEnvironmentSpeechInput({ + environmentId: environmentId as EnvironmentId, + ownerKey: owner, + readDraft, + commitDraft: (next) => { + mocks.drafts[owner] = next; + }, + captureSubmissionDraft: () => ({ + readDraft, + commitDraft: (next: string) => { + mocks.drafts[owner] = next; + }, + }), + }); + useLayoutEffect(() => { + mocks.voice = value; + }); + return null; +} + +it("does not retain a send when transcription finishes before the editor rerenders", async () => { + setup({ destination: "env-a" }); + let finish!: (text: string) => void; + mocks.transcripts = [ + new Promise((resolve) => { + finish = resolve; + }), + ]; + mocks.drafts = { "thread-a": "Fix", "thread-b": "" }; + const submit = vi.fn(); + const cancel = vi.fn(); + await using screen = await render(); + await act(() => mocks.voice.start()); + let stopping!: Promise; + await act(() => { + stopping = mocks.voice.stop(); + }); + await vi.waitFor(() => expect(mocks.voice.state.phase).toBe("transcribing")); + const stale = mocks.voice; + await act(async () => { + finish("the login bug"); + await stopping; + expect(mocks.voice).toBe(stale); + stale.submitAfterDictation(submit, cancel); + }); + expect(cancel).toHaveBeenCalledOnce(); + expect(submit).not.toHaveBeenCalled(); + expect(mocks.drafts["thread-a"]).toBe("Fix the login bug"); + await screen.rerender(
); + await screen.rerender(); + await act(() => mocks.voice.start()); + expect(mocks.microphoneRequests).toEqual(["env-a", "env-a"]); + await act(() => mocks.voice.cancel()); +}); + +it.each([ + ["transcribing", "finish"], + ["post-processing", "finish"], + ["post-processing", "cancel"], + ["post-processing", "skip"], + ["transcribing", "finish after session expiry"], +] as const)( + "reattaches a confirmed %s submission after remount to %s it", + async (phase, action) => { + setup({ destination: "env-a" }); + const expires = action === "finish after session expiry"; + if (expires) { + let reads = 0; + mocks.sessionAtoms.set( + "env-a", + Atom.make(Effect.suspend(() => (++reads === 1 ? Effect.succeed(true) : Effect.never))).pipe( + Atom.setIdleTTL(5 * 60_000), + ), + ); + } + let finish!: (text: string) => void; + const result = new Promise((resolve) => { + finish = resolve; + }); + if (phase === "post-processing") { + chooseTranscriptCleanup(true); + mocks.cleanupResult = result; + } else mocks.transcripts = [result]; + await using _response = { + [Symbol.asyncDispose]: () => + act(async () => { + finish("the login bug"); + }), + }; + mocks.drafts = { "thread-a": "Fix", "thread-b": "Do not send" }; + const submit = vi.fn(); + const cancel = vi.fn(); + await using screen = await render(); + using _timers = expires ? fakeTimers() : { [Symbol.dispose]() {} }; + await act(() => mocks.voice.start()); + await act(() => mocks.voice.submitAfterDictation(submit, cancel)); + await vi.waitFor(() => expect(mocks.voice.state.phase).toBe(phase)); + await screen.rerender(
); + if (expires) await act(() => vi.advanceTimersByTimeAsync(6 * 60_000)); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("idle"); + await act(() => mocks.voice.cancel()); + expect(cancel).not.toHaveBeenCalled(); + await act(() => mocks.voice.start()); + expect(mocks.voice.state.phase).toBe("idle"); + expect(mocks.microphoneRequests).toEqual(["env-a"]); + await screen.rerender(
); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("idle"); + await act(() => mocks.voice.cancel()); + expect(cancel).not.toHaveBeenCalled(); + await screen.rerender(
); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe(phase); + expect(mocks.voice.freezesEditor).toBe(true); + expect(mocks.voice.blocksSubmission).toBe(true); + const duplicate = vi.fn(); + await act(() => mocks.voice.submitAfterDictation(duplicate)); + expect(duplicate).not.toHaveBeenCalled(); + if (action === "cancel") await act(() => mocks.voice.cancel()); + if (action === "skip") await act(() => mocks.voice.skipPostProcessing()); + await act(async () => { + finish("the login bug"); + }); + await vi.waitFor(() => expect(mocks.voice.state.phase).toBe("idle")); + if (action === "cancel") { + expect(cancel).toHaveBeenCalledOnce(); + expect(submit).not.toHaveBeenCalled(); + expect(mocks.drafts["thread-a"]).toBe("Fix"); + } else { + const expected = action === "skip" ? "Fix dictated" : "Fix the login bug"; + await vi.waitFor(() => expect(submit).toHaveBeenCalledExactlyOnceWith(expected)); + expect(mocks.drafts["thread-a"]).toBe(expected); + expect(cancel).not.toHaveBeenCalled(); + } + expect(duplicate).not.toHaveBeenCalled(); + expect(mocks.drafts["thread-b"]).toBe("Do not send"); + }, +); + +it.each(["cancel", "off", "destination", "cloud-destination", "cleanup", "error"])( + "does not send a confirmed recording after %s", + async (action) => { + setup({ destination: "env-a" }); + if (action === "cleanup") chooseTranscriptCleanup(true); + let finish!: (text: string) => void; + let fail!: (error: Error) => void; + mocks.transcripts = [ + new Promise((resolve, reject) => { + finish = resolve; + fail = reject; + }), + ]; + const drafts = { "thread-a": "Fix" }; + mocks.drafts = drafts; + const submit = vi.fn(); + const cancel = vi.fn(); + await using screen = await render(); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.submitAfterDictation(submit, cancel)); + await vi.waitFor(() => expect(mocks.transcribeCalls).toEqual(["env-a"])); + if (action === "cancel") await act(() => mocks.voice.cancel()); + else { + await screen.rerender(
); + if (action === "cleanup") chooseTranscriptCleanup(false); + else if (action === "cloud-destination") { + const saved = pendingCloudAudioChoice({ + kind: "cloud", + provider: "openai", + environmentId: EnvironmentId.make("env-b"), + }); + expect(cancel).not.toHaveBeenCalled(); + saved(); + } else if (action !== "error") setDestination(action === "off" ? null : "env-b"); + if (action !== "error") expect(cancel).toHaveBeenCalledOnce(); + } + await act(async () => { + if (action === "error") fail(new Error("Transcription failed")); + else finish("the login bug"); + }); + await vi.waitFor(() => expect(cancel).toHaveBeenCalledOnce()); + expect(submit).not.toHaveBeenCalled(); + expect(drafts["thread-a"]).toBe("Fix"); + await screen.rerender(
); + await screen.rerender(); + expect(mocks.voice.state.phase).toBe("idle"); + expect(mocks.voice.blocksSubmission).toBe(false); + }, +); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 94dd7cfb8b4cf..9ef36e53f52be 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -26,7 +26,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech"; import { useNavigate } from "@tanstack/react-router"; import * as Option from "effect/Option"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState, useSyncExternalStore } from "react"; import { useStableCallback } from "~/hooks/useStableCallback"; import { toastManager } from "../components/ui/toast"; @@ -36,12 +36,18 @@ import { useEnvironmentSettings, } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; -import { useEnvironmentScope, usePreparedConnection } from "../state/session"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import { environmentSession, useEnvironmentScope, usePreparedConnection } from "../state/session"; import { chooseRecordingMicrophone } from "./browserMicrophones"; import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; -import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent"; +import { + bindAudioGrant, + readCleanupGrant, + subscribeVoiceConsent, + useAudioChoice, +} from "./voiceConsent"; const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null }; @@ -85,6 +91,11 @@ function cleanupContext({ text, selection }: DraftInput): DraftInput { }; } +type SubmissionDraft = { + readonly readDraft: () => DraftInput | null; + readonly commitDraft: HookInput["commitDraft"]; +}; + type HookInput = { /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */ readonly environmentId: EnvironmentId; @@ -100,6 +111,8 @@ type HookInput = { readonly offerSetup?: boolean | undefined; /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */ readonly cleanup?: boolean | undefined; + /** Binds a confirmed send to a draft that can be reached after this editor leaves. */ + readonly captureSubmissionDraft?: (() => SubmissionDraft) | undefined; readonly readDraft: () => DraftInput; readonly commitDraft: ( text: string, @@ -107,6 +120,51 @@ type HookInput = { ) => void; }; +type PendingSubmission = { + readonly environmentId: EnvironmentId; + readonly ownerKey: string; + readonly submit: (text: string) => void; + readonly cancel?: (() => void) | undefined; + readonly draft?: SubmissionDraft | undefined; + readonly snapshot?: VoiceDraftSnapshot | undefined; + text?: string; +}; + +type ConfirmedSubmission = { + readonly pending: PendingSubmission; + readonly controller: VoiceInputController; + readonly prepared: PreparedConnection; + readonly grantId: string; + readonly state: VoiceInputState; + readonly cancel: () => void; + readonly releaseSession: () => void; + unsubscribeConsent: () => void; +}; + +// A confirmed send outlives its editor; remounts need its phase and cancellation, not its controller lifecycle. +const confirmedSubmissions = new Map(); +const submissionListeners = new Set<() => void>(); +const submissionKey = (environmentId: EnvironmentId, ownerKey: string) => + JSON.stringify([environmentId, ownerKey]); +const subscribeSubmissions = (notify: () => void) => { + submissionListeners.add(notify); + return () => { + submissionListeners.delete(notify); + }; +}; +const notifySubmissions = () => { + for (const notify of submissionListeners) notify(); +}; +function clearConfirmedSubmission(pending: PendingSubmission) { + const key = submissionKey(pending.environmentId, pending.ownerKey); + const entry = confirmedSubmissions.get(key); + if (entry?.pending !== pending) return; + entry.unsubscribeConsent(); + entry.releaseSession(); + confirmedSubmissions.delete(key); + notifySubmissions(); +} + /** * Microphone audio leaves this device only for the environment the person chose in * Voice settings. Nothing is inferred from the primary or the draft's environment, and @@ -148,6 +206,19 @@ export function useEnvironmentSpeechInput(input: HookInput) { const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; + const retainedSubmission = useSyncExternalStore( + subscribeSubmissions, + () => confirmedSubmissions.get(submissionKey(input.environmentId, input.ownerKey)), + () => undefined, + ); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- revokes retained speech work when the destination connection or this owner's permission changes + useEffect(() => { + for (const entry of confirmedSubmissions.values()) { + if (entry.grantId !== grantId || entry.prepared !== prepared) entry.cancel(); + } + if (input.disabled) + confirmedSubmissions.get(submissionKey(input.environmentId, input.ownerKey))?.cancel(); + }, [prepared, grantId, input.disabled, input.environmentId, input.ownerKey]); const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -171,6 +242,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { prepared, grantId, value: INITIAL_STATE, + ownerKey: input.ownerKey, }); const [queuedStart, setQueuedStart] = useState(null); const [level, setLevel] = useState(0); @@ -181,7 +253,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { disabled: input.disabled, }); if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) { - setControllerState({ prepared, grantId, value: INITIAL_STATE }); + setControllerState({ prepared, grantId, value: INITIAL_STATE, ownerKey: input.ownerKey }); setSetup(INITIAL_SETUP); setQueuedStart(null); setPreview(null); @@ -193,19 +265,31 @@ export function useEnvironmentSpeechInput(input: HookInput) { setSetup(INITIAL_SETUP); } const state: VoiceInputState = - queuedStart === prepared && prepared + retainedSubmission?.state ?? + (queuedStart === prepared && prepared ? WAITING_STATE - : controllerState.prepared === prepared + : controllerState.prepared === prepared && controllerState.ownerKey === input.ownerKey ? controllerState.value - : INITIAL_STATE; + : INITIAL_STATE); const controllerRef = useRef(null); const startRequestRef = useRef(0); - const pendingSubmissionRef = useRef<{ - ownerKey: string; - submit: (text: string) => void; - text?: string; - } | null>(null); + const pendingSubmissionRef = useRef(null); + const cancelSubmission = useStableCallback(() => { + const pending = pendingSubmissionRef.current; + pendingSubmissionRef.current = null; + if (pending) clearConfirmedSubmission(pending); + pending?.cancel?.(); + }); + const lastSnapshotRef = useRef(null); + const mountedRef = useRef(false); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- distinguishes composer unmount from a changed destination in the controller cleanup + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + }; + }, []); const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords }); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller useEffect(() => { @@ -222,13 +306,24 @@ export function useEnvironmentSpeechInput(input: HookInput) { readonly cleanup: SpeechPostProcessingOptions | null; readonly cleanupGrant: string | null; readonly environmentId: EnvironmentId; + readonly target: PreparedConnection | null; } | undefined; - const readDraft = (): VoiceDraftSnapshot => { + const readDraft = (): VoiceDraftSnapshot | null => { + const pending = pendingSubmissionRef.current; + if (pending?.draft && pending.snapshot) { + const draft = pending.draft.readDraft(); + return draft && { ...pending.snapshot, text: draft.text }; + } const current = latestRef.current.input; const draft = current.readDraft(); - return { ownerKey: current.ownerKey, text: draft.text, selection: draft.selection }; + lastSnapshotRef.current = { + ownerKey: current.ownerKey, + text: draft.text, + selection: draft.selection, + }; + return lastSnapshotRef.current; }; const platform = createBrowserVoiceInputPlatform({ prepared, @@ -253,6 +348,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { recordingPreferences = { cleanupGrant, environmentId: current.environmentId, + target: latestRef.current.cleanupPrepared, cleanup: cleanupEnabled ? { speechCustomWords, @@ -285,14 +381,17 @@ export function useEnvironmentSpeechInput(input: HookInput) { options, ) => { const preferences = recordingPreferences; - const { input: current, cleanupPrepared: target } = latestRef.current; + const { input: current, cleanupPrepared } = latestRef.current; + const submission = pendingSubmissionRef.current; + const target = submission?.draft ? preferences?.target : cleanupPrepared; if ( !preferences?.cleanup || !hasAudioConsent() || preferences.cleanupGrant !== readCleanupGrant() || - current.cleanup === false || - current.environmentId !== preferences.environmentId || - current.ownerKey !== options.draft.ownerKey + (!submission?.draft && + (current.cleanup === false || + current.environmentId !== preferences.environmentId || + current.ownerKey !== options.draft.ownerKey)) ) return null; if (!target) @@ -327,27 +426,74 @@ export function useEnvironmentSpeechInput(input: HookInput) { }), readDraft, commitDraft: (text, selection) => { - if (latestRef.current.input.disabled) return; - latestRef.current.input.commitDraft(text, selection); - if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text; + const pending = pendingSubmissionRef.current; + if (pending?.draft) { + if ( + !hasAudioConsent() || + (recordingPreferences?.cleanupGrant && + recordingPreferences.cleanupGrant !== readCleanupGrant()) + ) { + cancelSubmission(); + return; + } + pending.draft.commitDraft(text, selection); + } else { + if (latestRef.current.input.disabled) return; + latestRef.current.input.commitDraft(text, selection); + } + if (pending) pending.text = text; }, onStateChange: (value) => { - if (disposed) return; - if (value.phase === "error") pendingSubmissionRef.current = null; - setControllerState({ prepared, grantId, value }); + const pending = pendingSubmissionRef.current; + if (value.phase === "error") { + cancelSubmission(); + if (pending?.draft) { + toastManager.add({ + type: "error", + title: "Voice message was not sent", + description: value.error, + }); + } + } + if (value.phase === "idle" && pending?.draft) { + clearConfirmedSubmission(pending); + pendingSubmissionRef.current = null; + if (pending.text !== undefined) pending.submit(pending.text); + else pending.cancel?.(); + } + if (pending?.draft && value.phase !== "idle" && value.phase !== "error") { + const key = submissionKey(pending.environmentId, pending.ownerKey); + const entry = confirmedSubmissions.get(key); + if (entry?.pending === pending) { + confirmedSubmissions.set(key, { ...entry, state: value }); + notifySubmissions(); + } + } + if (disposed) { + if (value.phase === "error") controller.dispose(); + return; + } + setControllerState({ + prepared, + grantId, + value, + ownerKey: lastSnapshotRef.current?.ownerKey ?? latestRef.current.input.ownerKey, + }); if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null); }, }); controllerRef.current = controller; return () => { disposed = true; - pendingSubmissionRef.current = null; startRequestRef.current += 1; setQueuedStart(null); - controller.dispose(); + if (mountedRef.current || !pendingSubmissionRef.current?.draft) { + cancelSubmission(); + controller.dispose(); + } if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider, cloudModel]); + }, [prepared, grantId, cloudProvider, cloudModel, cancelSubmission]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { @@ -393,35 +539,86 @@ export function useEnvironmentSpeechInput(input: HookInput) { }, [prepared, setup.downloading, setup.open, setupModelId]); const previousOwnerRef = useRef(input.ownerKey); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- cancels unsent dictation on navigation while a captured submission keeps its original draft useEffect(() => { if (previousOwnerRef.current === input.ownerKey && !input.disabled) return; previousOwnerRef.current = input.ownerKey; - pendingSubmissionRef.current = null; startRequestRef.current += 1; + const pending = pendingSubmissionRef.current; + if (pending?.draft && (pending.ownerKey !== input.ownerKey || !input.disabled)) return; + cancelSubmission(); controllerRef.current?.ownerChanged(); - }, [input.ownerKey, input.disabled]); + }, [input.ownerKey, input.disabled, cancelSubmission]); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- editor-owned submissions wait for React to commit the inserted transcript useEffect(() => { const pending = pendingSubmissionRef.current; - if (state.phase !== "idle" || !pending) return; + if (state.phase !== "idle" || !pending || pending.draft) return; pendingSubmissionRef.current = null; if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined) pending.submit(pending.text); + else pending.cancel?.(); }, [state, input.ownerKey, input.disabled]); - const submitAfterDictation = useStableCallback((submit: (text: string) => void) => { - if (latestRef.current.input.disabled || state.phase === "preparing") return; - const controller = controllerRef.current; - if (!controller || !voiceInputBlocksSubmission(state)) { - submit(latestRef.current.input.readDraft().text); - return; - } - if (pendingSubmissionRef.current) return; - pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit }; - if (controller.currentState.phase === "recording") void controller.stop(); - }); + const submitAfterDictation = useStableCallback( + (submit: (text: string) => void, cancel?: () => void) => { + const current = latestRef.current.input; + if ( + current.disabled || + state.phase === "preparing" || + pendingSubmissionRef.current || + confirmedSubmissions.has(submissionKey(current.environmentId, current.ownerKey)) + ) { + cancel?.(); + return; + } + const controller = controllerRef.current; + if (!controller || !voiceInputBlocksSubmission(state)) { + submit(current.readDraft().text); + return; + } + if (!voiceInputBlocksSubmission(controller.currentState)) { + cancel?.(); + return; + } + const pending: PendingSubmission = { + environmentId: current.environmentId, + ownerKey: current.ownerKey, + submit, + cancel, + draft: current.captureSubmissionDraft?.(), + snapshot: lastSnapshotRef.current ?? undefined, + }; + pendingSubmissionRef.current = pending; + if (pending.draft && prepared && grantId) { + const entry: ConfirmedSubmission = { + pending, + controller, + prepared, + grantId, + state: controller.currentState, + // Remounts must not mistake an expired permission cache for revoked access. + releaseSession: appAtomRegistry.mount( + environmentSession.sessionStateAtom(prepared.environmentId), + ), + cancel: () => { + cancelSubmission(); + controller.dispose(); + }, + unsubscribeConsent: () => {}, + }; + confirmedSubmissions.set(submissionKey(current.environmentId, current.ownerKey), entry); + const hasConsent = bindAudioGrant(grantId); + const cleanupGrant = current.cleanup === false ? null : readCleanupGrant(); + entry.unsubscribeConsent = subscribeVoiceConsent(() => { + if (!hasConsent() || (cleanupGrant && cleanupGrant !== readCleanupGrant())) + entry.cancel(); + }); + notifySubmissions(); + } + if (controller.currentState.phase === "recording") void controller.stop(); + }, + ); const loadSetupCatalog = useStableCallback( async (connection: PreparedConnection, selectedModelId: string, request: number) => { @@ -451,6 +648,10 @@ export function useEnvironmentSpeechInput(input: HookInput) { const start = useStableCallback(async () => { if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return; + if (confirmedSubmissions.size > 0 || pendingSubmissionRef.current) { + toastManager.add({ type: "info", title: "Finishing the previous voice message" }); + return; + } if (!grantId) { if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false) await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" }); @@ -601,14 +802,32 @@ export function useEnvironmentSpeechInput(input: HookInput) { freezesEditor: voiceInputBlocksSubmission(state), start, submitAfterDictation, - stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()), + stop: useStableCallback( + () => + ( + confirmedSubmissions.get(submissionKey(input.environmentId, input.ownerKey)) + ?.controller ?? controllerRef.current + )?.stop() ?? Promise.resolve(), + ), cancel: useStableCallback(() => { - pendingSubmissionRef.current = null; + const retained = confirmedSubmissions.get(submissionKey(input.environmentId, input.ownerKey)); + if (retained) { + retained.cancel(); + return; + } + const pending = pendingSubmissionRef.current; + if (pending?.draft && pending.ownerKey !== latestRef.current.input.ownerKey) return; + cancelSubmission(); startRequestRef.current += 1; setQueuedStart(null); setSetup(INITIAL_SETUP); controllerRef.current?.cancel(); }), - skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()), + skipPostProcessing: useStableCallback(() => + ( + confirmedSubmissions.get(submissionKey(input.environmentId, input.ownerKey))?.controller ?? + controllerRef.current + )?.skipPostProcessing(), + ), }; } diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 7c897fbc37ea5..fb63e0f2c9558 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -110,6 +110,27 @@ export function bindAudioGrant(id: string) { } } +/** Observes revocation even while the recording's editor is unmounted. */ +export function subscribeVoiceConsent(onChange: () => void) { + let stopPending = () => {}; + const followPending = () => { + stopPending(); + const pending = pendingGrant(readChoice()); + stopPending = pending ? subscribeLocalStorageKey(savedKey(pending.id), onChange) : () => {}; + }; + followPending(); + const stopAudio = subscribeLocalStorageKey(AUDIO_KEY, () => { + followPending(); + onChange(); + }); + const stopCleanup = subscribeLocalStorageKey(CLEANUP_KEY, onChange); + return () => { + stopAudio(); + stopCleanup(); + stopPending(); + }; +} + export function useCleanupGrant() { return useLocalStorage(CLEANUP_KEY, null, CleanupGrant)[0]; } diff --git a/apps/web/test/fakeEnvironment.ts b/apps/web/test/fakeEnvironment.ts index 24aba2f822ea1..4e11cf189733b 100644 --- a/apps/web/test/fakeEnvironment.ts +++ b/apps/web/test/fakeEnvironment.ts @@ -63,7 +63,11 @@ const descriptor = { // A matching version, since a mismatch adds an update banner a typical user never sees. serverVersion: APP_VERSION, orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION, - capabilities: { repositoryIdentity: true }, + capabilities: { + repositoryIdentity: true, + voiceTranscription: true, + fileAttachments: { maxUploadBytes: 1000000 }, + }, }; const auth = { policy: "loopback-browser", @@ -385,6 +389,8 @@ const subscribe = (first: ReadonlyArray, register?: (queue: Queue.Queue ); const handlers: Record unknown> = { + [WS_METHODS.assetsCreateUrl]: () => + Effect.succeed({ relativeUrl: "/fixtures/report.txt", expiresAt: 4102444800000 }), // Recorded, and accepted only when the scenario asks, since a pending command holds every // later command on its thread. Accepted commands' events never arrive. [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: (command: OrchestrationV2Command) =>