diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index ef4b696c82ec8..b52bbac02cb58 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -514,6 +514,7 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string } : current, ); const voiceInput = useVoiceInputController({ + environmentId: editor?.environmentId ?? null, ownerKey, label: editor?.draft.title.trim() || title, subscribeToDraftChanges: (onChange) => diff --git a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx index 5f7328214cb16..f5f67bf03b6c7 100644 --- a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx @@ -24,6 +24,8 @@ import * as Dialog from "../../components/ui/Dialog"; import { useEnvironments, type EnvironmentPresentation } from "../../state/environments"; import { chooseVoiceDestination, + chooseTranscriptCleanup, + voiceCleanupGrantAtom, voiceConnectionAtom, voiceGrantAtom, voiceSpeechStatusAtom, @@ -37,6 +39,7 @@ import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActi import { SettingsDragHandle } from "./components/SettingsDragHandle"; import { SettingsScreen } from "./components/SettingsScreen"; import { SettingsSection } from "./components/SettingsSection"; +import { SettingsSwitchRow } from "./components/SettingsSwitchRow"; import { deviceMicrophoneOrder, localVoiceTranscriberAtom, @@ -169,6 +172,7 @@ function EnvironmentChoice({ export function SettingsVoiceRouteScreen() { const transcriber = useAtomValue(localVoiceTranscriberAtom); const grant = useAtomValue(voiceGrantAtom); + const cleanupGrant = useAtomValue(voiceCleanupGrantAtom); const { environments, isReady } = useEnvironments(); const grantEnvironment = grant ? environments.find( @@ -259,6 +263,34 @@ export function SettingsVoiceRouteScreen() { /> ) : null} + + { + const save = () => { + try { + chooseTranscriptCleanup(enabled); + } catch { + Dialog.alert( + "Could not save transcript cleanup choice", + "Cleanup is off for this session. Save again before closing the app, or your previous choice may return when it restarts.", + ); + } + }; + if (enabled) + Dialog.confirm({ + title: "Clean up transcripts?", + message: + "Your transcript and nearby draft text will be sent to the cleanup model on the draft’s environment. Depending on the model, this may use an external service. Audio is not sent for cleanup. You can turn this off at any time.", + confirmText: "Enable cleanup", + onConfirm: save, + }); + else save(); + }} + /> + {hasEnvironmentVoiceCapture && environments .filter( diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 07429b5439726..5f66b3c53afe1 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -572,6 +572,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps offersSideChat: props.offersSideChat === true, }); const voiceInput = useVoiceInputController({ + environmentId: props.environmentId, ownerKey: composerDraftKey, label: props.selectedThread.title || "Untitled thread", readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text, diff --git a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx index 6e891dd04e7e9..0deb41d58d212 100644 --- a/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx +++ b/apps/mobile/src/features/voice-input/VoiceInputProvider.tsx @@ -18,14 +18,24 @@ import { createVoiceRecorder } from "../../native/voiceRecorder"; import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences"; import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene"; import { + postProcessEnvironmentTranscript, voiceInputBlocksSubmission, + type VoiceDraftSnapshot, type VoiceTranscriber, } from "@t3tools/client-runtime/voice-input"; import { createLazyVoiceRecorder } from "./lazyVoiceRecorder"; import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; import { VoiceInputSession } from "./voiceInputSession"; import { createVoiceRecordingPermission } from "./voiceRecordingPermission"; -import { voiceRouteAtom } from "./voiceDestination"; +import { + runSpeech, + voiceCleanupGrantAtom, + voiceConnectionAtom, + voiceGrantAtom, + voiceRouteAtom, +} from "./voiceDestination"; +import * as Dialog from "../../components/ui/Dialog"; +import type { VoiceGrant } from "./voiceGrantStorage"; import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom"; import { acquireAudioSession, @@ -55,6 +65,18 @@ function keepScreenAwake(tag: string): () => void { export const deviceMicrophoneOrder = (saved?: ReadonlyArray) => microphoneOrder(saved, Platform.OS === "android" ? "android" : "ios"); +function cleanupContext({ text, selection }: VoiceDraftSnapshot) { + const limit = 1_024; + const start = Math.max(0, Math.min(selection.start, text.length)); + const end = Math.max(start, Math.min(selection.end, text.length)); + const before = text.slice(Math.max(0, start - limit), start); + const selected = text.slice(start, Math.min(end, start + limit)); + return { + text: before + selected + text.slice(end, end + limit), + selection: { start: before.length, end: before.length + selected.length }, + }; +} + /** "checking" until the first availability probe settles; re-probed on foreground. */ export const localVoiceTranscriberAtom = makePermissionStatusAtom< VoiceTranscriber | null | "checking" @@ -80,11 +102,17 @@ const voiceInputAtom = Atom.make((get) => { // Mounted so the recorder can read and remember microphones synchronously. get.mount(mobilePreferencesAtom); get.mount(updateMobilePreferencesAtom); + get.mount(voiceCleanupGrantAtom); const audioLevels = makeMutable>( Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0), ); let keepAwakeSequence = 0; let audioToken: ReturnType | null = null; + let recordingCleanup: { + readonly grant: VoiceGrant; + readonly cleanupGrant: string; + readonly ownerKey: string; + } | null = null; const permission = createVoiceRecordingPermission(() => { void session.controller.appMovedToBackground(); }); @@ -121,11 +149,56 @@ const voiceInputAtom = Atom.make((get) => { const session: VoiceInputSession = new VoiceInputSession({ recorder, getTranscriber: () => { + const grant = registry.get(voiceGrantAtom); + const cleanupGrant = registry.get(voiceCleanupGrantAtom); + const ownerKey = session.ownerKey; + recordingCleanup = + grant?.destination.kind === "environment" && + grant.destination.environmentId === session.environmentId && + cleanupGrant && + ownerKey + ? { grant, cleanupGrant, ownerKey } + : null; const route = registry.get(voiceRouteAtom); if (route) return route.transcriber; const transcriber = registry.get(localVoiceTranscriberAtom); return transcriber === "checking" ? null : transcriber; }, + postProcess: (transcript, { draft, signal }) => { + const preferences = recordingCleanup; + const canCleanUp = () => + preferences !== null && + registry.get(voiceGrantAtom) === preferences.grant && + registry.get(voiceCleanupGrantAtom) === preferences.cleanupGrant && + preferences.grant.destination.kind === "environment" && + preferences.grant.destination.environmentId === session.environmentId && + session.ownerKey === preferences.ownerKey && + draft.ownerKey === preferences.ownerKey; + if (!preferences || !canCleanUp()) return null; + const connectionAtom = voiceConnectionAtom(preferences.grant.destination.environmentId); + const prepared = registry.get(connectionAtom); + if (!prepared) + return Promise.reject( + new Error("The draft's environment is unavailable for transcript cleanup."), + ); + const canSend = () => canCleanUp() && registry.get(connectionAtom) === prepared; + return runSpeech( + postProcessEnvironmentTranscript( + prepared, + transcript, + cleanupContext(draft), + { + speechCustomWords: [], + speechPostProcessingPrompt: { mode: "default", customInstructions: "" }, + }, + canSend, + ), + signal, + canSend, + ).then((result) => (canSend() ? result.text : transcript)); + }, + onPostProcessingError: () => + Dialog.alert("Transcript cleanup failed", "The original transcript was added."), requestPermission: permission.requestPermission, configureRecording: async () => { const token = acquireAudioSession("recording"); diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts index e26672b5b4809..5098a71fd8d79 100644 --- a/apps/mobile/src/features/voice-input/useVoiceInputController.ts +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -6,12 +6,14 @@ import { } from "@t3tools/client-runtime/voice-input"; import type { ComposerEditorSelection } from "../../components/ComposerEditor"; +import type { EnvironmentId } from "@t3tools/contracts"; import { useGlobalVoiceInput } from "./VoiceInputProvider"; import { createVoiceInputTarget } from "./voiceInputSession"; const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; export function useVoiceInputController(input: { + readonly environmentId: EnvironmentId | null; readonly ownerKey: string | null; readonly label: string; readonly readDraftMessage: () => string | null; @@ -48,6 +50,7 @@ export function useVoiceInputController(input: { const captured = latestInput.current; if (!captured.ownerKey || captured.disabled) return; void session.start({ + environmentId: captured.environmentId, ...createVoiceInputTarget( captured.ownerKey, captured.readDraftMessage, diff --git a/apps/mobile/src/features/voice-input/voiceCleanupStorage.ts b/apps/mobile/src/features/voice-input/voiceCleanupStorage.ts new file mode 100644 index 0000000000000..d579168171e15 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceCleanupStorage.ts @@ -0,0 +1,37 @@ +import * as Schema from "effect/Schema"; +import { File, Paths } from "expo-file-system"; +import * as SecureStore from "expo-secure-store"; +import { whenProtectedDataAvailable } from "../../lib/protectedData"; + +const key = "t3code.voice.cleanup-grant.v1"; +const deviceOnly = { keychainAccessible: SecureStore.WHEN_UNLOCKED_THIS_DEVICE_ONLY }; +const markerFile = () => new File(Paths.document, "voice-cleanup-grant-id"); +const decodeGrant = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.NullOr(Schema.NonEmptyString)), +); + +export async function readVoiceCleanupGrant() { + await whenProtectedDataAvailable(); + try { + const raw = await SecureStore.getItemAsync(key, deviceOnly); + const grant = raw ? decodeGrant(raw) : null; + return grant && markerFile().textSync() === grant ? grant : null; + } catch { + return null; + } +} + +export function writeVoiceCleanupGrant(grant: string | null) { + try { + markerFile().writeSync(grant ?? ""); + } catch (cause) { + SecureStore.setItem(key, "null", deviceOnly); + if (grant) throw cause; + return; + } + try { + SecureStore.setItem(key, JSON.stringify(grant), deviceOnly); + } catch (cause) { + if (grant) throw cause; + } +} diff --git a/apps/mobile/src/features/voice-input/voiceDestination.test.ts b/apps/mobile/src/features/voice-input/voiceDestination.test.ts index dca713c3975c1..28de9ee8fa209 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.test.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.test.ts @@ -1,14 +1,20 @@ import { EnvironmentId } from "@t3tools/contracts"; -import { Atom } from "effect/reactivity"; +import { Atom, AsyncResult } from "effect/reactivity"; import { expect, it, vi } from "vite-plus/test"; import { appAtomRegistry } from "../../state/atom-registry"; import { chooseVoiceDestination, + chooseTranscriptCleanup, pendingCloudVoiceChoice, voiceGrantAtom, voiceRouteAtom, + voiceCleanupGrantAtom, } from "./voiceDestination"; import { readVoiceGrant } from "./voiceGrantStorage"; +import { readVoiceCleanupGrant } from "./voiceCleanupStorage"; +import { useGlobalVoiceInput, localVoiceTranscriberAtom } from "./VoiceInputProvider"; +import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; +import type { VoiceInputTarget } from "./voiceInputSession"; const mocks = vi.hoisted(() => ({ read: vi.fn<() => Promise>(async () => null), @@ -17,7 +23,57 @@ const mocks = vi.hoisted(() => ({ markerWrite: vi.fn<(id: string) => void>(), probe: vi.fn(), fetch: vi.fn(), + alert: vi.fn(), })); +// Drive the production provider atom without mounting a native React tree. +vi.mock("@effect/atom-react", () => ({ + useAtomValue: (atom: Atom.Atom) => appAtomRegistry.get(atom), +})); +vi.mock("react", () => ({ + useSyncExternalStore: (_subscribe: unknown, getSnapshot: () => unknown) => getSnapshot(), +})); +vi.mock("react-native", () => ({ + Platform: { OS: "android" }, + AppState: { currentState: "active", addEventListener: () => ({ remove() {} }) }, +})); +vi.mock("react-native-reanimated", () => ({ + makeMutable: (value: unknown) => ({ value, set() {} }), +})); +vi.mock("expo-keep-awake", () => ({ + activateKeepAwakeAsync: async () => {}, + deactivateKeepAwake: () => {}, +})); +vi.mock("expo-audio", () => ({ + RecordingPresets: { HIGH_QUALITY: { ios: {}, android: {} } }, + requestRecordingPermissionsAsync: async () => ({ granted: true, canAskAgain: true }), + setAudioModeAsync: async () => {}, + setIsAudioActiveAsync: async () => {}, + AudioModule: { + AudioRecorder: class { + uri = "file:///voice.m4a"; + async prepareToRecordAsync() {} + record() {} + async stop() {} + getStatus() { + return { isRecording: false }; + } + addListener() { + return { remove() {} }; + } + getAvailableInputs() { + return []; + } + release() {} + }, + }, +})); +vi.mock("../../state/preferences", () => ({ + mobilePreferencesAtom: Atom.make(AsyncResult.success({})), + updateMobilePreferencesAtom: Atom.make({}), +})); +vi.mock("../../components/ui/Dialog", () => ({ alert: mocks.alert })); +vi.mock("../showcase/nativeShowcaseScene", () => ({ getNativeShowcaseScene: () => null })); +vi.mock("../../native/voiceTranscription", () => ({ getLocalVoiceTranscriber: async () => null })); vi.mock("expo", () => ({ requireOptionalNativeModule: () => ({ decodeRecording: async () => new Uint8Array([0, 0]) }), })); @@ -27,6 +83,7 @@ vi.mock("expo-file-system", () => ({ File: class { textSync = mocks.markerRead; writeSync = mocks.markerWrite; + delete() {} write(id: string) { return Promise.resolve().then(() => mocks.markerWrite(id)); } @@ -79,52 +136,63 @@ vi.mock("../../state/session", async () => { }; }); -it.each(["secure store", "marker"])( - "revokes across restarts when writing the %s fails", - async (failedStore) => { - let saved: string | null = null; - let marker = ""; - mocks.read.mockImplementation(async () => saved); - mocks.write.mockImplementation((_key: string, value: string) => { - saved = value; +it.each([ + ["audio", "secure store"], + ["audio", "marker"], + ["cleanup", "secure store"], + ["cleanup", "marker"], +])("revokes %s consent across restarts when writing the %s fails", async (kind, failedStore) => { + let saved: string | null = null; + let marker = ""; + mocks.read.mockImplementation(async () => saved); + mocks.write.mockImplementation((_key: string, value: string) => { + saved = value; + }); + mocks.markerRead.mockImplementation(() => marker); + mocks.markerWrite.mockImplementation((id) => { + marker = id; + }); + const host = EnvironmentId.make("my-server"); + const choose = (enabled: boolean) => + kind === "audio" + ? chooseVoiceDestination(enabled ? host : null) + : chooseTranscriptCleanup(enabled); + const read = () => (kind === "audio" ? readVoiceGrant() : readVoiceCleanupGrant()); + const current = () => + kind === "audio" + ? appAtomRegistry.get(voiceGrantAtom) + : appAtomRegistry.get(voiceCleanupGrantAtom); + try { + choose(true); + expect(marker).toBe(kind === "audio" ? appAtomRegistry.get(voiceGrantAtom)?.id : current()); + expect(await read()).not.toBeNull(); + mocks.markerRead.mockReturnValueOnce(""); + expect(await read()).toBeNull(); + const old = { saved, marker }; + const failedWrite = failedStore === "secure store" ? mocks.write : mocks.markerWrite; + failedWrite.mockImplementationOnce(() => { + throw new Error("storage unavailable"); }); - mocks.markerRead.mockImplementation(() => marker); - mocks.markerWrite.mockImplementation((id) => { - marker = id; + choose(false); + expect(failedStore === "secure store" ? saved : marker).toBe( + failedStore === "secure store" ? old.saved : old.marker, + ); + expect(await read()).toBeNull(); + choose(true); + failedWrite.mockImplementationOnce(() => { + throw new Error("storage unavailable"); }); - const host = EnvironmentId.make("my-server"); - try { - chooseVoiceDestination(host); - expect(marker).toBe(appAtomRegistry.get(voiceGrantAtom)?.id); - expect((await readVoiceGrant())?.destination.environmentId).toBe(host); - mocks.markerRead.mockReturnValueOnce(""); - expect(await readVoiceGrant()).toBeNull(); - const old = { saved, marker }; - const failedWrite = failedStore === "secure store" ? mocks.write : mocks.markerWrite; - failedWrite.mockImplementationOnce(() => { - throw new Error("storage unavailable"); - }); - chooseVoiceDestination(null); - expect(failedStore === "secure store" ? saved : marker).toBe( - failedStore === "secure store" ? old.saved : old.marker, - ); - expect(await readVoiceGrant()).toBeNull(); - chooseVoiceDestination(host); - failedWrite.mockImplementationOnce(() => { - throw new Error("storage unavailable"); - }); - expect(() => chooseVoiceDestination(EnvironmentId.make("other-host"))).toThrow(); - expect(appAtomRegistry.get(voiceGrantAtom)).toBeNull(); - expect(await readVoiceGrant()).toBeNull(); - } finally { - mocks.read.mockReset(); - mocks.write.mockReset(); - mocks.markerRead.mockReset(); - mocks.markerWrite.mockReset(); - appAtomRegistry.reset(); - } - }, -); + expect(() => choose(true)).toThrow(); + expect(current()).toBeNull(); + expect(await read()).toBeNull(); + } finally { + mocks.read.mockReset(); + mocks.write.mockReset(); + mocks.markerRead.mockReset(); + mocks.markerWrite.mockReset(); + appAtomRegistry.reset(); + } +}); it("keeps local as the default and never reuses consent for an offline or reselected host", async () => { const unmount = appAtomRegistry.mount(voiceRouteAtom); @@ -257,3 +325,144 @@ it("a late cloud key save cannot override a newer destination choice", () => { appAtomRegistry.reset(); } }); + +it.each([ + "off", + "enabled", + "failure", + "different environment", + "different owner", + "local", + "cloud", + "revoked", + "re-enabled", + "enabled after start", + "bounded context", + "revoked during cleanup", + "cancelled during cleanup", +] as const)("dictation through the mobile provider with cleanup %s", async (scenario) => { + mocks.fetch.mockReset(); + mocks.alert.mockClear(); + const host = EnvironmentId.make("my-server"); + const raw = "um ship twenty four widgets sorry forty two widgets"; + const cleaned = "Ship 42 widgets."; + const bounded = scenario === "bounded context"; + const context = bounded + ? `${"a".repeat(2047)} ${"r".repeat(2048)} ${"z".repeat(2047)}` + : "before replace after"; + const selection = bounded ? { start: 2048, end: 4096 } : { start: 7, end: 14 }; + const commit = vi.fn(); + const cleanup = Promise.withResolvers(); + const cleanupEntered = Promise.withResolvers(); + const requests: { transcript: string; draft: unknown; options: unknown }[] = []; + mocks.fetch.mockImplementation(async (url, init) => { + const path = new URL(String(url)).pathname; + if (path === "/api/voice/status") + return Response.json({ + supported: true, + state: "ready", + modelId: "fixture", + model: "Fixture", + size: 1, + supportsStreaming: false, + supportsTranslation: false, + acceleration: "cpu", + modelUnloadTimeout: "never", + gpuDevices: [], + }); + if (path === "/api/voice/cloud/keys") return Response.json({ openai: true, xai: true }); + if (path.endsWith("/transcribe")) return Response.json({ text: raw }); + requests.push(JSON.parse(String(init?.body))); + cleanupEntered.resolve(); + return cleanup.promise; + }); + chooseTranscriptCleanup(scenario !== "off" && scenario !== "enabled after start"); + chooseVoiceDestination( + scenario === "local" ? null : host, + scenario === "cloud" ? "openai" : undefined, + ); + const { session } = useGlobalVoiceInput(); + appAtomRegistry.set(localVoiceTranscriberAtom, { + prepare: async () => ({ locale: "en-US", transcribe: async () => raw }), + }); + const target: VoiceInputTarget = { + ownerKey: "draft-owner", + environmentId: scenario === "different environment" ? EnvironmentId.make("other-server") : host, + readDraft: () => ({ + ownerKey: scenario === "different owner" ? "another-owner" : "draft-owner", + text: context, + selection, + revision: 0, + }), + commitDraft: commit, + subscribe: () => () => {}, + }; + try { + await session.start(target); + expect(session.controller.currentState.phase).toBe("recording"); + if (scenario === "revoked" || scenario === "re-enabled") chooseTranscriptCleanup(false); + if (scenario === "re-enabled" || scenario === "enabled after start") + chooseTranscriptCleanup(true); + const stopping = session.controller.stop(); + const runsCleanup = + scenario === "enabled" || + scenario === "failure" || + bounded || + scenario === "revoked during cleanup" || + scenario === "cancelled during cleanup"; + if (runsCleanup) { + await Promise.race([cleanupEntered.promise, stopping]); + expect(session.getSnapshot().state.phase).toBe("post-processing"); + expect(resolveVoiceComposerPresentation(session.getSnapshot().state, 0).statusLabel).toBe( + "Cleaning up transcript", + ); + expect(commit).not.toHaveBeenCalled(); + if (scenario === "revoked during cleanup") chooseTranscriptCleanup(false); + if (scenario === "cancelled during cleanup") session.cancel(target.ownerKey); + if (scenario === "failure") cleanup.reject(new Error("provider unavailable")); + else cleanup.resolve(Response.json({ text: cleaned })); + } + await stopping; + const inserted = scenario === "enabled" || bounded ? cleaned : raw; + if (scenario === "cancelled during cleanup") expect(commit).not.toHaveBeenCalled(); + else + expect(commit).toHaveBeenCalledExactlyOnceWith( + `${context.slice(0, selection.start)}${inserted}${context.slice(selection.end)}`, + { + start: selection.start + inserted.length, + end: selection.start + inserted.length, + }, + ); + expect(requests).toEqual( + runsCleanup + ? [ + { + transcript: raw, + draft: bounded + ? { + text: `${"a".repeat(1023)} ${"r".repeat(1024)} ${"z".repeat(1023)}`, + selection: { start: 1024, end: 2048 }, + } + : { text: context, selection }, + options: { + speechCustomWords: [], + speechPostProcessingPrompt: { mode: "default", customInstructions: "" }, + }, + }, + ] + : [], + ); + if (runsCleanup) { + const [url, init] = mocks.fetch.mock.lastCall!; + expect(String(url)).toBe("https://my-server.test/api/voice/post-process"); + expect(init?.redirect).toBe("error"); + } + expect(mocks.alert).toHaveBeenCalledTimes(scenario === "failure" ? 1 : 0); + expect(session.controller.currentState.phase).toBe("idle"); + } finally { + session.dispose(); + appAtomRegistry.reset(); + mocks.fetch.mockReset(); + mocks.alert.mockClear(); + } +}); diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index dd27cca674888..b180f9b6e0b47 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -32,6 +32,22 @@ import { connectionAtomRuntime } from "../../connection/runtime"; import { uuidv4 } from "../../lib/uuid"; import { createEnvironmentTranscriber } from "./environmentTranscriber"; import { readVoiceGrant, writeVoiceGrant, type VoiceGrant } from "./voiceGrantStorage"; +import { readVoiceCleanupGrant, writeVoiceCleanupGrant } from "./voiceCleanupStorage"; + +const storedCleanupGrant = Atom.make(Effect.promise(readVoiceCleanupGrant)).pipe(Atom.keepAlive); +const cleanupChoice = Atom.make(undefined).pipe(Atom.keepAlive); +export const voiceCleanupGrantAtom = Atom.make((get) => { + const chosen = get(cleanupChoice); + if (chosen !== undefined) return chosen; + return AsyncResult.getOrElse(get(storedCleanupGrant), () => null); +}); + +export function chooseTranscriptCleanup(enabled: boolean) { + appAtomRegistry.set(cleanupChoice, null); + const grant = enabled ? uuidv4() : null; + writeVoiceCleanupGrant(grant); + appAtomRegistry.set(cleanupChoice, grant); +} const storedGrant = Atom.make(Effect.promise(readVoiceGrant)).pipe(Atom.keepAlive); const choice = Atom.make(undefined).pipe(Atom.keepAlive); diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts index 7433b930014eb..46ccaaec75827 100644 --- a/apps/mobile/src/features/voice-input/voiceInputPresentation.ts +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts @@ -59,7 +59,7 @@ export function resolveVoiceComposerPresentation( trailingAction: "confirm", showsSend: false, statusKind: "active", - statusLabel: "Transcribing", + statusLabel: state.phase === "post-processing" ? "Cleaning up transcript" : "Transcribing", confirmationEnabled: false, }; } diff --git a/apps/mobile/src/features/voice-input/voiceInputSession.ts b/apps/mobile/src/features/voice-input/voiceInputSession.ts index 5bf9395daf59c..60c08e9bdb72f 100644 --- a/apps/mobile/src/features/voice-input/voiceInputSession.ts +++ b/apps/mobile/src/features/voice-input/voiceInputSession.ts @@ -7,6 +7,7 @@ import { type VoiceInputPhase, type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; +import type { EnvironmentId } from "@t3tools/contracts"; import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; @@ -16,6 +17,7 @@ const SILENT_LEVELS: ReadonlyArray = Array(VOICE_WAVEFORM_SAMPLE export type VoiceInputTarget = { readonly ownerKey: string; + readonly environmentId?: EnvironmentId | null; readonly label?: string; readonly readDraft: () => VoiceDraftSnapshot | null; readonly commitDraft: VoiceInputControllerDependencies["commitDraft"]; @@ -111,6 +113,10 @@ export class VoiceInputSession { return this.target?.ownerKey ?? null; } + get environmentId(): EnvironmentId | null { + return this.target?.environmentId ?? null; + } + get label(): string | null { return this.target?.label ?? null; } diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 7e5c36fa4d1eb..e53f797c822da 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -125,6 +125,10 @@ Cleanup uses the environment's text generation model unless you pick a It can be a text generator instead of a coding agent; see [text generation models](./text-generation.md). +On mobile, enable cleanup separately in **Settings → Voice → Cleanup**. It runs +only when **Transcribe on** selects the draft's own environment; device-local +and cloud dictation skip it. Mobile uses the built-in cleanup instructions. + If cleanup misses the way you correct yourself, set a **Correction cue**: the word or phrase you say to flag a correction, such as "scratch that". Cleanup applies the correction only where the context shows one and drops the cue, and