Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
import { VoiceInputSession } from "./voiceInputSession";
import { createVoiceRecordingPermission } from "./voiceRecordingPermission";
import { voiceRouteAtom } from "./voiceDestination";
import { voiceGrantAtom, voiceRouteAtom } from "./voiceDestination";
import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom";
import {
acquireAudioSession,
Expand Down Expand Up @@ -119,6 +119,7 @@ const voiceInputAtom = Atom.make((get) => {
},
});
const session: VoiceInputSession = new VoiceInputSession({
retryFailedRecording: true,
recorder,
getTranscriber: () => {
const route = registry.get(voiceRouteAtom);
Expand Down Expand Up @@ -151,7 +152,9 @@ const voiceInputAtom = Atom.make((get) => {
publishAudioLevels: (levels) => audioLevels.set(levels),
keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`),
});
get.subscribe(voiceRouteAtom, () => session.controller.cancel());
// The route also changes on every reconnect, which must not discard a dictation.
get.mount(voiceRouteAtom);
get.subscribe(voiceGrantAtom, () => session.controller.cancel());
get.addFinalizer(() => {
permission.dispose();
session.dispose();
Expand Down
78 changes: 54 additions & 24 deletions apps/mobile/src/features/voice-input/voiceDestination.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ const mocks = vi.hoisted(() => ({
probe: vi.fn(),
fetch: vi.fn<typeof fetch>(),
}));
// null takes a host offline; each number is one established connection to it.
const link = await vi.hoisted(async () => {
const { Atom } = await import("effect/reactivity");
return Atom.family((_id: string) => Atom.make<number | null>(0));
});
vi.mock("expo", () => ({
requireOptionalNativeModule: () => ({ decodeRecording: async () => new Uint8Array([0, 0]) }),
}));
Expand Down Expand Up @@ -49,14 +54,14 @@ vi.mock("../../state/presentation", () => ({
environmentPresentations: {
presentationAtom: Atom.family((id: string) => {
mocks.probe(id);
return Atom.make({
connection: { phase: id === "offline" ? "offline" : "connected" },
return Atom.make((get) => ({
connection: { phase: id === "offline" || get(link(id)) === null ? "offline" : "connected" },
serverConfig: {
environment: {
capabilities: { voiceTranscription: true, cloudVoiceTranscription: true },
},
},
});
}));
}),
},
}));
Expand All @@ -65,12 +70,14 @@ vi.mock("../../state/session", async () => {
return {
environmentSession: {
preparedConnectionValueAtom: Atom.family((environmentId: string) =>
Atom.make(
Option.some({
environmentId,
httpBaseUrl: `https://${environmentId}.test`,
httpAuthorization: { _tag: "Bearer", token: "fixture" },
}),
Atom.make((get) =>
get(link(environmentId)) === null
? Option.none()
: Option.some({
environmentId,
httpBaseUrl: `https://${environmentId}.test`,
httpAuthorization: { _tag: "Bearer", token: "fixture" },
}),
),
),
sessionStateAtom: Atom.family((_id: string) => Atom.make(true)),
Expand All @@ -79,6 +86,21 @@ vi.mock("../../state/session", async () => {
};
});

const readyStatus = () =>
Response.json({
supported: true,
state: "ready",
modelId: "fixture",
model: "Fixture",
size: 1,
supportsStreaming: false,
supportsTranslation: false,
acceleration: "cpu",
modelUnloadTimeout: "never",
gpuDevices: [],
benchmarkTier: "cpu",
});

it.each(["secure store", "marker"])(
"revokes across restarts when writing the %s fails",
async (failedStore) => {
Expand Down Expand Up @@ -146,21 +168,7 @@ it("keeps local as the default and never reuses consent for an offline or resele
);
expect(mocks.fetch).not.toHaveBeenCalled();
mocks.fetch
.mockResolvedValueOnce(
Response.json({
supported: true,
state: "ready",
modelId: "fixture",
model: "Fixture",
size: 1,
supportsStreaming: false,
supportsTranslation: false,
acceleration: "cpu",
modelUnloadTimeout: "never",
gpuDevices: [],
benchmarkTier: "cpu",
}),
)
.mockResolvedValueOnce(readyStatus())
.mockResolvedValueOnce(Response.json({ text: "hello" }));
const options = { signal: new AbortController().signal };
const recording = await current!.prepare(options);
Expand Down Expand Up @@ -235,6 +243,28 @@ it("keeps local as the default and never reuses consent for an offline or resele
}
});

it("keeps a recording sendable across a reconnect to the chosen host", async () => {
const unmount = appAtomRegistry.mount(voiceRouteAtom);
const host = EnvironmentId.make("flaky-host");
try {
chooseVoiceDestination(host);
mocks.fetch.mockResolvedValueOnce(readyStatus());
const options = { signal: new AbortController().signal };
const recording = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options);
appAtomRegistry.set(link(host), null);
await expect(recording.transcribe("file:///voice.m4a", options)).rejects.toMatchObject({
code: "transcription-failed",
message: expect.stringContaining("Your recording is saved"),
});
appAtomRegistry.set(link(host), 1);
mocks.fetch.mockResolvedValueOnce(Response.json({ text: "kept" }));
await expect(recording.transcribe("file:///voice.m4a", options)).resolves.toBe("kept");
} finally {
unmount();
appAtomRegistry.reset();
}
});

it("a late cloud key save cannot override a newer destination choice", () => {
const host = EnvironmentId.make("my-server");
try {
Expand Down
24 changes: 18 additions & 6 deletions apps/mobile/src/features/voice-input/voiceDestination.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,15 +136,23 @@ export function runSpeech<A, E>(
);
}
function environmentTranscriber(
prepared: PreparedConnection,
currentConnection: () => PreparedConnection | null,
canSend: () => boolean,
cloud?: CloudDestination,
) {
const provider = cloud?.provider;
const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
// Resolved per request: a reconnect replaces the connection, and a recording
// made across it still belongs to the same chosen environment.
const connection = (message: string) => {
const prepared = currentConnection();
if (!prepared) throw new VoiceTranscriptionError({ code: "transcription-failed", message });
return prepared;
};
return createEnvironmentTranscriber({
canSend,
prepare: async (signal) => {
const prepared = connection("Connection lost. Try again once reconnected.");
if (provider) {
await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend);
return "auto";
Expand Down Expand Up @@ -176,8 +184,11 @@ function environmentTranscriber(
});
return native.decodeRecording(uri);
},
upload: async (pcm, signal) =>
(
upload: async (pcm, signal) => {
const prepared = connection(
"Connection lost. Your recording is saved; retry once reconnected.",
);
return (
await runSpeech(
provider
? transcribeCloudPcm(
Expand All @@ -197,7 +208,8 @@ function environmentTranscriber(
signal,
canSend,
)
).text,
).text;
},
});
}

Expand Down Expand Up @@ -253,8 +265,8 @@ export const voiceRouteAtom = Atom.make((get) => {
transcriber:
connection &&
environmentTranscriber(
connection,
() => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
() => registry.get(connectionAtom),
() => registry.get(voiceGrantAtom) === grant,
cloud,
),
};
Expand Down
Loading