diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 347b43dd6f44c..f3feab808460b 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1753,7 +1753,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" const hub = UsageLimitSourceId.make("hub"); const router = ProviderInstanceId.make("openrouter"); const withUsage = yield* serverSettings.updateSettings({ - cloudSpeechCredentials: { openai: usageReference, xai: "literal-fixture" }, + cloudSpeechCredentials: { + openai: usageReference, + xai: "literal-fixture", + soniox: usageReference, + }, textGenerators: { [router]: { kind: "openaiCompatible", @@ -1811,6 +1815,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: SECRET_REDACTED, xai: SECRET_REDACTED, openrouter: "", + soniox: SECRET_REDACTED, }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); @@ -1841,6 +1846,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: usageReference, xai: "literal-fixture", openrouter: "", + soniox: usageReference, }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); @@ -1926,7 +1932,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); + assert.deepEqual(removed.cloudSpeechCredentials, { + openai: "", + xai: work, + openrouter: "", + soniox: "", + }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1939,6 +1950,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", "cloudSpeechCredentials.openrouter", + "cloudSpeechCredentials.soniox", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 9f73b2512841c..51e2a2d1e0c1d 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, + cloudSpeechCredentials: { + openai: source("voice#key"), + xai: "", + openrouter: "", + soniox: "", + }, usageLimitSources, textGenerators: {}, easDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 112c7a88d9b3b..4b5314d54230f 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -2,6 +2,8 @@ import { it } from "@effect/vitest"; import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts"; import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as TestClock from "effect/testing/TestClock"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; @@ -49,7 +51,12 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + expect(yield* service.configured).toEqual({ + openai: true, + xai: true, + openrouter: false, + soniox: false, + }); yield* Effect.flip(transcribe(new Uint8Array(3))); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe()).toBe("hello"); @@ -198,3 +205,169 @@ it.effect("removing a key while a catalog read is pending prevents the upload", expect(exit._tag).toBe("Failure"); }), ); + +it.effect.each([false, true])( + "transcribes Soniox uploads with vocabulary hints and deletes temporary data (slow upload: %s)", + (slowUpload) => + Effect.gen(function* () { + const polled = yield* Deferred.make(); + const uploading = yield* Deferred.make(); + const upload = Promise.withResolvers(); + let polls = 0; + const upstream = vi.fn(async (input, init) => { + const path = new URL(String(input)).pathname; + if (init?.method === "DELETE") return new Response(null, { status: 204 }); + if (path === "/v1/files") { + if (slowUpload && polls === 0) { + Deferred.doneUnsafe(uploading, Effect.void); + return upload.promise; + } + return Response.json({ id: "file-id" }); + } + if (path === "/v1/transcriptions") return Response.json({ id: "job-id" }); + if (path.endsWith("/transcript")) return Response.json({ text: " uh tea three " }); + polls++; + if (polls === 1) { + Deferred.doneUnsafe(polled, Effect.void); + return Response.json({ status: "processing" }); + } + return Response.json({ status: "completed" }); + }); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const transcribe = (language = "en") => + service + .transcribe("soniox", new Uint8Array(new Float32Array([0, 0.5]).buffer), { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: language, + speechCustomWords: [{ term: "T3", aliases: ["tea three"] }], + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey("openai", Redacted.make("other-key")); + expect(yield* Effect.flip(transcribe())).toMatchObject({ reason: "missing-key" }); + expect(upstream).not.toHaveBeenCalled(); + yield* service.setKey("soniox", Redacted.make("soniox-key")); + expect((yield* service.configured).soniox).toBe(true); + const fiber = yield* transcribe().pipe(Effect.forkChild); + if (slowUpload) { + yield* Deferred.await(uploading); + yield* TestClock.adjust("20 seconds"); + upload.resolve(Response.json({ id: "file-id" })); + } + yield* Effect.raceFirst(Deferred.await(polled), Fiber.join(fiber)); + yield* TestClock.adjust("1 second"); + expect(yield* Fiber.join(fiber)).toBe("T3"); + expect(polls).toBe(2); + expect(upstream.mock.calls.map(([url, init]) => `${init?.method} ${String(url)}`)).toEqual([ + "POST https://api.soniox.com/v1/files", + "POST https://api.soniox.com/v1/transcriptions", + "GET https://api.soniox.com/v1/transcriptions/job-id", + "GET https://api.soniox.com/v1/transcriptions/job-id", + "GET https://api.soniox.com/v1/transcriptions/job-id/transcript", + "DELETE https://api.soniox.com/v1/transcriptions/job-id", + "DELETE https://api.soniox.com/v1/files/file-id", + ]); + for (const [, init] of upstream.mock.calls) { + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer soniox-key"); + } + const form = upstream.mock.calls[0]![1]!.body as FormData; + const wav = new DataView( + yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()), + ); + expect(wav.getUint32(24, true)).toBe(16_000); + expect(wav.getInt16(46, true)).toBe(16384); + const body = yield* Effect.promise(() => + new Response(upstream.mock.calls[1]![1]!.body).json(), + ); + expect(body).toEqual({ + model: "stt-async-v6", + file_id: "file-id", + language_hints: ["en"], + context: { terms: ["T3"] }, + }); + expect(yield* transcribe("auto")).toBe("T3"); + const auto = yield* Effect.promise(() => + new Response(upstream.mock.calls[8]![1]!.body).json(), + ); + expect(auto).not.toHaveProperty("language_hints"); + yield* service.setKey("soniox", null); + expect((yield* service.configured).soniox).toBe(false); + expect(yield* Effect.flip(transcribe())).toMatchObject({ reason: "missing-key" }); + }), +); + +it.effect.each([ + [401, "rejected-key"], + [403, "rejected-key"], + [429, "rate-limit"], + [500, "unavailable"], + ["error", "unavailable"], + ["malformed", "unavailable"], + ["cancel", "cancel"], + ["timeout", "unavailable"], +] as const)("handles Soniox failure %s and releases uploaded data", ([failure, reason]) => + Effect.gen(function* () { + const pending = yield* Deferred.make(); + const upstream = vi.fn(async (input, init) => { + const path = new URL(String(input)).pathname; + if (init?.method === "DELETE") return new Response(null, { status: 204 }); + if (path === "/v1/files") return Response.json({ id: "file-id" }); + if (path === "/v1/transcriptions") { + if (typeof failure === "number") + return new Response("sensitive diagnostic", { status: failure }); + return Response.json({ id: "job-id" }); + } + Deferred.doneUnsafe(pending, Effect.void); + return Response.json({ + status: failure === "cancel" || failure === "timeout" ? "processing" : failure, + }); + }); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + yield* service.setKey("soniox", Redacted.make("fixture")); + const run = service + .transcribe( + "soniox", + new Uint8Array(new Float32Array([0.5]).buffer), + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + if (failure === "cancel") { + const fiber = yield* run.pipe(Effect.forkChild); + yield* Deferred.await(pending); + yield* Fiber.interrupt(fiber); + } else if (failure === "timeout") { + const fiber = yield* Effect.flip(run).pipe(Effect.forkChild); + yield* Deferred.await(pending); + yield* TestClock.adjust("2 minutes"); + expect(yield* Fiber.join(fiber)).toMatchObject({ provider: "soniox", reason }); + } else { + const error = yield* Effect.flip(run); + expect(error).toMatchObject({ provider: "soniox", reason }); + expect(error.message).not.toContain("sensitive diagnostic"); + } + expect( + upstream.mock.calls + .filter(([, init]) => init?.method === "DELETE") + .map(([url]) => String(url)), + ).toEqual([ + ...(typeof failure === "number" ? [] : ["https://api.soniox.com/v1/transcriptions/job-id"]), + "https://api.soniox.com/v1/files/file-id", + ]); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3b7eab6b5242b..72e75262ed608 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -20,6 +20,7 @@ import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; import * as ServerSettings from "../serverSettings.ts"; import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; @@ -30,10 +31,12 @@ import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts"; const endpoints = { openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, + soniox: { url: "https://api.soniox.com/v1/files" }, openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, }; const OPENROUTER_CATALOG_URL = "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true"; +const isCloudSpeechError = Schema.is(CloudSpeechError); const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); const sameCredential = Schema.toEquivalence(SettingsSecretValue); @@ -130,10 +133,11 @@ export const layer = Layer.effect( Effect.map((value) => value.cloudSpeechCredentials), ); const configured = credentials.pipe( - Effect.map(({ openai, xai, openrouter }) => ({ + Effect.map(({ openai, xai, openrouter, soniox }) => ({ openai: Boolean(openai), xai: Boolean(xai), openrouter: Boolean(openrouter), + soniox: Boolean(soniox), })), ); const execute = (request: HttpClientRequest.HttpClientRequest) => @@ -143,6 +147,103 @@ export const layer = Layer.effect( Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), Effect.provideService(HttpClient.TracerPropagationEnabled, false), ); + const sonioxTranscribe = Effect.fn("CloudSpeech.sonioxTranscribe")(function* ( + key: string, + audio: ArrayBuffer, + language: string, + options: SpeechTranscriptionOptions, + ) { + const unavailable = () => new CloudSpeechError({ provider: "soniox", reason: "unavailable" }); + const request = Effect.fn("CloudSpeech.sonioxRequest")(function* ( + input: HttpClientRequest.HttpClientRequest, + timeoutMs = 15_000, + ) { + const response = yield* execute( + input.pipe(HttpClientRequest.bearerToken(Redacted.make(key))), + ).pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => Effect.fail(unavailable()), + }), + Effect.mapError(unavailable), + ); + if (response.status < 200 || response.status >= 300) + return yield* new CloudSpeechError({ + provider: "soniox", + reason: + response.status === 429 + ? "rate-limit" + : [401, 402, 403].includes(response.status) + ? "rejected-key" + : "unavailable", + }); + return response; + }); + const resource = Schema.Struct({ id: Schema.String.check(Schema.isMinLength(1)) }); + const cleanup = (path: string) => + request(HttpClientRequest.delete(`https://api.soniox.com/v1/${path}`)).pipe( + Effect.timeout("5 seconds"), + Effect.catch(() => Effect.logWarning("Could not delete temporary Soniox speech data.")), + Effect.asVoid, + ); + const form = new FormData(); + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); + const file = yield* Effect.acquireRelease( + request( + HttpClientRequest.post(endpoints.soniox.url).pipe(HttpClientRequest.bodyFormData(form)), + 60_000, + ).pipe( + Effect.flatMap(HttpClientResponse.schemaBodyJson(resource)), + Effect.timeoutOrElse({ + duration: "60 seconds", + orElse: () => Effect.fail(unavailable()), + }), + Effect.mapError((error) => (isCloudSpeechError(error) ? error : unavailable())), + ), + ({ id }) => cleanup(`files/${encodeURIComponent(id)}`), + ); + const job = yield* Effect.acquireRelease( + request( + HttpClientRequest.post("https://api.soniox.com/v1/transcriptions").pipe( + HttpClientRequest.bodyJsonUnsafe({ + model: "stt-async-v6", + file_id: file.id, + ...(language !== "auto" ? { language_hints: [language] } : {}), + context: { terms: options.speechCustomWords.map(({ term }) => term) }, + }), + ), + ).pipe( + Effect.flatMap(HttpClientResponse.schemaBodyJson(resource)), + Effect.timeoutOrElse({ + duration: "15 seconds", + orElse: () => Effect.fail(unavailable()), + }), + Effect.mapError((error) => (isCloudSpeechError(error) ? error : unavailable())), + ), + ({ id }) => cleanup(`transcriptions/${encodeURIComponent(id)}`), + ); + const url = `https://api.soniox.com/v1/transcriptions/${encodeURIComponent(job.id)}`; + const status = yield* request(HttpClientRequest.get(url)).pipe( + Effect.flatMap( + HttpClientResponse.schemaBodyJson( + Schema.Struct({ + status: Schema.Literals(["queued", "processing", "completed", "error"]), + }), + ), + ), + Effect.mapError((error) => (isCloudSpeechError(error) ? error : unavailable())), + Effect.repeat({ + schedule: Schedule.spaced("1 second"), + until: ({ status }) => status === "completed" || status === "error", + }), + ); + if (status.status === "error") return yield* unavailable(); + const result = yield* request(HttpClientRequest.get(`${url}/transcript`)).pipe( + Effect.flatMap(HttpClientResponse.schemaBodyJson(EnvironmentSpeechTranscriptionResult)), + Effect.mapError((error) => (isCloudSpeechError(error) ? error : unavailable())), + ); + return result.text; + }, Effect.scoped); const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () { const unavailable = () => new CloudSpeechError({ provider: "openrouter", reason: "unavailable" }); @@ -208,12 +309,6 @@ export const layer = Layer.effect( ); if (!isApiKey(key)) return yield* unreadable(); const audio = wave(samples); - const request = HttpClientRequest.post(endpoints[provider].url).pipe( - HttpClientRequest.bearerToken(Redacted.make(key)), - provider === "openrouter" - ? openRouterBody(model!, audio, language) - : audioBody(provider, audio, language, options), - ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); // A catalog fetch or vault lookup may have outlived the credential it started with. const current = (yield* credentials)[provider]; @@ -222,22 +317,34 @@ export const layer = Layer.effect( provider, reason: current ? "key-changed" : "missing-key", }); - const response = yield* execute(request).pipe(Effect.mapError(unavailable)); - if (response.status < 200 || response.status >= 300) - return yield* new CloudSpeechError({ - provider, - reason: - response.status === 429 - ? "rate-limit" - : [401, 402, 403].includes(response.status) - ? "rejected-key" - : response.status === 404 && provider === "openrouter" - ? "model-unavailable" - : "unavailable", - }); - const { text } = yield* HttpClientResponse.schemaBodyJson( - EnvironmentSpeechTranscriptionResult, - )(response).pipe(Effect.mapError(unavailable)); + const text = + provider === "soniox" + ? yield* sonioxTranscribe(key, audio, language, options) + : yield* Effect.gen(function* () { + const request = HttpClientRequest.post(endpoints[provider].url).pipe( + HttpClientRequest.bearerToken(Redacted.make(key)), + provider === "openrouter" + ? openRouterBody(model!, audio, language) + : audioBody(provider, audio, language, options), + ); + const response = yield* execute(request).pipe(Effect.mapError(unavailable)); + if (response.status < 200 || response.status >= 300) + return yield* new CloudSpeechError({ + provider, + reason: + response.status === 429 + ? "rate-limit" + : [401, 402, 403].includes(response.status) + ? "rejected-key" + : response.status === 404 && provider === "openrouter" + ? "model-unavailable" + : "unavailable", + }); + const result = yield* HttpClientResponse.schemaBodyJson( + EnvironmentSpeechTranscriptionResult, + )(response).pipe(Effect.mapError(unavailable)); + return result.text; + }); const corrected = applySpeechAliases(text, options.speechCustomWords); return ( options.speechRemoveFillerWords diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 0295603ed2038..529e10b060996 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -40,98 +40,106 @@ vi.mock("../../state/environments", async (importOriginal) => ({ }), })); -it("enables the chosen provider only after a successful setup and keeps its grant when editing the key", async () => { - const id = EnvironmentId.make("voice-host"); - const target = new PrimaryConnectionTarget({ - environmentId: id, - label: "Laptop", - httpBaseUrl: window.location.origin, - wsBaseUrl: "ws://localhost", - }); - const prepared = Option.some({ - ...target, - target, - socketUrl: target.wsBaseUrl, - httpAuthorization: null, - }); - using cleanup = new DisposableStack(); - cleanup.defer(() => chooseAudioDestination(null)); - chooseAudioDestination({ kind: "environment", environmentId: id }); - const writes: unknown[] = []; - let rejectNextWrite = true; - const fetch = globalThis.fetch; - using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { - const url = new URL(request instanceof Request ? request.url : request, window.location.href); - if (url.pathname === "/api/voice/cloud/keys") - return Promise.resolve(Response.json({ openai: writes.length > 0, xai: true })); - if (url.pathname === "/api/voice/cloud/openai/key") { - if (rejectNextWrite) { - rejectNextWrite = false; - return Promise.resolve(new Response(null, { status: 403 })); +it.each(["openai", "soniox"] as const)( + "%s enables the chosen provider only after a successful setup and keeps its grant when editing the key", + async (provider) => { + const label = provider === "openai" ? "OpenAI" : "Soniox"; + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination({ kind: "environment", environmentId: id }); + const writes: unknown[] = []; + let rejectNextWrite = true; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve( + Response.json({ openai: false, xai: true, [provider]: writes.length > 0 }), + ); + if (url.pathname === `/api/voice/cloud/${provider}/key`) { + if (rejectNextWrite) { + rejectNextWrite = false; + return Promise.resolve(new Response(null, { status: 403 })); + } + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); } - return new Request(request, init).json().then((body) => { - writes.push(body); - return new Response(null, { status: 204 }); - }); - } - return fetch(request, init); - }); - await vi.mocked(useEnvironmentScope).withImplementation( - () => true, - async () => { - await vi.mocked(usePreparedConnection).withImplementation( - () => prepared, - async () => { - await using screen = await render( - {}}> - - , - ); - const destination = screen.getByRole("combobox", { name: "Transcribe with" }); - await expect.element(destination).toHaveTextContent("Laptop"); - await destination.click(); - await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); - const dialog = page.getByRole("dialog"); - await dialog.getByRole("button", { name: "Cancel" }).click(); - await expect.element(destination).toHaveTextContent("Laptop"); - expect(writes).toEqual([]); - await destination.click(); - await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); - await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click(); - await page.getByRole("option", { name: "Work vault" }).click(); - await dialog.getByLabelText("OpenAI API key reference").fill("op://Work/Voice/key"); - await userEvent.keyboard("{Enter}"); - await expect.element(dialog.getByRole("alert")).toBeVisible(); - expect(writes).toEqual([]); - await expect - .element(dialog.getByRole("button", { name: "Save and enable" })) - .toBeEnabled(); - await dialog.getByLabelText("OpenAI API key reference").click(); - await userEvent.keyboard("{Enter}"); - await expect.element(dialog).not.toBeInTheDocument(); - await expect.element(destination).toHaveTextContent("OpenAI via Laptop"); - expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]); - const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1"); - await screen.getByRole("button", { name: "Manage key" }).click(); - await dialog.getByLabelText("OpenAI API key", { exact: true }).fill("replacement"); - await userEvent.keyboard("{Enter}"); - await expect.element(dialog).not.toBeInTheDocument(); - expect(writes.at(-1)).toEqual({ key: "replacement" }); - expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant); - await destination.click(); - await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click(); - await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click(); - await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop"); - await destination.click(); - await page.getByRole("option", { name: "Off", exact: true }).click(); - await expect.element(destination).toHaveTextContent("Off"); - }, - ); - }, - ); -}); + return fetch(request, init); + }); + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await expect.element(destination).toHaveTextContent("Laptop"); + await destination.click(); + await page.getByRole("option", { name: `${label} via Laptop` }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect.element(destination).toHaveTextContent("Laptop"); + expect(writes).toEqual([]); + await destination.click(); + await page.getByRole("option", { name: `${label} via Laptop` }).click(); + await dialog.getByRole("combobox", { name: `${label} API key source` }).click(); + await page.getByRole("option", { name: "Work vault" }).click(); + await dialog.getByLabelText(`${label} API key reference`).fill("op://Work/Voice/key"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog.getByRole("alert")).toBeVisible(); + expect(writes).toEqual([]); + await expect + .element(dialog.getByRole("button", { name: "Save and enable" })) + .toBeEnabled(); + await dialog.getByLabelText(`${label} API key reference`).click(); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent(`${label} via Laptop`); + expect(writes).toEqual([ + { key: { provider: "work", reference: "op://Work/Voice/key" } }, + ]); + const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1"); + await screen.getByRole("button", { name: "Manage key" }).click(); + await dialog.getByLabelText(`${label} API key`, { exact: true }).fill("replacement"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + expect(writes.at(-1)).toEqual({ key: "replacement" }); + expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant); + await destination.click(); + await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click(); + await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click(); + await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop"); + await destination.click(); + await page.getByRole("option", { name: "Off", exact: true }).click(); + await expect.element(destination).toHaveTextContent("Off"); + }, + ); + }, + ); + }, +); -it("OpenRouter needs an exact model, and changing it writes no credential", async () => { +it("OpenRouter needs an exact model, changing it writes no credential, and providers an older server omits ask for an update", async () => { const id = EnvironmentId.make("voice-host"); const target = new PrimaryConnectionTarget({ environmentId: id, @@ -188,9 +196,15 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn , ); const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + const dialog = page.getByRole("dialog"); + await destination.click(); + await page.getByRole("option", { name: "Soniox via Laptop" }).click(); + await expect + .element(dialog.getByText("Update Laptop to use Soniox dictation.")) + .toBeVisible(); + await dialog.getByRole("button", { name: "Cancel" }).click(); await destination.click(); await page.getByRole("option", { name: "OpenRouter via Laptop" }).click(); - const dialog = page.getByRole("dialog"); const save = dialog.getByRole("button", { name: "Save and enable" }); await expect.element(save).toBeDisabled(); const picker = dialog.getByRole("combobox", { name: "OpenRouter model" }); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 95dcee4cffb45..3a546dd9503cf 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -218,7 +218,7 @@ export function CloudVoiceConfiguration({ const destination = (): CloudDestination => routed && isModelId(model) ? { kind: "cloud", provider: "openrouter", environmentId: id, model } - : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + : { kind: "cloud", provider: provider as "openai" | "xai" | "soniox", environmentId: id }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed ? modelChosen && keyReady && (!active || modelChanged || keyChanged) @@ -343,11 +343,10 @@ export function CloudVoiceConfiguration({ - ) : routed && - AsyncResult.isSuccess(presence) && - presence.value && - saved === undefined ? ( -

Update {environment.label} to use OpenRouter dictation.

+ ) : AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( +

+ Update {environment.label} to use {name} dictation. +

) : saved === undefined ? (

Checking for an API key…

) : routed ? ( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 495abdefb6b6e..3f43de572327a 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -123,6 +123,7 @@ import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings" const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { + if (provider === "soniox") return ; if (provider === "openrouter") return ; return ( diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index f7492035874da..e0ede81ded555 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -524,7 +524,7 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Transcribe with", to: "/settings/voice", searchTerms: [ - "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter", + "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter Soniox", ], }, { diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 7e5c36fa4d1eb..271e3d58ad1d5 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -24,7 +24,7 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. -For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment +For cloud dictation, choose **OpenAI**, **Grok Voice Transcribe**, or **Soniox** via the environment you want to use. If it needs a key, enter one and choose **Save and enable**. You can also choose a [secret provider](secrets.md) instead of **Value** and enter a reference. Unlock the vault on that environment before dictating. Anyone with @@ -36,6 +36,10 @@ subscription is needed. Switch back to **This device** on mobile or **Off** on web and desktop to stop sharing. Removing the key stops that provider for everyone on the environment. +**Soniox** uses asynchronous transcription after you stop recording. Use an API key +from a US Soniox project. T3 requests deletion of the uploaded audio and transcription +afterward; if cleanup fails, data remains subject to Soniox’s retention policy. + **OpenRouter** works the same way but has no default model: pick one from its live speech-to-text catalog when you enable it, or later under **Manage OpenRouter**. The model choice is yours alone and needs no settings access, so anyone with diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd112d84bf9f4..f4f2ae0afff53 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -375,7 +375,7 @@ export const VoiceTranscriptionDestination = Schema.Union([ Schema.Struct({ kind: Schema.Literal("cloud"), environmentId: EnvironmentId, - provider: Schema.Literals(["openai", "xai"]), + provider: Schema.Literals(["openai", "xai", "soniox"]), }), Schema.Struct({ kind: Schema.Literal("cloud"), @@ -1328,6 +1328,7 @@ export const CloudSpeechCredentials = Schema.Struct({ openai: CloudSpeechCredential, xai: CloudSpeechCredential, openrouter: CloudSpeechCredential, + soniox: CloudSpeechCredential, }); export const ServerSettings = Schema.Struct({ @@ -1358,7 +1359,7 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })), + Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "", soniox: "" })), ), speechModelId: Schema.String.pipe( Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), @@ -1808,6 +1809,7 @@ export const ServerSettingsPatch = Schema.Struct({ openai: Schema.optionalKey(SettingsSecretValue), xai: Schema.optionalKey(SettingsSecretValue), openrouter: Schema.optionalKey(SettingsSecretValue), + soniox: Schema.optionalKey(SettingsSecretValue), }), ), speechModelId: Schema.optionalKey(Schema.String), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ac87096a24cce..d5bbf434db2a7 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -8,23 +8,26 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check( Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i), ); -export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]); +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter", "soniox"]); export type CloudSpeechProvider = typeof CloudSpeechProvider.Type; export const CLOUD_SPEECH_LABELS = { openai: "OpenAI", xai: "Grok Voice Transcribe", openrouter: "OpenRouter", + soniox: "Soniox", } as const; export const CLOUD_SPEECH_PROVIDER_NAMES = { openai: "OpenAI", xai: "xAI", openrouter: "OpenRouter", + soniox: "Soniox", } as const; -/** Servers that predate OpenRouter report only the first two providers. */ +/** Older servers may omit newer providers. */ export const CloudSpeechKeys = Schema.Struct({ openai: Schema.Boolean, xai: Schema.Boolean, openrouter: Schema.optionalKey(Schema.Boolean), + soniox: Schema.optionalKey(Schema.Boolean), }); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type;