From 0188d0e7d537fef867e0e8ce3e519193164454d0 Mon Sep 17 00:00:00 2001 From: cd-slash <29688941+cd-slash@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:40:20 +0000 Subject: [PATCH 1/3] feat(voice): add custom OpenAI-compatible cloud speech provider Adds a fourth cloud dictation provider, custom, that sends recordings to the user's own OpenAI-compatible endpoint instead of a hosted provider. - Contracts: 'custom' joins CloudSpeechProvider; the destination grant names an exact model (loose id) like OpenRouter; ServerSettings gains cloudSpeechCustomBaseUrl; CloudSpeechKeys reports custom key presence and the configured endpoint for disclosure. - Server: the transcribe endpoint accepts the loose model id (OpenRouter keeps its strict catalog check); custom uploads are OpenAI-compatible multipart (model, language, file) to /audio/transcriptions with a bearer key. The URL must be https, or plain http only for loopback hosts. - Web: the cloud voice dialog gains endpoint URL and model inputs for custom; the URL saves as a server setting before the key; the disclosure names the endpoint host; settings panel lists the provider with a manage entry point. - Docs and tests: user/internals voice docs updated; server CloudSpeech custom flow test; settings secret-field fixtures; browser test covering validation, URL persistence, key write, and the per-device grant. Fixes #32514 --- apps/server/src/serverSettings.test.ts | 10 +- .../sourceControlHostSecrets.test.ts | 7 +- apps/server/src/speech/CloudSpeech.test.ts | 82 ++++++++- apps/server/src/speech/CloudSpeech.ts | 50 +++++- .../CloudVoiceConfiguration.browser.test.tsx | 94 ++++++++++ .../settings/CloudVoiceConfiguration.tsx | 165 ++++++++++++++---- .../settings/VoiceSettingsPanel.tsx | 12 +- apps/web/src/speech/browserVoiceInput.ts | 6 +- .../src/speech/useEnvironmentSpeechInput.ts | 5 +- docs/internals/voice-input.md | 7 + docs/user/voice-input.md | 9 + .../src/voice-input/environment.ts | 4 +- packages/contracts/src/environmentHttp.ts | 4 +- packages/contracts/src/settings.ts | 17 +- packages/contracts/src/speech.ts | 25 ++- packages/shared/src/speech.ts | 23 ++- 16 files changed, 462 insertions(+), 58 deletions(-) diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 6c470d1f83f05..bfc44d6ddbf83 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1692,6 +1692,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: SECRET_REDACTED, xai: SECRET_REDACTED, openrouter: "", + custom: "", }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); @@ -1719,6 +1720,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: usageReference, xai: "literal-fixture", openrouter: "", + custom: "", }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); @@ -1804,7 +1806,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); + assert.deepEqual(removed.cloudSpeechCredentials, { + openai: "", + xai: work, + openrouter: "", + custom: "", + }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1817,6 +1824,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", "cloudSpeechCredentials.openrouter", + "cloudSpeechCredentials.custom", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 58c84365dfcef..a7137e01ae796 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, + cloudSpeechCredentials: { + openai: source("voice#key"), + xai: "", + openrouter: "", + custom: "", + }, usageLimitSources, easDeviceHosts, simlockDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 112c7a88d9b3b..48ae1dbfc9cb7 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -49,7 +49,13 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + expect(yield* service.configured).toEqual({ + openai: true, + xai: true, + openrouter: false, + custom: false, + customEndpoint: "", + }); yield* Effect.flip(transcribe(new Uint8Array(3))); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe()).toBe("hello"); @@ -198,3 +204,77 @@ it.effect("removing a key while a catalog read is pending prevents the upload", expect(exit._tag).toBe("Failure"); }), ); + +it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart", () => + Effect.gen(function* () { + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + const stack = (baseUrl: string | undefined) => + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide( + ServerSettings.ServerSettingsService.layerTest( + baseUrl === undefined ? {} : { cloudSpeechCustomBaseUrl: baseUrl }, + ), + ), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide(yield* Layer.build(stack("http://localhost:8080/v1"))), + ); + // Builds a separate stack for a different endpoint configuration. + const attempt = (baseUrl: string | undefined, model: string | undefined) => + Effect.gen(function* () { + const context = yield* Layer.build(stack(baseUrl)); + const scoped = yield* CloudSpeech.CloudSpeech.pipe(Effect.provide(context)); + yield* scoped.setKey("custom", Redacted.make("custom-key")); + return yield* scoped + .transcribe( + "custom", + new Uint8Array(new Float32Array([0, 0.5]).buffer), + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + }); + yield* service.setKey("custom", Redacted.make("custom-key")); + const pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer); + const transcribe = (model: string | undefined) => + service + .transcribe( + "custom", + pcm, + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ + provider: "custom", + reason: "model-unavailable", + }); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ + reason: "model-unavailable", + }); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* transcribe("whisper-1")).toBe("hello"); + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe("http://localhost:8080/v1/audio/transcriptions"); + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer custom-key"); + const form = init?.body as FormData; + expect(form.get("model")).toBe("whisper-1"); + expect(form.get("language")).toBe("en"); + expect([...form.keys()].at(-1)).toBe("file"); + expect(yield* Effect.flip(attempt("http://example.com/v1", "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(upstream).toHaveBeenCalledTimes(1); + yield* service.setKey("custom", null); + expect((yield* service.configured).custom).toBe(false); + expect(yield* Effect.flip(transcribe("whisper-1"))).toMatchObject({ reason: "missing-key" }); + expect(upstream).toHaveBeenCalledTimes(1); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3b7eab6b5242b..3c5bdec17f3ff 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -101,6 +101,32 @@ function openRouterBody(model: string, audio: ArrayBuffer, language: string) { }); } +/** The transcriptions URL for a configured custom base URL, or null when it is unusable. */ +function customTranscriptionsUrl(baseUrl: string) { + let url: URL; + try { + url = new URL(baseUrl); + } catch { + return null; + } + if (url.search || url.hash) return null; + const hostname = url.hostname.toLowerCase(); + const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + const local = + host === "localhost" || host === "::1" || host === "127.0.0.1" || host.endsWith(".localhost"); + if (url.protocol === "http:" && !local) return null; + return `${url.origin}${url.pathname.replace(/\/+$/, "")}/audio/transcriptions`; +} + +/** The generic multipart body every OpenAI-compatible transcription server accepts. */ +function customBody(model: string, audio: ArrayBuffer, language: string) { + const form = new FormData(); + form.set("model", model); + if (language !== "auto") form.set("language", language); + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); + return HttpClientRequest.bodyFormData(form); +} + export class CloudSpeech extends Context.Service< CloudSpeech, { @@ -129,11 +155,13 @@ export const layer = Layer.effect( const credentials = settings.getSettings.pipe( Effect.map((value) => value.cloudSpeechCredentials), ); - const configured = credentials.pipe( - Effect.map(({ openai, xai, openrouter }) => ({ - openai: Boolean(openai), - xai: Boolean(xai), - openrouter: Boolean(openrouter), + const configured = settings.getSettings.pipe( + Effect.map((value) => ({ + openai: Boolean(value.cloudSpeechCredentials.openai), + xai: Boolean(value.cloudSpeechCredentials.xai), + openrouter: Boolean(value.cloudSpeechCredentials.openrouter), + custom: Boolean(value.cloudSpeechCredentials.custom), + customEndpoint: value.cloudSpeechCustomBaseUrl, })), ); const execute = (request: HttpClientRequest.HttpClientRequest) => @@ -189,6 +217,10 @@ export const layer = Layer.effect( if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); if (provider === "openrouter" && (!model || !isModelId(model))) return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); + const customUrl = + provider === "custom" ? customTranscriptionsUrl((yield* settings.getSettings).cloudSpeechCustomBaseUrl) : null; + if (provider === "custom" && (!model || !customUrl)) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const samples = yield* Effect.try({ try: () => decodeSpeechPcm(pcm), catch: () => @@ -208,11 +240,15 @@ export const layer = Layer.effect( ); if (!isApiKey(key)) return yield* unreadable(); const audio = wave(samples); - const request = HttpClientRequest.post(endpoints[provider].url).pipe( + const request = HttpClientRequest.post( + provider === "custom" ? customUrl! : endpoints[provider].url, + ).pipe( HttpClientRequest.bearerToken(Redacted.make(key)), provider === "openrouter" ? openRouterBody(model!, audio, language) - : audioBody(provider, audio, language, options), + : provider === "custom" + ? customBody(model!, audio, language) + : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); // A catalog fetch or vault lookup may have outlived the credential it started with. diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 0295603ed2038..8e29739399aae 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -11,6 +11,14 @@ import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; import { SettingsScopeProvider } from "./SettingsScopeContext"; vi.mock(import("../../state/session"), { spy: true }); +const persistedSettings: unknown[] = []; +vi.mock(import("../../state/use-atom-command"), async (original) => ({ + ...(await original()), + useAtomCommand: (() => async (input: unknown) => { + persistedSettings.push(input); + return { _tag: "Success" }; + }) as never, +})); vi.mock("../../env", async (original) => ({ ...(await original()), isSecretsFeatureEnabled: true, @@ -225,3 +233,89 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn }, ); }); + +it("configures a custom OpenAI-compatible endpoint with its URL, model, and key", async () => { + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination(null); + const writes: unknown[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: true, xai: true, custom: true })); + if (url.pathname === "/api/voice/cloud/openrouter/models") + return Promise.resolve(Response.json({ models: [] })); + if (url.pathname === "/api/voice/cloud/custom/key") + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + return fetch(request, init); + }); + const grantDestination = () => { + const raw = window.localStorage.getItem("t3code:voice-audio-grant:v1"); + const parsed = raw + ? (JSON.parse(raw) as { + destination?: Record; + pending?: { destination?: Record }; + }) + : null; + return parsed?.destination ?? parsed?.pending?.destination ?? null; + }; + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await destination.click(); + await page.getByRole("option", { name: "Custom endpoint via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + const save = dialog.getByRole("button", { name: "Save and enable" }); + await expect.element(save).toBeDisabled(); + await dialog.getByLabelText("Custom endpoint API key", { exact: true }).fill("custom-key"); + await dialog.getByRole("textbox", { name: "Transcription model" }).fill("whisper-1"); + const endpoint = dialog.getByRole("textbox", { name: "Endpoint base URL" }); + await endpoint.fill("http://insecure.example.com/v1"); + await expect.element(save).toBeDisabled(); + await endpoint.fill("https://stt.example.com/v1"); + await expect.element( + dialog.getByText("your endpoint at stt.example.com", { exact: false }), + ).toBeVisible(); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("Custom endpoint via Laptop"); + expect(persistedSettings).toEqual([ + { environmentId: id, input: { patch: { cloudSpeechCustomBaseUrl: "https://stt.example.com/v1" } } }, + ]); + expect(writes).toEqual([{ key: "custom-key" }]); + expect(grantDestination()).toMatchObject({ + provider: "custom", + environmentId: "voice-host", + model: "whisper-1", + }); + }, + ); + }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 95dcee4cffb45..114630f4bde5a 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -18,6 +18,7 @@ import { CLOUD_SPEECH_LABELS, CLOUD_SPEECH_PROVIDER_NAMES, isSecretReference, + SpeechModelId, type CloudSpeechModel, type CloudSpeechProvider, type SecretProviderId, @@ -31,6 +32,8 @@ import type { HttpClient } from "effect/http"; import { Atom, AsyncResult } from "effect/reactivity"; import { useId, useState } from "react"; import { isSecretsFeatureEnabled } from "../../env"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; import { inEnvironmentSpeechRuntime, runEnvironmentSpeech, @@ -83,6 +86,21 @@ function useCloudRead( } const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); +const isCustomModelId = Schema.is(SpeechModelId); +/** Mirror of the server check: https, or plain http only for loopback hosts. */ +function isCustomEndpoint(value: string) { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return false; + } + if (url.search || url.hash) return false; + const host = url.hostname.toLowerCase(); + const loopback = + host === "localhost" || host === "[::1]" || host === "127.0.0.1" || host.endsWith(".localhost"); + return url.protocol === "https:" || (url.protocol === "http:" && loopback); +} type CloudDestination = Extract; function ModelPicker({ @@ -162,7 +180,8 @@ export function CloudVoiceConfiguration({ const id = environment.environmentId; const name = CLOUD_SPEECH_LABELS[provider]; const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; - const routed = provider === "openrouter"; + const routed = provider === "openrouter" || provider === "custom"; + const custom = provider === "custom"; const formId = useId(); const prepared = Option.getOrNull(usePreparedConnection(id)); const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); @@ -175,9 +194,19 @@ export function CloudVoiceConfiguration({ const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; const currentModel = - active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter" + active && + grant?.destination.kind === "cloud" && + (grant.destination.provider === "openrouter" || grant.destination.provider === "custom") ? grant.destination.model : undefined; + const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { + label: "custom speech endpoint update", + reportFailure: false, + }); + const savedEndpoint = environment.serverConfig?.settings?.cloudSpeechCustomBaseUrl ?? ""; + const [endpoint, setEndpoint] = useState(savedEndpoint); + const endpointChanged = custom && endpoint.trim() !== savedEndpoint; + const endpointReady = !custom || isCustomEndpoint(endpoint); const [remove, setRemove] = useState(false); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); @@ -212,16 +241,24 @@ export function CloudVoiceConfiguration({ // Fixed providers edit the key only when nothing is saved or the device already uses them. const editingKey = canManage && saved !== undefined && (routed || active || !saved); const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; - const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); + const customModelReady = isCustomModelId(model.trim()); + const modelChosen = custom + ? customModelReady + : isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); const keyReady = draft.trim() ? credential !== undefined : Boolean(saved); const modelChanged = model !== (currentModel ?? ""); const destination = (): CloudDestination => - routed && isModelId(model) + custom && customModelReady + ? { kind: "cloud", provider: "custom", environmentId: id, model: model.trim() } + : routed && isModelId(model) ? { kind: "cloud", provider: "openrouter", environmentId: id, model } : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed - ? modelChosen && keyReady && (!active || modelChanged || keyChanged) + ? modelChosen && + endpointReady && + keyReady && + (!active || modelChanged || keyChanged || endpointChanged) : credential !== undefined; const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => { @@ -239,9 +276,28 @@ export function CloudVoiceConfiguration({ } setBusy(false); }; - const submit = () => { + const submit = async () => { if (!canSubmit || busy) return; - const grantChanges = !active || modelChanged; + const grantChanges = !active || modelChanged || endpointChanged; + if (endpointChanged) { + setError(""); + setBusy(true); + try { + const result = await updateSettings({ + environmentId: id, + input: { patch: { cloudSpeechCustomBaseUrl: endpoint.trim() } }, + }); + setBusy(false); + if (result._tag !== "Success") { + setError("Couldn’t save the endpoint URL. Check your connection and try again."); + return; + } + } catch { + setBusy(false); + setError("Couldn’t save the endpoint URL. Check your connection and try again."); + return; + } + } if (credential !== undefined && (!routed || keyChanged)) return void saveKey(credential, grantChanges); // Model-only changes never write to the environment. @@ -323,7 +379,9 @@ export function CloudVoiceConfiguration({ ? `Remove the ${company} key?` : active ? routed - ? "Manage OpenRouter dictation" + ? custom + ? "Manage custom dictation" + : "Manage OpenRouter dictation" : `Manage ${company} key` : `Use ${name} for dictation?`} @@ -332,7 +390,9 @@ export function CloudVoiceConfiguration({ ? `Everyone using ${name} for dictation on ${environment.label} will lose access.` : active && !routed ? `Anyone with dictation access to ${environment.label} can use this API key.` - : cloudSpeechDisclosure(provider, environment.label)} + : cloudSpeechDisclosure(provider, environment.label, { + endpoint: custom ? (endpoint.trim() || savedEndpoint) : undefined, + })} {!remove && ( @@ -347,7 +407,7 @@ export function CloudVoiceConfiguration({ AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( -

Update {environment.label} to use OpenRouter dictation.

+

Update {environment.label} to use {name} dictation.

) : saved === undefined ? (

Checking for an API key…

) : routed ? ( @@ -356,29 +416,70 @@ export function CloudVoiceConfiguration({ className="grid gap-3" onSubmit={(event) => { event.preventDefault(); - submit(); + void submit(); }} > -
- - {listedModels ? ( - - ) : AsyncResult.isFailure(models) ? ( - - ) : ( -

- Loading transcription models… -

- )} -
+ {custom ? ( + <> +
+ + setEndpoint(event.target.value)} + /> +

+ Base URL of an OpenAI-compatible server, including the version path. Audio + is sent to its /audio/transcriptions route. Plain http is only allowed for + localhost. +

+ {endpoint.trim() && !endpointReady ? ( +

+ Enter an https URL, or plain http for localhost. +

+ ) : null} +
+
+ + setModel(event.target.value)} + /> +
+ + ) : ( +
+ + {listedModels ? ( + + ) : AsyncResult.isFailure(models) ? ( + + ) : ( +

+ Loading transcription models… +

+ )} +
+ )} {editingKey ? (
{keyField}
) : !saved ? ( @@ -396,7 +497,7 @@ export function CloudVoiceConfiguration({ className="grid gap-2" onSubmit={(event) => { event.preventDefault(); - submit(); + void submit(); }} > {keyField} diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 4a3df08f40f14..67a3408c276c6 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -125,6 +125,8 @@ const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; + if (provider === "custom") + return ; return ( environment.environmentId === destination.environmentId, ); @@ -1375,7 +1381,9 @@ export function VoiceSettingsPanel() { {destination.cloudProvider && chosenEnvironment && - (canManageKey || destination.cloudProvider === "openrouter") && ( + (canManageKey || + destination.cloudProvider === "openrouter" || + destination.cloudProvider === "custom") && ( boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index c569c14500be6..1c0f36d10a957 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -122,7 +122,10 @@ export function useVoiceTranscriptionDestination() { const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; const provider = cloud?.provider; - const model = cloud?.provider === "openrouter" ? cloud.model : undefined; + const model = + cloud && (cloud.provider === "openrouter" || cloud.provider === "custom") + ? cloud.model + : undefined; const supported = environment?.serverConfig?.environment.capabilities[ provider ? "cloudVoiceTranscription" : "voiceTranscription" diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 2bf73af0ab36d..2d3ab3f38c58a 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -53,6 +53,13 @@ applies the account's safety-retention policy when choosing an endpoint. The cat holds no credential or consent state, which is why it may be cached while key presence is read fresh on every request. +The custom provider is the generic OpenAI-compatible multipart client: the base +URL is a server setting, validated to https (or loopback http), with +`/audio/transcriptions` appended and the grant's model sent in the form. The +grant names the exact model for the same reason OpenRouter's does. The endpoint +is disclosed by host in the consent dialog but is otherwise unvetted; the +response must still decode as `{ text }`. + ## Server side The [speech service](../../apps/server/src/speech/SpeechService.ts) runs diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index c116c9e67a395..06eb417ef3d88 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -47,6 +47,15 @@ Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied after transcription. If OpenRouter stops listing your model, dictation stops until you choose another. +**Custom endpoint** points the environment at your own OpenAI-compatible +transcription server instead of a hosted provider. Enter the server's base URL +(including the version path, for example `https://stt.example.com/v1`), a model +name, and an API key; audio is sent to the URL's `/audio/transcriptions` route. +Plain `http` is only accepted for localhost servers. The model, like the +OpenRouter model, is per device; the URL and key are shared by the environment. +T3 cannot verify what a custom endpoint retains, so only point it at servers you +trust. + ## Set up a model In **Settings → Voice → Transcription**, download a model and choose **Use model** if it diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 52fa5a0d1fe5b..74d44567dca92 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -4,10 +4,10 @@ import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts"; import type { - CloudSpeechModelId, CloudSpeechProvider, SettingsSecretValue, SpeechAcceleration, + SpeechModelId, SpeechModelUnloadTimeout, SpeechTranscriptionOptions, SpeechPostProcessingOptions, @@ -130,7 +130,7 @@ export const transcribeCloudPcm = ( pcm: Uint8Array, options: SpeechTranscriptionOptions, canSend: () => boolean, - model?: CloudSpeechModelId, + model?: SpeechModelId, ) => { const query = model === undefined ? {} : { model }; return request({ diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 46dfcde197ac2..55dc2f7916be4 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -52,7 +52,6 @@ import { CloudSpeechApiKey, CloudSpeechError, CloudSpeechKeys, - CloudSpeechModelId, CloudSpeechModels, CloudSpeechProvider, EnvironmentSpeechAccelerationRequest, @@ -63,6 +62,7 @@ import { EnvironmentSpeechPostProcessingResult, EnvironmentSpeechStatus, EnvironmentSpeechTranscriptionResult, + SpeechModelId, } from "./speech.ts"; import { SecretReference } from "./secretReference.ts"; import { @@ -814,7 +814,7 @@ export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice") HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", { headers: OptionalBearerHeaders, params: Schema.Struct({ provider: CloudSpeechProvider }), - query: Schema.Struct({ model: Schema.optionalKey(CloudSpeechModelId) }), + query: Schema.Struct({ model: Schema.optionalKey(SpeechModelId) }), payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()), success: EnvironmentSpeechTranscriptionResult, error: [ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index f6409a87074fd..69fe4ff46f155 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -58,6 +58,7 @@ import { SpeechCorrectionCue, SpeechCustomWords, SpeechLanguage, + SpeechModelId, SpeechModelUnloadTimeout, SpeechPostProcessingPromptSettings, } from "./speech.ts"; @@ -369,6 +370,7 @@ export type VoiceShortcutMode = typeof VoiceShortcutMode.Type; /** * Where this device sends microphone audio. A cloud grant names the forwarding environment and * the provider; an OpenRouter grant also names the exact model, since OpenRouter has no default. + * A custom endpoint grant names the exact model too: OpenAI-compatible servers have no default. */ export const VoiceTranscriptionDestination = Schema.Union([ Schema.Struct({ kind: Schema.Literal("environment"), environmentId: EnvironmentId }), @@ -383,6 +385,12 @@ export const VoiceTranscriptionDestination = Schema.Union([ provider: Schema.Literal("openrouter"), model: CloudSpeechModelId, }), + Schema.Struct({ + kind: Schema.Literal("cloud"), + environmentId: EnvironmentId, + provider: Schema.Literal("custom"), + model: SpeechModelId, + }), ]); export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type; @@ -1245,6 +1253,7 @@ export const CloudSpeechCredentials = Schema.Struct({ openai: CloudSpeechCredential, xai: CloudSpeechCredential, openrouter: CloudSpeechCredential, + custom: CloudSpeechCredential, }); export const ServerSettings = Schema.Struct({ @@ -1273,8 +1282,12 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })), + Schema.withDecodingDefault( + Effect.succeed({ openai: "", xai: "", openrouter: "", custom: "" }), + ), ), + /** Base URL of the OpenAI-compatible transcription endpoint, including the version path (e.g. `https://host/v1`). */ + cloudSpeechCustomBaseUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), speechModelId: Schema.String.pipe( Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), ), @@ -1696,8 +1709,10 @@ export const ServerSettingsPatch = Schema.Struct({ openai: Schema.optionalKey(SettingsSecretValue), xai: Schema.optionalKey(SettingsSecretValue), openrouter: Schema.optionalKey(SettingsSecretValue), + custom: Schema.optionalKey(SettingsSecretValue), }), ), + cloudSpeechCustomBaseUrl: Schema.optionalKey(Schema.String), speechModelId: Schema.optionalKey(Schema.String), speechAcceleration: Schema.optionalKey(SpeechAcceleration), speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index e6e4d3ddbc51e..db04fee8869d0 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -7,23 +7,28 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check( Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i), ); -export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]); +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter", "custom"]); export type CloudSpeechProvider = typeof CloudSpeechProvider.Type; export const CLOUD_SPEECH_LABELS = { openai: "OpenAI", xai: "Grok Voice Transcribe", openrouter: "OpenRouter", + custom: "Custom endpoint", } as const; export const CLOUD_SPEECH_PROVIDER_NAMES = { openai: "OpenAI", xai: "xAI", openrouter: "OpenRouter", + custom: "Custom endpoint", } as const; /** Servers that predate OpenRouter report only the first two providers. */ export const CloudSpeechKeys = Schema.Struct({ openai: Schema.Boolean, xai: Schema.Boolean, openrouter: Schema.optionalKey(Schema.Boolean), + custom: Schema.optionalKey(Schema.Boolean), + /** The configured custom endpoint base URL, for disclosure; empty until one is set. */ + customEndpoint: Schema.optionalKey(Schema.String), }); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type; @@ -57,6 +62,24 @@ export class CloudSpeechError extends Schema.TaggedError()( ) { override get message() { const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider]; + if (this.provider === "custom") { + switch (this.reason) { + case "missing-key": + return "Add an API key for the custom endpoint in Settings → Voice."; + case "key-changed": + return "The custom endpoint key changed before your recording was sent. Try again."; + case "unreadable-key": + return "Couldn’t read the custom endpoint API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent."; + case "rejected-key": + return "The custom endpoint rejected this API key. Check the key and endpoint URL in Settings → Voice."; + case "rate-limit": + return "The custom endpoint reached a usage limit. Check the endpoint's quota or try again later."; + case "model-unavailable": + return "Set the custom endpoint URL and a model name in Settings → Voice."; + case "unavailable": + return "The custom endpoint couldn’t transcribe this recording. Check its URL in Settings → Voice."; + } + } switch (this.reason) { case "missing-key": return `Add your ${name} API key in Settings → Voice.`; diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index c5ed9ab14ea0a..8e7ce2f79583e 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -10,10 +10,25 @@ import * as Schema from "effect/Schema"; const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); -export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) { - return provider === "openrouter" - ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.` - : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; +export function cloudSpeechDisclosure( + provider: CloudSpeechProvider, + environment: string, + { endpoint }: { endpoint?: string | undefined } = {}, +) { + if (provider === "openrouter") + return `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`; + if (provider === "custom") { + let host = ""; + try { + host = endpoint ? new URL(endpoint).host : ""; + } catch { + host = ""; + } + return host + ? `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to your endpoint at ${host}.` + : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to the custom endpoint configured on it.`; + } + return `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; } // Only offer explicit hints understood by all cloud APIs; other languages can still be detected. From 028b71f75c17dc80122c3097827ee58de28e5689 Mon Sep 17 00:00:00 2001 From: cd-slash <29688941+cd-slash@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:14:39 +0000 Subject: [PATCH 2/3] fix(voice): bind custom grants to their endpoint and harden the save flow Review follow-ups for the custom speech provider: - The custom destination grant now carries the endpoint it consented to; useVoiceTranscriptionDestination voids the grant when the environment's configured base URL no longer matches, so a server-side URL change can never redirect an existing device's recordings to a host the user never approved. - The pending audio choice is created before the endpoint-save await and the same choice is acknowledged afterward, so opting out in another tab during a save can no longer be overwritten by this submission. - The endpoint field is disabled without settings-write access; model selection stays per device, like OpenRouter. - The custom dialog no longer fetches the OpenRouter catalog. --- .../CloudVoiceConfiguration.browser.test.tsx | 16 ++++++++++ .../settings/CloudVoiceConfiguration.tsx | 29 ++++++++++++++----- .../src/speech/useEnvironmentSpeechInput.ts | 7 ++++- packages/contracts/src/settings.ts | 2 ++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 8e29739399aae..856669f75e71b 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -12,10 +12,24 @@ import { SettingsScopeProvider } from "./SettingsScopeContext"; vi.mock(import("../../state/session"), { spy: true }); const persistedSettings: unknown[] = []; +let configuredEndpoint = ""; vi.mock(import("../../state/use-atom-command"), async (original) => ({ ...(await original()), useAtomCommand: (() => async (input: unknown) => { persistedSettings.push(input); + configuredEndpoint = + typeof input === "object" && + input !== null && + "input" in input && + typeof input.input === "object" && + input.input !== null && + "patch" in input.input && + typeof input.input.patch === "object" && + input.input.patch !== null && + "cloudSpeechCustomBaseUrl" in input.input.patch && + typeof input.input.patch.cloudSpeechCustomBaseUrl === "string" + ? input.input.patch.cloudSpeechCustomBaseUrl + : configuredEndpoint; return { _tag: "Success" }; }) as never, })); @@ -37,6 +51,7 @@ vi.mock("../../state/environments", async (importOriginal) => ({ secretProviders: { work: { kind: "one_password", label: "Work vault", config: { account: "work" } }, }, + cloudSpeechCustomBaseUrl: configuredEndpoint, }, environment: { platform: { os: "linux", arch: "x64" }, @@ -313,6 +328,7 @@ it("configures a custom OpenAI-compatible endpoint with its URL, model, and key" provider: "custom", environmentId: "voice-host", model: "whisper-1", + endpoint: "https://stt.example.com/v1", }); }, ); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 114630f4bde5a..f9e418bb14d5e 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -188,7 +188,7 @@ export function CloudVoiceConfiguration({ const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys); const [models, refreshModels] = useCloudRead( - routed && canUse ? prepared : null, + provider === "openrouter" && canUse ? prepared : null, getCloudSpeechModels, ); const choice = useAudioChoice(); @@ -249,7 +249,13 @@ export function CloudVoiceConfiguration({ const modelChanged = model !== (currentModel ?? ""); const destination = (): CloudDestination => custom && customModelReady - ? { kind: "cloud", provider: "custom", environmentId: id, model: model.trim() } + ? { + kind: "cloud", + provider: "custom", + environmentId: id, + model: model.trim(), + endpoint: endpoint.trim(), + } : routed && isModelId(model) ? { kind: "cloud", provider: "openrouter", environmentId: id, model } : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; @@ -261,12 +267,11 @@ export function CloudVoiceConfiguration({ (!active || modelChanged || keyChanged || endpointChanged) : credential !== undefined; - const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => { + const saveKey = async (value: SettingsSecretValue | null, commit: (() => void) | null) => { if (!connected || !canManage || busy) return; setError(""); setBusy(true); try { - const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null; await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value)); refresh(); commit?.(); @@ -279,6 +284,9 @@ export function CloudVoiceConfiguration({ const submit = async () => { if (!canSubmit || busy) return; const grantChanges = !active || modelChanged || endpointChanged; + // Created before the first await so an Off chosen elsewhere during the save + // is never overwritten by this submission's commit. + const commit = grantChanges ? pendingCloudAudioChoice(destination()) : null; if (endpointChanged) { setError(""); setBusy(true); @@ -299,9 +307,14 @@ export function CloudVoiceConfiguration({ } } if (credential !== undefined && (!routed || keyChanged)) - return void saveKey(credential, grantChanges); + return void saveKey(credential, commit); // Model-only changes never write to the environment. - if (grantChanges) enable(); + if (commit) { + // The endpoint save awaited above, so this acknowledges the choice created + // before it; without that await, choosing directly is equally safe. + if (endpointChanged) commit(); + else enable(); + } close(); }; @@ -428,7 +441,7 @@ export function CloudVoiceConfiguration({ aria-label="Endpoint base URL" placeholder="https://example.com/v1" value={endpoint} - disabled={busy} + disabled={busy || !canManage} autoComplete="off" spellCheck={false} onChange={(event) => setEndpoint(event.target.value)} @@ -540,7 +553,7 @@ export function CloudVoiceConfiguration({ size="sm" variant="destructive" disabled={busy} - onClick={() => void saveKey(null, false)} + onClick={() => void saveKey(null, null)} > Remove key diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 1c0f36d10a957..934645d2c697c 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -126,6 +126,11 @@ export function useVoiceTranscriptionDestination() { cloud && (cloud.provider === "openrouter" || cloud.provider === "custom") ? cloud.model : undefined; + // A custom grant is bound to the exact endpoint it consented to; a server-side + // URL change voids it until the destination is chosen again. + const staleEndpoint = + cloud?.provider === "custom" && + cloud.endpoint !== (environment?.serverConfig?.settings?.cloudSpeechCustomBaseUrl ?? ""); const supported = environment?.serverConfig?.environment.capabilities[ provider ? "cloudVoiceTranscription" : "voiceTranscription" @@ -140,7 +145,7 @@ export function useVoiceTranscriptionDestination() { provider && environment ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` : (environment?.label ?? null), - prepared: supported && canDictate ? connection : null, + prepared: supported && canDictate && !staleEndpoint ? connection : null, }; } diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 69fe4ff46f155..882e6a0304c8a 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -390,6 +390,8 @@ export const VoiceTranscriptionDestination = Schema.Union([ environmentId: EnvironmentId, provider: Schema.Literal("custom"), model: SpeechModelId, + /** The base URL this device consented to; a later server-side change voids the grant. */ + endpoint: Schema.String, }), ]); export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type; From 132e5c5b279e96045c45ff07465a432172f9e1dd Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 10 Nov 2026 04:23:51 +0000 Subject: [PATCH 3/3] fix(voice): verify custom endpoint consent on the server and fix CI - send the consented endpoint with each custom recording; the server uploads only when it still matches the setting - reject non-https (except loopback http) and userinfo endpoint URLs - treat a custom grant bound to an old URL as not active so Manage endpoint can re-approve it - custom disclosure no longer claims vocabulary hints are sent - hide the custom provider on mobile, which has no endpoint form - update the transcribe query test for generic model ids and format --- .../features/settings/CloudVoiceSection.tsx | 2 +- .../features/voice-input/voiceDestination.ts | 2 ++ apps/server/src/speech/CloudSpeech.test.ts | 10 ++++++++- apps/server/src/speech/CloudSpeech.ts | 18 ++++++++++----- apps/server/src/speech/http.test.ts | 4 +--- apps/server/src/speech/http.ts | 1 + .../CloudVoiceConfiguration.browser.test.tsx | 20 ++++++++++++----- .../settings/CloudVoiceConfiguration.tsx | 22 +++++++++++++------ .../settings/VoiceSettingsPanel.tsx | 3 +-- apps/web/src/speech/browserVoiceInput.ts | 3 +++ .../src/speech/useEnvironmentSpeechInput.ts | 5 ++++- docs/internals/voice-input.md | 5 ++++- docs/user/voice-input.md | 2 ++ .../src/voice-input/environment.ts | 6 ++++- packages/contracts/src/environmentHttp.ts | 5 ++++- packages/contracts/src/settings.ts | 4 +--- packages/contracts/src/speech.ts | 2 +- packages/shared/src/speech.ts | 4 ++-- 18 files changed, 84 insertions(+), 34 deletions(-) diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 684f8dbc83769..61e66200e78cc 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -238,7 +238,7 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () => /> ) : ( CloudSpeechProvider.literals.map((provider) => { - if (editing && provider !== editing.provider) return null; + if (provider === "custom" || (editing && provider !== editing.provider)) return null; const label = CLOUD_SPEECH_LABELS[provider]; const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; const routed = provider === "openrouter"; diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index dd27cca674888..1c36ad66ba253 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -52,6 +52,8 @@ function cloudVoiceDestination( ): CloudDestination | null { if (provider === "openrouter") return model ? { kind: "cloud", environmentId, provider, model } : null; + // Custom endpoints are configured from web and desktop only. + if (provider === "custom") return null; return { kind: "cloud", environmentId, provider }; } diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 48ae1dbfc9cb7..10fd7cf5f980b 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -233,18 +233,20 @@ it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart" new Uint8Array(new Float32Array([0, 0.5]).buffer), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, model, + baseUrl, ) .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); }); yield* service.setKey("custom", Redacted.make("custom-key")); const pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer); - const transcribe = (model: string | undefined) => + const transcribe = (model: string | undefined, endpoint = "http://localhost:8080/v1") => service .transcribe( "custom", pcm, { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, model, + endpoint, ) .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ @@ -255,6 +257,9 @@ it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart" expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ reason: "model-unavailable", }); + expect(yield* Effect.flip(transcribe("whisper-1", "http://localhost:9090/v1"))).toMatchObject({ + reason: "model-unavailable", + }); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe("whisper-1")).toBe("hello"); const [url, init] = upstream.mock.calls[0]!; @@ -268,6 +273,9 @@ it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart" expect(yield* Effect.flip(attempt("http://example.com/v1", "whisper-1"))).toMatchObject({ reason: "model-unavailable", }); + expect(yield* Effect.flip(attempt("ftp://localhost/v1", "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ reason: "model-unavailable", }); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3c5bdec17f3ff..4ac4327a7df95 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -109,12 +109,13 @@ function customTranscriptionsUrl(baseUrl: string) { } catch { return null; } - if (url.search || url.hash) return null; + if (url.search || url.hash || url.username || url.password) return null; const hostname = url.hostname.toLowerCase(); - const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + const host = + hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; const local = host === "localhost" || host === "::1" || host === "127.0.0.1" || host.endsWith(".localhost"); - if (url.protocol === "http:" && !local) return null; + if (url.protocol !== "https:" && !(url.protocol === "http:" && local)) return null; return `${url.origin}${url.pathname.replace(/\/+$/, "")}/audio/transcriptions`; } @@ -142,6 +143,7 @@ export class CloudSpeech extends Context.Service< pcm: Uint8Array, options: SpeechTranscriptionOptions, model?: string | undefined, + endpoint?: string | undefined, ) => Effect.Effect; } >()("t3/speech/CloudSpeech") {} @@ -212,13 +214,17 @@ export const layer = Layer.effect( pcm: Uint8Array, options: SpeechTranscriptionOptions, model?: string, + endpoint?: string, ) { const source = (yield* credentials)[provider]; if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); if (provider === "openrouter" && (!model || !isModelId(model))) return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); - const customUrl = - provider === "custom" ? customTranscriptionsUrl((yield* settings.getSettings).cloudSpeechCustomBaseUrl) : null; + // A custom grant names the base URL it consented to; audio never goes anywhere else. + const consentedUrl = Effect.map(settings.getSettings, ({ cloudSpeechCustomBaseUrl }) => + endpoint === cloudSpeechCustomBaseUrl ? customTranscriptionsUrl(endpoint) : null, + ); + const customUrl = provider === "custom" ? yield* consentedUrl : null; if (provider === "custom" && (!model || !customUrl)) return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const samples = yield* Effect.try({ @@ -258,6 +264,8 @@ export const layer = Layer.effect( provider, reason: current ? "key-changed" : "missing-key", }); + if (provider === "custom" && (yield* consentedUrl) !== customUrl) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const response = yield* execute(request).pipe(Effect.mapError(unavailable)); if (response.status < 200 || response.status >= 300) return yield* new CloudSpeechError({ diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index f67129be60423..7709bfe6b47d0 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -221,9 +221,7 @@ effectIt.effect( headers: { "content-type": "application/octet-stream" }, body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), }; - expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe( - 400, - ); + expect((yield* call("openrouter/transcribe?model=", audio)).status).toBe(400); expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe( 422, ); diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index eb4a7a01b88fb..47e8198d17f8a 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -105,6 +105,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group( decoded.value.pcm, decoded.value.options, args.query.model, + args.query.endpoint, ) .pipe( Effect.catchTags({ diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 856669f75e71b..8f483fa7f2b16 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -13,6 +13,10 @@ import { SettingsScopeProvider } from "./SettingsScopeContext"; vi.mock(import("../../state/session"), { spy: true }); const persistedSettings: unknown[] = []; let configuredEndpoint = ""; +function setup() { + persistedSettings.length = 0; + configuredEndpoint = ""; +} vi.mock(import("../../state/use-atom-command"), async (original) => ({ ...(await original()), useAtomCommand: (() => async (input: unknown) => { @@ -250,6 +254,7 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn }); it("configures a custom OpenAI-compatible endpoint with its URL, model, and key", async () => { + setup(); const id = EnvironmentId.make("voice-host"); const target = new PrimaryConnectionTarget({ environmentId: id, @@ -308,20 +313,25 @@ it("configures a custom OpenAI-compatible endpoint with its URL, model, and key" const dialog = page.getByRole("dialog"); const save = dialog.getByRole("button", { name: "Save and enable" }); await expect.element(save).toBeDisabled(); - await dialog.getByLabelText("Custom endpoint API key", { exact: true }).fill("custom-key"); + await dialog + .getByLabelText("Custom endpoint API key", { exact: true }) + .fill("custom-key"); await dialog.getByRole("textbox", { name: "Transcription model" }).fill("whisper-1"); const endpoint = dialog.getByRole("textbox", { name: "Endpoint base URL" }); await endpoint.fill("http://insecure.example.com/v1"); await expect.element(save).toBeDisabled(); await endpoint.fill("https://stt.example.com/v1"); - await expect.element( - dialog.getByText("your endpoint at stt.example.com", { exact: false }), - ).toBeVisible(); + await expect + .element(dialog.getByText("your endpoint at stt.example.com", { exact: false })) + .toBeVisible(); await userEvent.keyboard("{Enter}"); await expect.element(dialog).not.toBeInTheDocument(); await expect.element(destination).toHaveTextContent("Custom endpoint via Laptop"); expect(persistedSettings).toEqual([ - { environmentId: id, input: { patch: { cloudSpeechCustomBaseUrl: "https://stt.example.com/v1" } } }, + { + environmentId: id, + input: { patch: { cloudSpeechCustomBaseUrl: "https://stt.example.com/v1" } }, + }, ]); expect(writes).toEqual([{ key: "custom-key" }]); expect(grantDestination()).toMatchObject({ diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index f9e418bb14d5e..bc74c438ad0ea 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -95,7 +95,7 @@ function isCustomEndpoint(value: string) { } catch { return false; } - if (url.search || url.hash) return false; + if (url.search || url.hash || url.username || url.password) return false; const host = url.hostname.toLowerCase(); const loopback = host === "localhost" || host === "[::1]" || host === "127.0.0.1" || host.endsWith(".localhost"); @@ -169,7 +169,7 @@ function ModelPicker({ export function CloudVoiceConfiguration({ environment, provider, - active, + active: selected, close, }: { environment: Pick; @@ -194,7 +194,7 @@ export function CloudVoiceConfiguration({ const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; const currentModel = - active && + selected && grant?.destination.kind === "cloud" && (grant.destination.provider === "openrouter" || grant.destination.provider === "custom") ? grant.destination.model @@ -206,6 +206,12 @@ export function CloudVoiceConfiguration({ const savedEndpoint = environment.serverConfig?.settings?.cloudSpeechCustomBaseUrl ?? ""; const [endpoint, setEndpoint] = useState(savedEndpoint); const endpointChanged = custom && endpoint.trim() !== savedEndpoint; + // A custom grant bound to another URL is void, so it must be approved again. + const active = + selected && + (grant?.destination.kind !== "cloud" || + grant.destination.provider !== "custom" || + grant.destination.endpoint === savedEndpoint); const endpointReady = !custom || isCustomEndpoint(endpoint); const [remove, setRemove] = useState(false); const [busy, setBusy] = useState(false); @@ -257,8 +263,8 @@ export function CloudVoiceConfiguration({ endpoint: endpoint.trim(), } : routed && isModelId(model) - ? { kind: "cloud", provider: "openrouter", environmentId: id, model } - : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + ? { kind: "cloud", provider: "openrouter", environmentId: id, model } + : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed ? modelChosen && @@ -404,7 +410,7 @@ export function CloudVoiceConfiguration({ : active && !routed ? `Anyone with dictation access to ${environment.label} can use this API key.` : cloudSpeechDisclosure(provider, environment.label, { - endpoint: custom ? (endpoint.trim() || savedEndpoint) : undefined, + endpoint: custom ? endpoint.trim() || savedEndpoint : undefined, })} @@ -420,7 +426,9 @@ export function CloudVoiceConfiguration({ AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( -

Update {environment.label} to use {name} dictation.

+

+ Update {environment.label} to use {name} dictation. +

) : saved === undefined ? (

Checking for an API key…

) : routed ? ( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 0445c2288a3bc..d8febe222c8b4 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -125,8 +125,7 @@ const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; - if (provider === "custom") - return ; + if (provider === "custom") return ; return ( boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; @@ -51,6 +53,7 @@ export function createBrowserVoiceInputPlatform(input: { recordingOptions, input.hasAudioConsent, input.cloudModel, + input.cloudEndpoint, ) : transcribeEnvironmentPcm( input.prepared, diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 934645d2c697c..00908e176108e 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -140,6 +140,7 @@ export function useVoiceTranscriptionDestination() { environmentId, ...(provider ? { cloudProvider: provider } : {}), ...(model ? { cloudModel: model } : {}), + ...(cloud?.provider === "custom" ? { cloudEndpoint: cloud.endpoint } : {}), grantId: grant?.id ?? null, label: provider && environment @@ -156,6 +157,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; + const cloudEndpoint = destination.cloudEndpoint; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -268,6 +270,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { prepared, cloudProvider, cloudModel, + cloudEndpoint, hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -381,7 +384,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider, cloudModel]); + }, [prepared, grantId, cloudProvider, cloudModel, cloudEndpoint]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index ecb38f7a2d1a1..5b4edaa965c06 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -58,7 +58,10 @@ URL is a server setting, validated to https (or loopback http), with `/audio/transcriptions` appended and the grant's model sent in the form. The grant names the exact model for the same reason OpenRouter's does. The endpoint is disclosed by host in the consent dialog but is otherwise unvetted; the -response must still decode as `{ text }`. +response must still decode as `{ text }`. A custom grant also names the base URL +it consented to; the client sends it with each recording and the server refuses +to upload unless it still matches the setting, so changing the URL voids every +device's grant. ## Server side diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 73eddf4062bd0..c3b66510bec36 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -53,6 +53,8 @@ transcription server instead of a hosted provider. Enter the server's base URL name, and an API key; audio is sent to the URL's `/audio/transcriptions` route. Plain `http` is only accepted for localhost servers. The model, like the OpenRouter model, is per device; the URL and key are shared by the environment. +Changing the URL stops dictation on every device until it chooses the custom +endpoint again under **Manage endpoint**. T3 cannot verify what a custom endpoint retains, so only point it at servers you trust. diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 74d44567dca92..1fe160b5374c1 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -131,8 +131,12 @@ export const transcribeCloudPcm = ( options: SpeechTranscriptionOptions, canSend: () => boolean, model?: SpeechModelId, + endpoint?: string, ) => { - const query = model === undefined ? {} : { model }; + const query = { + ...(model === undefined ? {} : { model }), + ...(endpoint === undefined ? {} : { endpoint }), + }; return request({ group: "voice", prepared, diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 2439120eec7a9..0dffecb3da639 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -822,7 +822,10 @@ export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice") HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", { headers: OptionalBearerHeaders, params: Schema.Struct({ provider: CloudSpeechProvider }), - query: Schema.Struct({ model: Schema.optionalKey(SpeechModelId) }), + query: Schema.Struct({ + model: Schema.optionalKey(SpeechModelId), + endpoint: Schema.optionalKey(Schema.String), + }), payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()), success: EnvironmentSpeechTranscriptionResult, error: [ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index e78af2ecab957..5edfa54a3049f 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1369,9 +1369,7 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault( - Effect.succeed({ openai: "", xai: "", openrouter: "", custom: "" }), - ), + Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "", custom: "" })), ), /** Base URL of the OpenAI-compatible transcription endpoint, including the version path (e.g. `https://host/v1`). */ cloudSpeechCustomBaseUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index 403e30cfe4295..65c586424b738 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -76,7 +76,7 @@ export class CloudSpeechError extends Schema.TaggedError()( case "rate-limit": return "The custom endpoint reached a usage limit. Check the endpoint's quota or try again later."; case "model-unavailable": - return "Set the custom endpoint URL and a model name in Settings → Voice."; + return "Check the custom endpoint URL and model in Settings → Voice, then choose it again."; case "unavailable": return "The custom endpoint couldn’t transcribe this recording. Check its URL in Settings → Voice."; } diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index 8e7ce2f79583e..ae5067d89c652 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -25,8 +25,8 @@ export function cloudSpeechDisclosure( host = ""; } return host - ? `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to your endpoint at ${host}.` - : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to the custom endpoint configured on it.`; + ? `After you stop, ${environment} sends your recording to your endpoint at ${host}.` + : `After you stop, ${environment} sends your recording to the custom endpoint configured on it.`; } return `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; }