diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 6ca61ece7b90a..345bb77d070a7 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -276,6 +276,7 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () => return ( {CloudSpeechProvider.literals.map((provider, index) => { + if (provider === "custom") return null; const label = CLOUD_SPEECH_LABELS[provider]; const routed = provider === "openrouter"; const key = saved[provider]; diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index dd27cca674888..1c36ad66ba253 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -52,6 +52,8 @@ function cloudVoiceDestination( ): CloudDestination | null { if (provider === "openrouter") return model ? { kind: "cloud", environmentId, provider, model } : null; + // Custom endpoints are configured from web and desktop only. + if (provider === "custom") return null; return { kind: "cloud", environmentId, provider }; } diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 347b43dd6f44c..d9ded2c303dc9 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1811,6 +1811,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: SECRET_REDACTED, xai: SECRET_REDACTED, openrouter: "", + custom: "", }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); @@ -1841,6 +1842,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: usageReference, xai: "literal-fixture", openrouter: "", + custom: "", }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); @@ -1926,7 +1928,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); + assert.deepEqual(removed.cloudSpeechCredentials, { + openai: "", + xai: work, + openrouter: "", + custom: "", + }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1939,6 +1946,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", "cloudSpeechCredentials.openrouter", + "cloudSpeechCredentials.custom", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 9f73b2512841c..9b91cfad60cb5 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, + cloudSpeechCredentials: { + openai: source("voice#key"), + xai: "", + openrouter: "", + custom: "", + }, usageLimitSources, textGenerators: {}, easDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 112c7a88d9b3b..10fd7cf5f980b 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -49,7 +49,13 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + expect(yield* service.configured).toEqual({ + openai: true, + xai: true, + openrouter: false, + custom: false, + customEndpoint: "", + }); yield* Effect.flip(transcribe(new Uint8Array(3))); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe()).toBe("hello"); @@ -198,3 +204,85 @@ it.effect("removing a key while a catalog read is pending prevents the upload", expect(exit._tag).toBe("Failure"); }), ); + +it.effect("sends to a configured custom OpenAI-compatible endpoint as multipart", () => + Effect.gen(function* () { + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + const stack = (baseUrl: string | undefined) => + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide( + ServerSettings.ServerSettingsService.layerTest( + baseUrl === undefined ? {} : { cloudSpeechCustomBaseUrl: baseUrl }, + ), + ), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide(yield* Layer.build(stack("http://localhost:8080/v1"))), + ); + // Builds a separate stack for a different endpoint configuration. + const attempt = (baseUrl: string | undefined, model: string | undefined) => + Effect.gen(function* () { + const context = yield* Layer.build(stack(baseUrl)); + const scoped = yield* CloudSpeech.CloudSpeech.pipe(Effect.provide(context)); + yield* scoped.setKey("custom", Redacted.make("custom-key")); + return yield* scoped + .transcribe( + "custom", + new Uint8Array(new Float32Array([0, 0.5]).buffer), + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + baseUrl, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + }); + yield* service.setKey("custom", Redacted.make("custom-key")); + const pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer); + const transcribe = (model: string | undefined, endpoint = "http://localhost:8080/v1") => + service + .transcribe( + "custom", + pcm, + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + endpoint, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ + provider: "custom", + reason: "model-unavailable", + }); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ + reason: "model-unavailable", + }); + expect(yield* Effect.flip(transcribe("whisper-1", "http://localhost:9090/v1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* transcribe("whisper-1")).toBe("hello"); + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe("http://localhost:8080/v1/audio/transcriptions"); + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer custom-key"); + const form = init?.body as FormData; + expect(form.get("model")).toBe("whisper-1"); + expect(form.get("language")).toBe("en"); + expect([...form.keys()].at(-1)).toBe("file"); + expect(yield* Effect.flip(attempt("http://example.com/v1", "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(yield* Effect.flip(attempt("ftp://localhost/v1", "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(yield* Effect.flip(attempt(undefined, "whisper-1"))).toMatchObject({ + reason: "model-unavailable", + }); + expect(upstream).toHaveBeenCalledTimes(1); + yield* service.setKey("custom", null); + expect((yield* service.configured).custom).toBe(false); + expect(yield* Effect.flip(transcribe("whisper-1"))).toMatchObject({ reason: "missing-key" }); + expect(upstream).toHaveBeenCalledTimes(1); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3b7eab6b5242b..4ac4327a7df95 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -101,6 +101,33 @@ function openRouterBody(model: string, audio: ArrayBuffer, language: string) { }); } +/** The transcriptions URL for a configured custom base URL, or null when it is unusable. */ +function customTranscriptionsUrl(baseUrl: string) { + let url: URL; + try { + url = new URL(baseUrl); + } catch { + return null; + } + if (url.search || url.hash || url.username || url.password) return null; + const hostname = url.hostname.toLowerCase(); + const host = + hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname; + const local = + host === "localhost" || host === "::1" || host === "127.0.0.1" || host.endsWith(".localhost"); + if (url.protocol !== "https:" && !(url.protocol === "http:" && local)) return null; + return `${url.origin}${url.pathname.replace(/\/+$/, "")}/audio/transcriptions`; +} + +/** The generic multipart body every OpenAI-compatible transcription server accepts. */ +function customBody(model: string, audio: ArrayBuffer, language: string) { + const form = new FormData(); + form.set("model", model); + if (language !== "auto") form.set("language", language); + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); + return HttpClientRequest.bodyFormData(form); +} + export class CloudSpeech extends Context.Service< CloudSpeech, { @@ -116,6 +143,7 @@ export class CloudSpeech extends Context.Service< pcm: Uint8Array, options: SpeechTranscriptionOptions, model?: string | undefined, + endpoint?: string | undefined, ) => Effect.Effect; } >()("t3/speech/CloudSpeech") {} @@ -129,11 +157,13 @@ export const layer = Layer.effect( const credentials = settings.getSettings.pipe( Effect.map((value) => value.cloudSpeechCredentials), ); - const configured = credentials.pipe( - Effect.map(({ openai, xai, openrouter }) => ({ - openai: Boolean(openai), - xai: Boolean(xai), - openrouter: Boolean(openrouter), + const configured = settings.getSettings.pipe( + Effect.map((value) => ({ + openai: Boolean(value.cloudSpeechCredentials.openai), + xai: Boolean(value.cloudSpeechCredentials.xai), + openrouter: Boolean(value.cloudSpeechCredentials.openrouter), + custom: Boolean(value.cloudSpeechCredentials.custom), + customEndpoint: value.cloudSpeechCustomBaseUrl, })), ); const execute = (request: HttpClientRequest.HttpClientRequest) => @@ -184,11 +214,19 @@ export const layer = Layer.effect( pcm: Uint8Array, options: SpeechTranscriptionOptions, model?: string, + endpoint?: string, ) { const source = (yield* credentials)[provider]; if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); if (provider === "openrouter" && (!model || !isModelId(model))) return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); + // A custom grant names the base URL it consented to; audio never goes anywhere else. + const consentedUrl = Effect.map(settings.getSettings, ({ cloudSpeechCustomBaseUrl }) => + endpoint === cloudSpeechCustomBaseUrl ? customTranscriptionsUrl(endpoint) : null, + ); + const customUrl = provider === "custom" ? yield* consentedUrl : null; + if (provider === "custom" && (!model || !customUrl)) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const samples = yield* Effect.try({ try: () => decodeSpeechPcm(pcm), catch: () => @@ -208,11 +246,15 @@ export const layer = Layer.effect( ); if (!isApiKey(key)) return yield* unreadable(); const audio = wave(samples); - const request = HttpClientRequest.post(endpoints[provider].url).pipe( + const request = HttpClientRequest.post( + provider === "custom" ? customUrl! : endpoints[provider].url, + ).pipe( HttpClientRequest.bearerToken(Redacted.make(key)), provider === "openrouter" ? openRouterBody(model!, audio, language) - : audioBody(provider, audio, language, options), + : provider === "custom" + ? customBody(model!, audio, language) + : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); // A catalog fetch or vault lookup may have outlived the credential it started with. @@ -222,6 +264,8 @@ export const layer = Layer.effect( provider, reason: current ? "key-changed" : "missing-key", }); + if (provider === "custom" && (yield* consentedUrl) !== customUrl) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); const response = yield* execute(request).pipe(Effect.mapError(unavailable)); if (response.status < 200 || response.status >= 300) return yield* new CloudSpeechError({ diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index f67129be60423..7709bfe6b47d0 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -221,9 +221,7 @@ effectIt.effect( headers: { "content-type": "application/octet-stream" }, body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), }; - expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe( - 400, - ); + expect((yield* call("openrouter/transcribe?model=", audio)).status).toBe(400); expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe( 422, ); diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index eb4a7a01b88fb..47e8198d17f8a 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -105,6 +105,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group( decoded.value.pcm, decoded.value.options, args.query.model, + args.query.endpoint, ) .pipe( Effect.catchTags({ diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx index 0295603ed2038..8f483fa7f2b16 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -11,6 +11,32 @@ import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; import { SettingsScopeProvider } from "./SettingsScopeContext"; vi.mock(import("../../state/session"), { spy: true }); +const persistedSettings: unknown[] = []; +let configuredEndpoint = ""; +function setup() { + persistedSettings.length = 0; + configuredEndpoint = ""; +} +vi.mock(import("../../state/use-atom-command"), async (original) => ({ + ...(await original()), + useAtomCommand: (() => async (input: unknown) => { + persistedSettings.push(input); + configuredEndpoint = + typeof input === "object" && + input !== null && + "input" in input && + typeof input.input === "object" && + input.input !== null && + "patch" in input.input && + typeof input.input.patch === "object" && + input.input.patch !== null && + "cloudSpeechCustomBaseUrl" in input.input.patch && + typeof input.input.patch.cloudSpeechCustomBaseUrl === "string" + ? input.input.patch.cloudSpeechCustomBaseUrl + : configuredEndpoint; + return { _tag: "Success" }; + }) as never, +})); vi.mock("../../env", async (original) => ({ ...(await original()), isSecretsFeatureEnabled: true, @@ -29,6 +55,7 @@ vi.mock("../../state/environments", async (importOriginal) => ({ secretProviders: { work: { kind: "one_password", label: "Work vault", config: { account: "work" } }, }, + cloudSpeechCustomBaseUrl: configuredEndpoint, }, environment: { platform: { os: "linux", arch: "x64" }, @@ -225,3 +252,96 @@ it("OpenRouter needs an exact model, and changing it writes no credential", asyn }, ); }); + +it("configures a custom OpenAI-compatible endpoint with its URL, model, and key", async () => { + setup(); + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination(null); + const writes: unknown[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: true, xai: true, custom: true })); + if (url.pathname === "/api/voice/cloud/openrouter/models") + return Promise.resolve(Response.json({ models: [] })); + if (url.pathname === "/api/voice/cloud/custom/key") + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + return fetch(request, init); + }); + const grantDestination = () => { + const raw = window.localStorage.getItem("t3code:voice-audio-grant:v1"); + const parsed = raw + ? (JSON.parse(raw) as { + destination?: Record; + pending?: { destination?: Record }; + }) + : null; + return parsed?.destination ?? parsed?.pending?.destination ?? null; + }; + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await destination.click(); + await page.getByRole("option", { name: "Custom endpoint via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + const save = dialog.getByRole("button", { name: "Save and enable" }); + await expect.element(save).toBeDisabled(); + await dialog + .getByLabelText("Custom endpoint API key", { exact: true }) + .fill("custom-key"); + await dialog.getByRole("textbox", { name: "Transcription model" }).fill("whisper-1"); + const endpoint = dialog.getByRole("textbox", { name: "Endpoint base URL" }); + await endpoint.fill("http://insecure.example.com/v1"); + await expect.element(save).toBeDisabled(); + await endpoint.fill("https://stt.example.com/v1"); + await expect + .element(dialog.getByText("your endpoint at stt.example.com", { exact: false })) + .toBeVisible(); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("Custom endpoint via Laptop"); + expect(persistedSettings).toEqual([ + { + environmentId: id, + input: { patch: { cloudSpeechCustomBaseUrl: "https://stt.example.com/v1" } }, + }, + ]); + expect(writes).toEqual([{ key: "custom-key" }]); + expect(grantDestination()).toMatchObject({ + provider: "custom", + environmentId: "voice-host", + model: "whisper-1", + endpoint: "https://stt.example.com/v1", + }); + }, + ); + }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 95dcee4cffb45..bc74c438ad0ea 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -18,6 +18,7 @@ import { CLOUD_SPEECH_LABELS, CLOUD_SPEECH_PROVIDER_NAMES, isSecretReference, + SpeechModelId, type CloudSpeechModel, type CloudSpeechProvider, type SecretProviderId, @@ -31,6 +32,8 @@ import type { HttpClient } from "effect/http"; import { Atom, AsyncResult } from "effect/reactivity"; import { useId, useState } from "react"; import { isSecretsFeatureEnabled } from "../../env"; +import { serverEnvironment } from "../../state/server"; +import { useAtomCommand } from "../../state/use-atom-command"; import { inEnvironmentSpeechRuntime, runEnvironmentSpeech, @@ -83,6 +86,21 @@ function useCloudRead( } const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); +const isCustomModelId = Schema.is(SpeechModelId); +/** Mirror of the server check: https, or plain http only for loopback hosts. */ +function isCustomEndpoint(value: string) { + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return false; + } + if (url.search || url.hash || url.username || url.password) return false; + const host = url.hostname.toLowerCase(); + const loopback = + host === "localhost" || host === "[::1]" || host === "127.0.0.1" || host.endsWith(".localhost"); + return url.protocol === "https:" || (url.protocol === "http:" && loopback); +} type CloudDestination = Extract; function ModelPicker({ @@ -151,7 +169,7 @@ function ModelPicker({ export function CloudVoiceConfiguration({ environment, provider, - active, + active: selected, close, }: { environment: Pick; @@ -162,22 +180,39 @@ export function CloudVoiceConfiguration({ const id = environment.environmentId; const name = CLOUD_SPEECH_LABELS[provider]; const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; - const routed = provider === "openrouter"; + const routed = provider === "openrouter" || provider === "custom"; + const custom = provider === "custom"; const formId = useId(); const prepared = Option.getOrNull(usePreparedConnection(id)); const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys); const [models, refreshModels] = useCloudRead( - routed && canUse ? prepared : null, + provider === "openrouter" && canUse ? prepared : null, getCloudSpeechModels, ); const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; const currentModel = - active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter" + selected && + grant?.destination.kind === "cloud" && + (grant.destination.provider === "openrouter" || grant.destination.provider === "custom") ? grant.destination.model : undefined; + const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { + label: "custom speech endpoint update", + reportFailure: false, + }); + const savedEndpoint = environment.serverConfig?.settings?.cloudSpeechCustomBaseUrl ?? ""; + const [endpoint, setEndpoint] = useState(savedEndpoint); + const endpointChanged = custom && endpoint.trim() !== savedEndpoint; + // A custom grant bound to another URL is void, so it must be approved again. + const active = + selected && + (grant?.destination.kind !== "cloud" || + grant.destination.provider !== "custom" || + grant.destination.endpoint === savedEndpoint); + const endpointReady = !custom || isCustomEndpoint(endpoint); const [remove, setRemove] = useState(false); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); @@ -212,24 +247,37 @@ export function CloudVoiceConfiguration({ // Fixed providers edit the key only when nothing is saved or the device already uses them. const editingKey = canManage && saved !== undefined && (routed || active || !saved); const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; - const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); + const customModelReady = isCustomModelId(model.trim()); + const modelChosen = custom + ? customModelReady + : isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); const keyReady = draft.trim() ? credential !== undefined : Boolean(saved); const modelChanged = model !== (currentModel ?? ""); const destination = (): CloudDestination => - routed && isModelId(model) - ? { kind: "cloud", provider: "openrouter", environmentId: id, model } - : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + custom && customModelReady + ? { + kind: "cloud", + provider: "custom", + environmentId: id, + model: model.trim(), + endpoint: endpoint.trim(), + } + : routed && isModelId(model) + ? { kind: "cloud", provider: "openrouter", environmentId: id, model } + : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed - ? modelChosen && keyReady && (!active || modelChanged || keyChanged) + ? modelChosen && + endpointReady && + keyReady && + (!active || modelChanged || keyChanged || endpointChanged) : credential !== undefined; - const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => { + const saveKey = async (value: SettingsSecretValue | null, commit: (() => void) | null) => { if (!connected || !canManage || busy) return; setError(""); setBusy(true); try { - const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null; await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value)); refresh(); commit?.(); @@ -239,13 +287,40 @@ export function CloudVoiceConfiguration({ } setBusy(false); }; - const submit = () => { + const submit = async () => { if (!canSubmit || busy) return; - const grantChanges = !active || modelChanged; + const grantChanges = !active || modelChanged || endpointChanged; + // Created before the first await so an Off chosen elsewhere during the save + // is never overwritten by this submission's commit. + const commit = grantChanges ? pendingCloudAudioChoice(destination()) : null; + if (endpointChanged) { + setError(""); + setBusy(true); + try { + const result = await updateSettings({ + environmentId: id, + input: { patch: { cloudSpeechCustomBaseUrl: endpoint.trim() } }, + }); + setBusy(false); + if (result._tag !== "Success") { + setError("Couldn’t save the endpoint URL. Check your connection and try again."); + return; + } + } catch { + setBusy(false); + setError("Couldn’t save the endpoint URL. Check your connection and try again."); + return; + } + } if (credential !== undefined && (!routed || keyChanged)) - return void saveKey(credential, grantChanges); + return void saveKey(credential, commit); // Model-only changes never write to the environment. - if (grantChanges) enable(); + if (commit) { + // The endpoint save awaited above, so this acknowledges the choice created + // before it; without that await, choosing directly is equally safe. + if (endpointChanged) commit(); + else enable(); + } close(); }; @@ -323,7 +398,9 @@ export function CloudVoiceConfiguration({ ? `Remove the ${company} key?` : active ? routed - ? "Manage OpenRouter dictation" + ? custom + ? "Manage custom dictation" + : "Manage OpenRouter dictation" : `Manage ${company} key` : `Use ${name} for dictation?`} @@ -332,7 +409,9 @@ export function CloudVoiceConfiguration({ ? `Everyone using ${name} for dictation on ${environment.label} will lose access.` : active && !routed ? `Anyone with dictation access to ${environment.label} can use this API key.` - : cloudSpeechDisclosure(provider, environment.label)} + : cloudSpeechDisclosure(provider, environment.label, { + endpoint: custom ? endpoint.trim() || savedEndpoint : undefined, + })} {!remove && ( @@ -347,7 +426,9 @@ export function CloudVoiceConfiguration({ AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( -

Update {environment.label} to use OpenRouter dictation.

+

+ Update {environment.label} to use {name} dictation. +

) : saved === undefined ? (

Checking for an API key…

) : routed ? ( @@ -356,29 +437,70 @@ export function CloudVoiceConfiguration({ className="grid gap-3" onSubmit={(event) => { event.preventDefault(); - submit(); + void submit(); }} > -
- - {listedModels ? ( - - ) : AsyncResult.isFailure(models) ? ( - - ) : ( -

- Loading transcription models… -

- )} -
+ {custom ? ( + <> +
+ + setEndpoint(event.target.value)} + /> +

+ Base URL of an OpenAI-compatible server, including the version path. Audio + is sent to its /audio/transcriptions route. Plain http is only allowed for + localhost. +

+ {endpoint.trim() && !endpointReady ? ( +

+ Enter an https URL, or plain http for localhost. +

+ ) : null} +
+
+ + setModel(event.target.value)} + /> +
+ + ) : ( +
+ + {listedModels ? ( + + ) : AsyncResult.isFailure(models) ? ( + + ) : ( +

+ Loading transcription models… +

+ )} +
+ )} {editingKey ? (
{keyField}
) : !saved ? ( @@ -396,7 +518,7 @@ export function CloudVoiceConfiguration({ className="grid gap-2" onSubmit={(event) => { event.preventDefault(); - submit(); + void submit(); }} > {keyField} @@ -439,7 +561,7 @@ export function CloudVoiceConfiguration({ size="sm" variant="destructive" disabled={busy} - onClick={() => void saveKey(null, false)} + onClick={() => void saveKey(null, null)} > Remove key diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 495abdefb6b6e..d8febe222c8b4 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -125,6 +125,7 @@ const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; + if (provider === "custom") return ; return ( environment.environmentId === destination.environmentId, ); @@ -1397,7 +1402,9 @@ export function VoiceSettingsPanel() { {destination.cloudProvider && chosenEnvironment && - (canManageKey || destination.cloudProvider === "openrouter") && ( + (canManageKey || + destination.cloudProvider === "openrouter" || + destination.cloudProvider === "custom") && ( boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; @@ -51,6 +53,7 @@ export function createBrowserVoiceInputPlatform(input: { recordingOptions, input.hasAudioConsent, input.cloudModel, + input.cloudEndpoint, ) : transcribeEnvironmentPcm( input.prepared, diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index c569c14500be6..00908e176108e 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -122,7 +122,15 @@ export function useVoiceTranscriptionDestination() { const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; const provider = cloud?.provider; - const model = cloud?.provider === "openrouter" ? cloud.model : undefined; + const model = + cloud && (cloud.provider === "openrouter" || cloud.provider === "custom") + ? cloud.model + : undefined; + // A custom grant is bound to the exact endpoint it consented to; a server-side + // URL change voids it until the destination is chosen again. + const staleEndpoint = + cloud?.provider === "custom" && + cloud.endpoint !== (environment?.serverConfig?.settings?.cloudSpeechCustomBaseUrl ?? ""); const supported = environment?.serverConfig?.environment.capabilities[ provider ? "cloudVoiceTranscription" : "voiceTranscription" @@ -132,12 +140,13 @@ export function useVoiceTranscriptionDestination() { environmentId, ...(provider ? { cloudProvider: provider } : {}), ...(model ? { cloudModel: model } : {}), + ...(cloud?.provider === "custom" ? { cloudEndpoint: cloud.endpoint } : {}), grantId: grant?.id ?? null, label: provider && environment ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` : (environment?.label ?? null), - prepared: supported && canDictate ? connection : null, + prepared: supported && canDictate && !staleEndpoint ? connection : null, }; } @@ -148,6 +157,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; + const cloudEndpoint = destination.cloudEndpoint; const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); @@ -260,6 +270,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { prepared, cloudProvider, cloudModel, + cloudEndpoint, hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -373,7 +384,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider, cloudModel]); + }, [prepared, grantId, cloudProvider, cloudModel, cloudEndpoint]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index e381debdb792b..5b4edaa965c06 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -53,6 +53,16 @@ applies the account's safety-retention policy when choosing an endpoint. The cat holds no credential or consent state, which is why it may be cached while key presence is read fresh on every request. +The custom provider is the generic OpenAI-compatible multipart client: the base +URL is a server setting, validated to https (or loopback http), with +`/audio/transcriptions` appended and the grant's model sent in the form. The +grant names the exact model for the same reason OpenRouter's does. The endpoint +is disclosed by host in the consent dialog but is otherwise unvetted; the +response must still decode as `{ text }`. A custom grant also names the base URL +it consented to; the client sends it with each recording and the server refuses +to upload unless it still matches the setting, so changing the URL voids every +device's grant. + ## Server side The [speech service](../../apps/server/src/speech/SpeechService.ts) runs diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 7e5c36fa4d1eb..c3b66510bec36 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -47,6 +47,17 @@ Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied after transcription. If OpenRouter stops listing your model, dictation stops until you choose another. +**Custom endpoint** points the environment at your own OpenAI-compatible +transcription server instead of a hosted provider. Enter the server's base URL +(including the version path, for example `https://stt.example.com/v1`), a model +name, and an API key; audio is sent to the URL's `/audio/transcriptions` route. +Plain `http` is only accepted for localhost servers. The model, like the +OpenRouter model, is per device; the URL and key are shared by the environment. +Changing the URL stops dictation on every device until it chooses the custom +endpoint again under **Manage endpoint**. +T3 cannot verify what a custom endpoint retains, so only point it at servers you +trust. + ## Set up a model In **Settings → Voice → Transcription**, download a model and choose **Use model** if it diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 52fa5a0d1fe5b..1fe160b5374c1 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -4,10 +4,10 @@ import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts"; import type { - CloudSpeechModelId, CloudSpeechProvider, SettingsSecretValue, SpeechAcceleration, + SpeechModelId, SpeechModelUnloadTimeout, SpeechTranscriptionOptions, SpeechPostProcessingOptions, @@ -130,9 +130,13 @@ export const transcribeCloudPcm = ( pcm: Uint8Array, options: SpeechTranscriptionOptions, canSend: () => boolean, - model?: CloudSpeechModelId, + model?: SpeechModelId, + endpoint?: string, ) => { - const query = model === undefined ? {} : { model }; + const query = { + ...(model === undefined ? {} : { model }), + ...(endpoint === undefined ? {} : { endpoint }), + }; return request({ group: "voice", prepared, diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index db0bd2daae610..0dffecb3da639 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -52,7 +52,6 @@ import { CloudSpeechApiKey, CloudSpeechError, CloudSpeechKeys, - CloudSpeechModelId, CloudSpeechModels, CloudSpeechProvider, EnvironmentSpeechAccelerationRequest, @@ -63,6 +62,7 @@ import { EnvironmentSpeechPostProcessingResult, EnvironmentSpeechStatus, EnvironmentSpeechTranscriptionResult, + SpeechModelId, } from "./speech.ts"; import { SecretReference } from "./secretReference.ts"; import { @@ -822,7 +822,10 @@ export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice") HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", { headers: OptionalBearerHeaders, params: Schema.Struct({ provider: CloudSpeechProvider }), - query: Schema.Struct({ model: Schema.optionalKey(CloudSpeechModelId) }), + query: Schema.Struct({ + model: Schema.optionalKey(SpeechModelId), + endpoint: Schema.optionalKey(Schema.String), + }), payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()), success: EnvironmentSpeechTranscriptionResult, error: [ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd112d84bf9f4..5edfa54a3049f 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -58,6 +58,7 @@ import { SpeechCorrectionCue, SpeechCustomWords, SpeechLanguage, + SpeechModelId, SpeechModelUnloadTimeout, SpeechPostProcessingPromptSettings, } from "./speech.ts"; @@ -369,6 +370,7 @@ export type VoiceShortcutMode = typeof VoiceShortcutMode.Type; /** * Where this device sends microphone audio. A cloud grant names the forwarding environment and * the provider; an OpenRouter grant also names the exact model, since OpenRouter has no default. + * A custom endpoint grant names the exact model too: OpenAI-compatible servers have no default. */ export const VoiceTranscriptionDestination = Schema.Union([ Schema.Struct({ kind: Schema.Literal("environment"), environmentId: EnvironmentId }), @@ -383,6 +385,14 @@ export const VoiceTranscriptionDestination = Schema.Union([ provider: Schema.Literal("openrouter"), model: CloudSpeechModelId, }), + Schema.Struct({ + kind: Schema.Literal("cloud"), + environmentId: EnvironmentId, + provider: Schema.Literal("custom"), + model: SpeechModelId, + /** The base URL this device consented to; a later server-side change voids the grant. */ + endpoint: Schema.String, + }), ]); export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type; @@ -1328,6 +1338,7 @@ export const CloudSpeechCredentials = Schema.Struct({ openai: CloudSpeechCredential, xai: CloudSpeechCredential, openrouter: CloudSpeechCredential, + custom: CloudSpeechCredential, }); export const ServerSettings = Schema.Struct({ @@ -1358,8 +1369,10 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })), + Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "", custom: "" })), ), + /** Base URL of the OpenAI-compatible transcription endpoint, including the version path (e.g. `https://host/v1`). */ + cloudSpeechCustomBaseUrl: Schema.String.pipe(Schema.withDecodingDefault(Effect.succeed(""))), speechModelId: Schema.String.pipe( Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), ), @@ -1808,8 +1821,10 @@ export const ServerSettingsPatch = Schema.Struct({ openai: Schema.optionalKey(SettingsSecretValue), xai: Schema.optionalKey(SettingsSecretValue), openrouter: Schema.optionalKey(SettingsSecretValue), + custom: Schema.optionalKey(SettingsSecretValue), }), ), + cloudSpeechCustomBaseUrl: Schema.optionalKey(Schema.String), speechModelId: Schema.optionalKey(Schema.String), speechAcceleration: Schema.optionalKey(SpeechAcceleration), speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ac87096a24cce..65c586424b738 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -8,23 +8,28 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check( Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i), ); -export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]); +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter", "custom"]); export type CloudSpeechProvider = typeof CloudSpeechProvider.Type; export const CLOUD_SPEECH_LABELS = { openai: "OpenAI", xai: "Grok Voice Transcribe", openrouter: "OpenRouter", + custom: "Custom endpoint", } as const; export const CLOUD_SPEECH_PROVIDER_NAMES = { openai: "OpenAI", xai: "xAI", openrouter: "OpenRouter", + custom: "Custom endpoint", } as const; /** Servers that predate OpenRouter report only the first two providers. */ export const CloudSpeechKeys = Schema.Struct({ openai: Schema.Boolean, xai: Schema.Boolean, openrouter: Schema.optionalKey(Schema.Boolean), + custom: Schema.optionalKey(Schema.Boolean), + /** The configured custom endpoint base URL, for disclosure; empty until one is set. */ + customEndpoint: Schema.optionalKey(Schema.String), }); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type; @@ -58,6 +63,24 @@ export class CloudSpeechError extends Schema.TaggedError()( ) { override get message() { const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider]; + if (this.provider === "custom") { + switch (this.reason) { + case "missing-key": + return "Add an API key for the custom endpoint in Settings → Voice."; + case "key-changed": + return "The custom endpoint key changed before your recording was sent. Try again."; + case "unreadable-key": + return "Couldn’t read the custom endpoint API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent."; + case "rejected-key": + return "The custom endpoint rejected this API key. Check the key and endpoint URL in Settings → Voice."; + case "rate-limit": + return "The custom endpoint reached a usage limit. Check the endpoint's quota or try again later."; + case "model-unavailable": + return "Check the custom endpoint URL and model in Settings → Voice, then choose it again."; + case "unavailable": + return "The custom endpoint couldn’t transcribe this recording. Check its URL in Settings → Voice."; + } + } switch (this.reason) { case "missing-key": return `Add your ${name} API key in Settings → Voice.`; diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index c5ed9ab14ea0a..ae5067d89c652 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -10,10 +10,25 @@ import * as Schema from "effect/Schema"; const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); -export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) { - return provider === "openrouter" - ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.` - : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; +export function cloudSpeechDisclosure( + provider: CloudSpeechProvider, + environment: string, + { endpoint }: { endpoint?: string | undefined } = {}, +) { + if (provider === "openrouter") + return `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`; + if (provider === "custom") { + let host = ""; + try { + host = endpoint ? new URL(endpoint).host : ""; + } catch { + host = ""; + } + return host + ? `After you stop, ${environment} sends your recording to your endpoint at ${host}.` + : `After you stop, ${environment} sends your recording to the custom endpoint configured on it.`; + } + return `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; } // Only offer explicit hints understood by all cloud APIs; other languages can still be detected.