diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index 62a64102265de..f920b7d28d74d 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -11,7 +11,11 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts"; import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider"; import { AppText as Text } from "./ui/AppText"; import { cn } from "../lib/cn"; -import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; +import { + elevenLabsIcon, + getProviderClient, + openRouterIcon, +} from "@t3tools/client-runtime/provider-clients"; type ProviderIconProps = { readonly provider: string | null | undefined; @@ -100,7 +104,11 @@ export function ProviderIcon(props: ProviderIconProps) { } const packageIcon = - props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon; + props.provider === "openrouter" + ? openRouterIcon + : props.provider === "elevenlabs" + ? elevenLabsIcon + : getProviderClient(props.provider)?.icon; if (packageIcon) { const themeFill = (fill: { readonly light: string; readonly dark: string }) => isDarkMode ? fill.dark : fill.light; diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 347b43dd6f44c..8a0b79b7bd185 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1811,6 +1811,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: SECRET_REDACTED, xai: SECRET_REDACTED, openrouter: "", + elevenlabs: "", }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); @@ -1841,6 +1842,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" openai: usageReference, xai: "literal-fixture", openrouter: "", + elevenlabs: "", }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); @@ -1926,7 +1928,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); + assert.deepEqual(removed.cloudSpeechCredentials, { + openai: "", + xai: work, + openrouter: "", + elevenlabs: "", + }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1939,6 +1946,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", "cloudSpeechCredentials.openrouter", + "cloudSpeechCredentials.elevenlabs", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 9f73b2512841c..3acd0ae2b7e8d 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, + cloudSpeechCredentials: { + openai: source("voice#key"), + xai: "", + openrouter: "", + elevenlabs: "", + }, usageLimitSources, textGenerators: {}, easDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 112c7a88d9b3b..a5e5e4b1d553b 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -10,8 +10,36 @@ import { expect, vi } from "vite-plus/test"; import * as ServerSettings from "../serverSettings.ts"; import * as CloudSpeech from "./CloudSpeech.ts"; -it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved key", (provider) => +const wire = { + openai: { + url: "https://api.openai.com/v1/audio/transcriptions", + auth: ["authorization", "Bearer "], + model: ["model", "gpt-transcribe"], + keyterm: "keywords[]", + language: "languages[]", + }, + xai: { + url: "https://api.x.ai/v1/stt", + auth: ["authorization", "Bearer "], + model: ["model", "grok-voice-transcribe-2.0"], + keyterm: "keyterm", + language: "language", + }, + elevenlabs: { + url: "https://api.elevenlabs.io/v1/speech-to-text", + auth: ["xi-api-key", ""], + model: ["model_id", "scribe_v3"], + keyterm: "keyterms", + language: "language_code", + }, +} as const; + +const fixedProviders = Object.keys(wire) as Array; + +it.effect.each(fixedProviders)("sends only to %s using its own saved key", (provider) => Effect.gen(function* () { + const { url: endpoint, auth, model, keyterm, language } = wire[provider]; + const other = provider === "openai" ? "xai" : "openai"; const reference = { provider: SecretProviderId.make("work"), reference: "op://Work/Voice/key", @@ -43,31 +71,37 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved .transcribe(provider, pcm, { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en", - speechCustomWords: [{ term: "T3 Code", aliases: [] }], + speechCustomWords: [ + { term: "T3 Code", aliases: [] }, + { term: "{brace}", aliases: [] }, + ], }) .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); - yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); + yield* service.setKey(other, Redacted.make("other")); expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + expect(yield* service.configured).toEqual({ + openai: false, + xai: false, + openrouter: false, + elevenlabs: false, + [other]: true, + [provider]: true, + }); yield* Effect.flip(transcribe(new Uint8Array(3))); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe()).toBe("hello"); const [url, init] = upstream.mock.calls[0]!; - expect(String(url)).toBe( - { - openai: "https://api.openai.com/v1/audio/transcriptions", - xai: "https://api.x.ai/v1/stt", - }[provider], - ); + expect(String(url)).toBe(endpoint); expect(init?.redirect).toBe("error"); - expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen"); + expect(new Headers(init?.headers).get(auth[0])).toBe(`${auth[1]}chosen`); const form = init?.body as FormData; - expect(form.get("model")).toBe( - provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0", + expect(form.get(model[0])).toBe(model[1]); + // Scribe fails the whole request on bracket characters, so those terms stay local. + expect(form.getAll(keyterm)).toEqual( + provider === "elevenlabs" ? ["T3 Code"] : ["T3 Code", "{brace}"], ); - expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]); - expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en"); + expect(form.get(language)).toBe("en"); expect([...form.keys()].at(-1)).toBe("file"); const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer())); expect(wav.getUint32(24, true)).toBe(16_000); @@ -84,8 +118,8 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved resolved = "resolved-fixture"; expect(yield* transcribe()).toBe("hello"); expect(read).toHaveBeenCalledWith([reference]); - expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe( - "Bearer resolved-fixture", + expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get(auth[0])).toBe( + `${auth[1]}resolved-fixture`, ); resolved = undefined; expect(yield* Effect.flip(transcribe())).toMatchObject({ diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3b7eab6b5242b..6ffb2a2604c6d 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -20,7 +20,13 @@ import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; -import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import { + FetchHttpClient, + Headers, + HttpClient, + HttpClientRequest, + HttpClientResponse, +} from "effect/http"; import * as ServerSettings from "../serverSettings.ts"; import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; import { applySpeechAliases } from "./customWords.ts"; @@ -31,9 +37,12 @@ const endpoints = { openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, + elevenlabs: { url: "https://api.elevenlabs.io/v1/speech-to-text", model: "scribe_v3" }, }; const OPENROUTER_CATALOG_URL = "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true"; +// ElevenLabs authenticates with `xi-api-key`, which Effect's span headers don't redact by default. +const REDACTED_HEADERS = [...Headers.CurrentRedactedNames.defaultValue(), "xi-api-key"]; const isApiKey = Schema.is(CloudSpeechApiKey); const isModelId = Schema.is(CloudSpeechModelId); const sameCredential = Schema.toEquivalence(SettingsSecretValue); @@ -92,6 +101,22 @@ function audioBody( return HttpClientRequest.bodyFormData(form); } +function elevenLabsBody(audio: ArrayBuffer, language: string, options: SpeechTranscriptionOptions) { + const form = new FormData(); + form.set("model_id", endpoints.elevenlabs.model); + form.set("tag_audio_events", "false"); + form.set("timestamps_granularity", "none"); + if (language !== "auto") form.set("language_code", language); + form.set("no_verbatim", String(options.speechRemoveFillerWords && !options.speechCorrectionWord)); + for (const { term } of options.speechCustomWords) { + // Scribe rejects the whole request for a keyterm over five words or with these characters. + if (/[<>{}[\]\\]/.test(term) || term.trim().split(/\s+/).length > 5) continue; + form.append("keyterms", term); + } + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); + return HttpClientRequest.bodyFormData(form); +} + function openRouterBody(model: string, audio: ArrayBuffer, language: string) { return HttpClientRequest.bodyJsonUnsafe({ model, @@ -130,10 +155,11 @@ export const layer = Layer.effect( Effect.map((value) => value.cloudSpeechCredentials), ); const configured = credentials.pipe( - Effect.map(({ openai, xai, openrouter }) => ({ + Effect.map(({ openai, xai, openrouter, elevenlabs }) => ({ openai: Boolean(openai), xai: Boolean(xai), openrouter: Boolean(openrouter), + elevenlabs: Boolean(elevenlabs), })), ); const execute = (request: HttpClientRequest.HttpClientRequest) => @@ -142,6 +168,7 @@ export const layer = Layer.effect( .pipe( Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), Effect.provideService(HttpClient.TracerPropagationEnabled, false), + Effect.provideService(Headers.CurrentRedactedNames, REDACTED_HEADERS), ); const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () { const unavailable = () => @@ -209,10 +236,14 @@ export const layer = Layer.effect( if (!isApiKey(key)) return yield* unreadable(); const audio = wave(samples); const request = HttpClientRequest.post(endpoints[provider].url).pipe( - HttpClientRequest.bearerToken(Redacted.make(key)), + provider === "elevenlabs" + ? HttpClientRequest.setHeader("xi-api-key", key) + : HttpClientRequest.bearerToken(Redacted.make(key)), provider === "openrouter" ? openRouterBody(model!, audio, language) - : audioBody(provider, audio, language, options), + : provider === "elevenlabs" + ? elevenLabsBody(audio, language, options) + : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); // A catalog fetch or vault lookup may have outlived the credential it started with. diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 95dcee4cffb45..24194e6217ceb 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -218,7 +218,11 @@ export function CloudVoiceConfiguration({ const destination = (): CloudDestination => routed && isModelId(model) ? { kind: "cloud", provider: "openrouter", environmentId: id, model } - : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + : { + kind: "cloud", + provider: provider as Exclude, + environmentId: id, + }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed ? modelChosen && keyReady && (!active || modelChanged || keyChanged) @@ -343,11 +347,10 @@ export function CloudVoiceConfiguration({ - ) : routed && - AsyncResult.isSuccess(presence) && - presence.value && - saved === undefined ? ( -

Update {environment.label} to use OpenRouter dictation.

+ ) : AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( +

+ Update {environment.label} to use {name} dictation. +

) : saved === undefined ? (

Checking for an API key…

) : routed ? ( diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index cfddfce72d154..d0194daed8839 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -10,11 +10,12 @@ import { updateEnvironmentSpeechModelUnloadTimeout, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; -import { openRouterIcon } from "@t3tools/client-runtime/provider-clients"; +import { elevenLabsIcon, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; import { AuthFilesystemReadScope, AuthSettingsWriteScope, CLOUD_SPEECH_LABELS, + CLOUD_SPEECH_PROVIDER_NAMES, CloudSpeechProvider, DEFAULT_SPEECH_POST_PROCESSING_PROMPT, type EnvironmentId, @@ -142,6 +143,8 @@ const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; + if (provider === "elevenlabs") + return ; return ( env.environmentId === pendingCloud?.environmentId, ); + const ownsSilence = + destination.cloudProvider === "openai" || destination.cloudProvider === "elevenlabs" + ? destination.cloudProvider + : null; // OpenRouter users without settings access still manage their own model choice. const manageLabel = destination.cloudProvider === "openrouter" ? "Manage OpenRouter" : "Manage key"; @@ -1538,14 +1545,14 @@ export function VoiceSettingsPanel() { {...searchableSetting("speech-silence-filtering")} title="Silence filtering" description={ - destination.cloudProvider === "openai" - ? "OpenAI handles silence itself." + ownsSilence + ? `${CLOUD_SPEECH_PROVIDER_NAMES[ownsSilence]} handles silence itself.` : "Skip silence and background noise before transcribing." } control={ void updateClientSettings({ speechSilenceFiltering: checked }) diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index b087c6e38a656..fe5c6a0c22fa3 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -533,7 +533,7 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Transcribe with", to: "/settings/voice", searchTerms: [ - "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter", + "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter ElevenLabs Scribe", ], }, { diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 95ea15d707d00..cdf1defeaabde 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -24,8 +24,8 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. -For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment -you want to use. If it needs a key, enter one and choose **Save and enable**. +For cloud dictation, choose **OpenAI**, **Grok Voice Transcribe**, or **ElevenLabs Scribe** +via the environment you want to use. If it needs a key, enter one and choose **Save and enable**. You can also choose a [secret provider](secrets.md) instead of **Value** and enter a reference. Unlock the vault on that environment before dictating. Anyone with dictation access to the environment can use the key, but only this device opts in. diff --git a/packages/client-runtime/src/providerClients.ts b/packages/client-runtime/src/providerClients.ts index 3759cecbc7c7d..9897a112f2861 100644 --- a/packages/client-runtime/src/providerClients.ts +++ b/packages/client-runtime/src/providerClients.ts @@ -27,6 +27,13 @@ export const openRouterIcon: ProviderIcon = { ], }; +// ElevenLabs' two-bar logomark. +export const elevenLabsIcon: ProviderIcon = { + viewBox: "0 0 24 24", + fill: { light: "#000000", dark: "#FFFFFF" }, + paths: [{ d: "M7 3h3.5v18H7zM13.5 3H17v18h-3.5z" }], +}; + export const providerClients = makeProviderClientRegistry([ { driverKind: ProviderDriverKind.make("codex"), diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 00eeb1797870a..da8125a6bacd7 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -385,7 +385,7 @@ export const VoiceTranscriptionDestination = Schema.Union([ Schema.Struct({ kind: Schema.Literal("cloud"), environmentId: EnvironmentId, - provider: Schema.Literals(["openai", "xai"]), + provider: Schema.Literals(["openai", "xai", "elevenlabs"]), }), Schema.Struct({ kind: Schema.Literal("cloud"), @@ -1347,6 +1347,7 @@ export const CloudSpeechCredentials = Schema.Struct({ openai: CloudSpeechCredential, xai: CloudSpeechCredential, openrouter: CloudSpeechCredential, + elevenlabs: CloudSpeechCredential, }); export const ServerSettings = Schema.Struct({ @@ -1377,7 +1378,9 @@ export const ServerSettings = Schema.Struct({ enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), cloudSpeechCredentials: CloudSpeechCredentials.pipe( - Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })), + Schema.withDecodingDefault( + Effect.succeed({ openai: "", xai: "", openrouter: "", elevenlabs: "" }), + ), ), speechModelId: Schema.String.pipe( Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")), @@ -1831,6 +1834,7 @@ export const ServerSettingsPatch = Schema.Struct({ openai: Schema.optionalKey(SettingsSecretValue), xai: Schema.optionalKey(SettingsSecretValue), openrouter: Schema.optionalKey(SettingsSecretValue), + elevenlabs: Schema.optionalKey(SettingsSecretValue), }), ), speechModelId: Schema.optionalKey(Schema.String), diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts index ff3362f67fc0c..ae14b5c266c4c 100644 --- a/packages/contracts/src/speech.ts +++ b/packages/contracts/src/speech.ts @@ -8,23 +8,26 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check( Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i), ); -export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]); +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter", "elevenlabs"]); export type CloudSpeechProvider = typeof CloudSpeechProvider.Type; export const CLOUD_SPEECH_LABELS = { openai: "OpenAI", xai: "Grok Voice Transcribe", openrouter: "OpenRouter", + elevenlabs: "ElevenLabs Scribe", } as const; export const CLOUD_SPEECH_PROVIDER_NAMES = { openai: "OpenAI", xai: "xAI", openrouter: "OpenRouter", + elevenlabs: "ElevenLabs", } as const; -/** Servers that predate OpenRouter report only the first two providers. */ +/** Older servers omit providers added after OpenAI and xAI. */ export const CloudSpeechKeys = Schema.Struct({ openai: Schema.Boolean, xai: Schema.Boolean, openrouter: Schema.optionalKey(Schema.Boolean), + elevenlabs: Schema.optionalKey(Schema.Boolean), }); export type CloudSpeechKeys = typeof CloudSpeechKeys.Type;