();
+ upstream.mockImplementationOnce(() => {
+ uploadStarted.resolve();
+ return rejection.promise;
+ });
+ const uploads = upstream.mock.calls.length;
+ const job = yield* transcribe("en").pipe(Effect.forkChild);
+ yield* Effect.promise(() => uploadStarted.promise);
+ yield* service.setKey("deepgram", null);
+ rejection.resolve(keytermLimit());
+ expect(yield* Effect.flip(Fiber.join(job))).toMatchObject({ reason: "missing-key" });
+ expect(upstream).toHaveBeenCalledTimes(uploads + 1);
+ }),
+);
+
it.effect("removing a key while a catalog read is pending prevents the upload", () =>
Effect.gen(function* () {
const service = yield* CloudSpeech.CloudSpeech.pipe(
diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts
index 3b7eab6b5242b..8437f7fbd7d60 100644
--- a/apps/server/src/speech/CloudSpeech.ts
+++ b/apps/server/src/speech/CloudSpeech.ts
@@ -31,12 +31,22 @@ const endpoints = {
openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
+ deepgram: { url: "https://api.deepgram.com/v1/listen", model: "nova-3" },
};
const OPENROUTER_CATALOG_URL =
"https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
const isApiKey = Schema.is(CloudSpeechApiKey);
const isModelId = Schema.is(CloudSpeechModelId);
const sameCredential = Schema.toEquivalence(SettingsSecretValue);
+const DeepgramResult = Schema.Struct({
+ results: Schema.Struct({
+ channels: Schema.NonEmptyArray(
+ Schema.Struct({
+ alternatives: Schema.NonEmptyArray(Schema.Struct({ transcript: Schema.String })),
+ }),
+ ),
+ }),
+});
const OpenRouterCatalog = Schema.Struct({
data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })),
});
@@ -101,6 +111,20 @@ function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
});
}
+function deepgramUrl(language: string, options: SpeechTranscriptionOptions, keyterms: boolean) {
+ const url = new URL(endpoints.deepgram.url);
+ url.searchParams.set("model", endpoints.deepgram.model);
+ url.searchParams.set("smart_format", "true");
+ url.searchParams.set("redact_usage", "true");
+ // `multi` transcribes speech that switches languages, such as English terms in other speech.
+ url.searchParams.set("language", language === "auto" ? "multi" : language);
+ if (keyterms)
+ for (const { term } of options.speechCustomWords) url.searchParams.append("keyterm", term);
+ if (!options.speechRemoveFillerWords || options.speechCorrectionWord)
+ url.searchParams.set("filler_words", "true");
+ return url;
+}
+
export class CloudSpeech extends Context.Service<
CloudSpeech,
{
@@ -130,10 +154,11 @@ export const layer = Layer.effect(
Effect.map((value) => value.cloudSpeechCredentials),
);
const configured = credentials.pipe(
- Effect.map(({ openai, xai, openrouter }) => ({
+ Effect.map(({ openai, xai, openrouter, deepgram }) => ({
openai: Boolean(openai),
xai: Boolean(xai),
openrouter: Boolean(openrouter),
+ deepgram: Boolean(deepgram),
})),
);
const execute = (request: HttpClientRequest.HttpClientRequest) =>
@@ -208,21 +233,42 @@ export const layer = Layer.effect(
);
if (!isApiKey(key)) return yield* unreadable();
const audio = wave(samples);
- const request = HttpClientRequest.post(endpoints[provider].url).pipe(
- HttpClientRequest.bearerToken(Redacted.make(key)),
- provider === "openrouter"
- ? openRouterBody(model!, audio, language)
- : audioBody(provider, audio, language, options),
- );
+ const deepgramRequest = (keyterms: boolean) =>
+ HttpClientRequest.post(deepgramUrl(language, options, keyterms)).pipe(
+ HttpClientRequest.setHeader("authorization", `Token ${key}`),
+ HttpClientRequest.bodyUint8Array(new Uint8Array(audio), "audio/wav"),
+ );
+ const request =
+ provider === "deepgram"
+ ? deepgramRequest(true)
+ : HttpClientRequest.post(endpoints[provider].url).pipe(
+ HttpClientRequest.bearerToken(Redacted.make(key)),
+ provider === "openrouter"
+ ? openRouterBody(model!, audio, language)
+ : audioBody(provider, audio, language, options),
+ );
const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
- // A catalog fetch or vault lookup may have outlived the credential it started with.
- const current = (yield* credentials)[provider];
- if (!current || !sameCredential(source, current))
- return yield* new CloudSpeechError({
- provider,
- reason: current ? "key-changed" : "missing-key",
- });
- const response = yield* execute(request).pipe(Effect.mapError(unavailable));
+ // A catalog fetch, vault lookup, or earlier upload may have outlived the credential.
+ const ensureCurrentKey = Effect.gen(function* () {
+ const current = (yield* credentials)[provider];
+ if (!current || !sameCredential(source, current))
+ return yield* new CloudSpeechError({
+ provider,
+ reason: current ? "key-changed" : "missing-key",
+ });
+ });
+ yield* ensureCurrentKey;
+ let response = yield* execute(request).pipe(Effect.mapError(unavailable));
+ // Deepgram caps keyterms in tokens clients cannot count; the dictionary still fixes the text below.
+ if (
+ provider === "deepgram" &&
+ response.status === 400 &&
+ options.speechCustomWords.length > 0 &&
+ (yield* response.text.pipe(Effect.orElseSucceed(() => ""))).includes("Keyterm limit")
+ ) {
+ yield* ensureCurrentKey;
+ response = yield* execute(deepgramRequest(false)).pipe(Effect.mapError(unavailable));
+ }
if (response.status < 200 || response.status >= 300)
return yield* new CloudSpeechError({
provider,
@@ -235,9 +281,14 @@ export const layer = Layer.effect(
? "model-unavailable"
: "unavailable",
});
- const { text } = yield* HttpClientResponse.schemaBodyJson(
- EnvironmentSpeechTranscriptionResult,
- )(response).pipe(Effect.mapError(unavailable));
+ const text =
+ provider === "deepgram"
+ ? (yield* HttpClientResponse.schemaBodyJson(DeepgramResult)(response).pipe(
+ Effect.mapError(unavailable),
+ )).results.channels[0].alternatives[0].transcript
+ : (yield* HttpClientResponse.schemaBodyJson(EnvironmentSpeechTranscriptionResult)(
+ response,
+ ).pipe(Effect.mapError(unavailable))).text;
const corrected = applySpeechAliases(text, options.speechCustomWords);
return (
options.speechRemoveFillerWords
diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx
index b5eddbcc10df8..583d886ac87a1 100644
--- a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx
+++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx
@@ -101,8 +101,14 @@ it("cancelling a provider's setup keeps the current destination and writes nothi
const destination = screen.getByRole("combobox", { name: "Transcribe with" });
await expect.element(destination).toHaveTextContent("Laptop");
await destination.click();
- await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
+ await page.getByRole("option", { name: "Deepgram via Laptop" }).click();
const dialog = page.getByRole("dialog");
+ await expect
+ .element(dialog.getByText("Update Laptop to use Deepgram dictation."))
+ .toBeVisible();
+ await dialog.getByRole("button", { name: "Cancel" }).click();
+ await destination.click();
+ await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
await dialog.getByRole("button", { name: "Cancel" }).click();
await expect.element(dialog).not.toBeInTheDocument();
await expect.element(destination).toHaveTextContent("Laptop");
diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx
index 95dcee4cffb45..0155a161be5cd 100644
--- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx
+++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx
@@ -218,7 +218,7 @@ export function CloudVoiceConfiguration({
const destination = (): CloudDestination =>
routed && isModelId(model)
? { kind: "cloud", provider: "openrouter", environmentId: id, model }
- : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id };
+ : { kind: "cloud", provider: provider as "openai" | "xai" | "deepgram", environmentId: id };
const enable = () => chooseAudioDestination(destination());
const canSubmit = routed
? modelChosen && keyReady && (!active || modelChanged || keyChanged)
@@ -343,11 +343,10 @@ export function CloudVoiceConfiguration({
- ) : routed &&
- AsyncResult.isSuccess(presence) &&
- presence.value &&
- saved === undefined ? (
- Update {environment.label} to use OpenRouter dictation.
+ ) : AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? (
+
+ Update {environment.label} to use {name} dictation.
+
) : saved === undefined ? (
Checking for an API key…
) : routed ? (
diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
index 22ae59e55d31e..f65d5a6db3e60 100644
--- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx
+++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx
@@ -140,6 +140,7 @@ import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"
const OFF = "off";
function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) {
+ if (provider === "deepgram") return ;
if (provider === "openrouter")
return ;
return (
diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts
index 85e4878be73dd..29cc1ee5b192e 100644
--- a/apps/web/src/components/settings/settingsSearch.ts
+++ b/apps/web/src/components/settings/settingsSearch.ts
@@ -544,7 +544,7 @@ export const SETTINGS_SEARCH_ITEMS = [
title: "Transcribe with",
to: "/settings/voice",
searchTerms: [
- "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter",
+ "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter Deepgram",
],
},
{
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
index 592e110c896cc..8a013e4955bae 100644
--- a/docs/user/voice-input.md
+++ b/docs/user/voice-input.md
@@ -24,8 +24,8 @@ Set it back to **Off** at any time; a recording in progress is discarded.
If the chosen environment goes offline, voice input waits for it. T3 Code never
substitutes another environment.
-For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment
-you want to use. If it needs a key, enter one and choose **Save and enable**.
+For cloud dictation, choose **OpenAI**, **Grok Voice Transcribe**, or **Deepgram** via the
+environment you want to use. If it needs a key, enter one and choose **Save and enable**.
You can also choose a [secret provider](secrets.md) instead of **Value** and enter a
reference. Unlock the vault on that environment before dictating. Anyone with
dictation access to the environment can use the key, but only this device opts in.
diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts
index 49470541d90be..ac2f38f9a0150 100644
--- a/packages/contracts/src/settings.ts
+++ b/packages/contracts/src/settings.ts
@@ -385,7 +385,7 @@ export const VoiceTranscriptionDestination = Schema.Union([
Schema.Struct({
kind: Schema.Literal("cloud"),
environmentId: EnvironmentId,
- provider: Schema.Literals(["openai", "xai"]),
+ provider: Schema.Literals(["openai", "xai", "deepgram"]),
}),
Schema.Struct({
kind: Schema.Literal("cloud"),
@@ -1348,6 +1348,7 @@ export const CloudSpeechCredentials = Schema.Struct({
openai: CloudSpeechCredential,
xai: CloudSpeechCredential,
openrouter: CloudSpeechCredential,
+ deepgram: CloudSpeechCredential,
});
export const ServerSettings = Schema.Struct({
@@ -1380,7 +1381,9 @@ export const ServerSettings = Schema.Struct({
enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
cloudSpeechCredentials: CloudSpeechCredentials.pipe(
- Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })),
+ Schema.withDecodingDefault(
+ Effect.succeed({ openai: "", xai: "", openrouter: "", deepgram: "" }),
+ ),
),
speechModelId: Schema.String.pipe(
Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")),
@@ -1835,6 +1838,7 @@ export const ServerSettingsPatch = Schema.Struct({
openai: Schema.optionalKey(SettingsSecretValue),
xai: Schema.optionalKey(SettingsSecretValue),
openrouter: Schema.optionalKey(SettingsSecretValue),
+ deepgram: Schema.optionalKey(SettingsSecretValue),
}),
),
speechModelId: Schema.optionalKey(Schema.String),
diff --git a/packages/contracts/src/speech.ts b/packages/contracts/src/speech.ts
index ff3362f67fc0c..9b01aa67f898f 100644
--- a/packages/contracts/src/speech.ts
+++ b/packages/contracts/src/speech.ts
@@ -8,23 +8,26 @@ export const CloudSpeechApiKey = TrimmedNonEmptyString.check(
Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i),
);
-export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]);
+export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter", "deepgram"]);
export type CloudSpeechProvider = typeof CloudSpeechProvider.Type;
export const CLOUD_SPEECH_LABELS = {
openai: "OpenAI",
xai: "Grok Voice Transcribe",
openrouter: "OpenRouter",
+ deepgram: "Deepgram",
} as const;
export const CLOUD_SPEECH_PROVIDER_NAMES = {
openai: "OpenAI",
xai: "xAI",
openrouter: "OpenRouter",
+ deepgram: "Deepgram",
} as const;
-/** Servers that predate OpenRouter report only the first two providers. */
+/** Older servers omit providers added after OpenAI and xAI. */
export const CloudSpeechKeys = Schema.Struct({
openai: Schema.Boolean,
xai: Schema.Boolean,
openrouter: Schema.optionalKey(Schema.Boolean),
+ deepgram: Schema.optionalKey(Schema.Boolean),
});
export type CloudSpeechKeys = typeof CloudSpeechKeys.Type;