From dadf2d413d8509952c7963ffa9a8e119bd213194 Mon Sep 17 00:00:00 2001 From: azure-swing Date: Sun, 8 Nov 2026 22:24:28 +0800 Subject: [PATCH 1/4] feat(speech): support direct Gemini API cloud transcription --- .../features/settings/CloudVoiceSection.tsx | 8 ++- apps/server/src/serverSettings.test.ts | 10 ++- .../sourceControlHostSecrets.test.ts | 7 +- apps/server/src/speech/CloudSpeech.ts | 69 ++++++++++++++++--- .../settings/CloudVoiceConfiguration.tsx | 2 +- .../settings/VoiceSettingsPanel.tsx | 4 +- .../client-runtime/src/providerClients.ts | 10 +++ packages/contracts/src/settings.ts | 6 +- packages/contracts/src/speech.ts | 5 +- 9 files changed, 104 insertions(+), 17 deletions(-) diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 684f8dbc83769..671f5520c87c2 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -279,7 +279,13 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () => leading={ diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 6c470d1f83f05..45721de7b2df1 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1691,6 +1691,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(legacy.cloudSpeechCredentials, { openai: SECRET_REDACTED, xai: SECRET_REDACTED, + gemini: "", openrouter: "", }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); @@ -1718,6 +1719,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(echoed.cloudSpeechCredentials, { openai: usageReference, xai: "literal-fixture", + gemini: "", openrouter: "", }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); @@ -1804,7 +1806,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); - assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); + assert.deepEqual(removed.cloudSpeechCredentials, { + openai: "", + xai: work, + gemini: "", + openrouter: "", + }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1816,6 +1823,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [ "cloudSpeechCredentials.openai", "cloudSpeechCredentials.xai", + "cloudSpeechCredentials.gemini", "cloudSpeechCredentials.openrouter", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 58c84365dfcef..3ef6dacf9dfa8 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -176,7 +176,12 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, - cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, + cloudSpeechCredentials: { + openai: source("voice#key"), + xai: "", + gemini: "", + openrouter: "", + }, usageLimitSources, easDeviceHosts, simlockDeviceHosts, diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 3b7eab6b5242b..0e9bc4ec93ab5 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -30,6 +30,10 @@ import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts"; const endpoints = { openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, + gemini: { + url: "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-transcribe:generateContent", + model: "gemini-3.5-transcribe", + }, openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, }; const OPENROUTER_CATALOG_URL = @@ -40,6 +44,20 @@ const sameCredential = Schema.toEquivalence(SettingsSecretValue); const OpenRouterCatalog = Schema.Struct({ data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })), }); +const GeminiTranscript = Schema.Struct({ + candidates: Schema.Array( + Schema.Struct({ + content: Schema.Struct({ + parts: Schema.Array( + Schema.Struct({ + text: Schema.optionalKey(Schema.String), + audioTranscription: Schema.optionalKey(Schema.Struct({ text: Schema.String })), + }), + ), + }), + }), + ), +}); function wave(samples: Float32Array) { const buffer = new ArrayBuffer(44 + samples.length * 2); @@ -101,6 +119,26 @@ function openRouterBody(model: string, audio: ArrayBuffer, language: string) { }); } +function geminiBody(audio: ArrayBuffer, language: string, options: SpeechTranscriptionOptions) { + return HttpClientRequest.bodyJsonUnsafe({ + contents: [ + { + parts: [ + { inlineData: { mimeType: "audio/wav", data: Buffer.from(audio).toString("base64") } }, + ], + }, + ], + generationConfig: { + audioTranscriptionConfig: { + languageCodes: language === "auto" ? [] : [language], + customVocabulary: options.speechCustomWords.map(({ term }) => term), + mode: + options.speechRemoveFillerWords && !options.speechCorrectionWord ? "smart" : "verbatim", + }, + }, + }); +} + export class CloudSpeech extends Context.Service< CloudSpeech, { @@ -130,9 +168,10 @@ export const layer = Layer.effect( Effect.map((value) => value.cloudSpeechCredentials), ); const configured = credentials.pipe( - Effect.map(({ openai, xai, openrouter }) => ({ + Effect.map(({ openai, xai, gemini, openrouter }) => ({ openai: Boolean(openai), xai: Boolean(xai), + gemini: Boolean(gemini), openrouter: Boolean(openrouter), })), ); @@ -209,10 +248,14 @@ export const layer = Layer.effect( if (!isApiKey(key)) return yield* unreadable(); const audio = wave(samples); const request = HttpClientRequest.post(endpoints[provider].url).pipe( - HttpClientRequest.bearerToken(Redacted.make(key)), - provider === "openrouter" - ? openRouterBody(model!, audio, language) - : audioBody(provider, audio, language, options), + provider === "gemini" + ? HttpClientRequest.setHeader("x-goog-api-key", key) + : HttpClientRequest.bearerToken(Redacted.make(key)), + provider === "gemini" + ? geminiBody(audio, language, options) + : provider === "openrouter" + ? openRouterBody(model!, audio, language) + : audioBody(provider, audio, language, options), ); const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); // A catalog fetch or vault lookup may have outlived the credential it started with. @@ -229,15 +272,23 @@ export const layer = Layer.effect( reason: response.status === 429 ? "rate-limit" - : [401, 402, 403].includes(response.status) + : [401, 402, 403].includes(response.status) || + (response.status === 400 && provider === "gemini") ? "rejected-key" : response.status === 404 && provider === "openrouter" ? "model-unavailable" : "unavailable", }); - const { text } = yield* HttpClientResponse.schemaBodyJson( - EnvironmentSpeechTranscriptionResult, - )(response).pipe(Effect.mapError(unavailable)); + const text = + provider === "gemini" + ? ((yield* HttpClientResponse.schemaBodyJson(GeminiTranscript)(response).pipe( + Effect.mapError(unavailable), + )).candidates[0]?.content.parts + .map((part) => part.audioTranscription?.text ?? part.text ?? "") + .join("") ?? "") + : (yield* HttpClientResponse.schemaBodyJson(EnvironmentSpeechTranscriptionResult)( + response, + ).pipe(Effect.mapError(unavailable))).text; const corrected = applySpeechAliases(text, options.speechCustomWords); return ( options.speechRemoveFillerWords diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 95dcee4cffb45..72787936ebfca 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -218,7 +218,7 @@ export function CloudVoiceConfiguration({ const destination = (): CloudDestination => routed && isModelId(model) ? { kind: "cloud", provider: "openrouter", environmentId: id, model } - : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + : { kind: "cloud", provider: provider as "openai" | "xai" | "gemini", environmentId: id }; const enable = () => chooseAudioDestination(destination()); const canSubmit = routed ? modelChosen && keyReady && (!active || modelChanged || keyChanged) diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 4a3df08f40f14..55971d988968d 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -9,7 +9,7 @@ import { updateEnvironmentSpeechModelUnloadTimeout, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; -import { openRouterIcon } from "@t3tools/client-runtime/provider-clients"; +import { geminiIcon, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; import { AuthSettingsWriteScope, CLOUD_SPEECH_LABELS, @@ -125,6 +125,8 @@ const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { if (provider === "openrouter") return ; + if (provider === "gemini") + return ; return ( Date: Sun, 8 Nov 2026 22:35:13 +0800 Subject: [PATCH 2/4] fix(speech): inspect Gemini error body for API_KEY_INVALID on 400 --- apps/server/src/speech/CloudSpeech.test.ts | 72 +++++++++++++++++++++- apps/server/src/speech/CloudSpeech.ts | 14 ++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts index 112c7a88d9b3b..ca65d18477bc5 100644 --- a/apps/server/src/speech/CloudSpeech.test.ts +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -49,7 +49,12 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); yield* service.setKey(provider, Redacted.make("chosen")); - expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + expect(yield* service.configured).toEqual({ + openai: true, + xai: true, + gemini: false, + openrouter: false, + }); yield* Effect.flip(transcribe(new Uint8Array(3))); expect(upstream).not.toHaveBeenCalled(); expect(yield* transcribe()).toBe("hello"); @@ -198,3 +203,68 @@ it.effect("removing a key while a catalog read is pending prevents the upload", expect(exit._tag).toBe("Failure"); }), ); + +it.effect("sends audio to Gemini with API key and differentiates 400 errors", () => + Effect.gen(function* () { + const upstream = vi.fn(async () => + Response.json({ + candidates: [ + { + content: { + parts: [{ text: "Gemini transcribed text" }], + }, + }, + ], + }), + ); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const transcribe = () => + service + .transcribe("gemini", new Uint8Array(new Float32Array([0, 0.5]).buffer), { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechRemoveFillerWords: true, + speechCustomWords: [{ term: "T3 Code", aliases: [] }], + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + + yield* service.setKey("gemini", Redacted.make("gemini-test-key")); + expect(yield* transcribe()).toBe("Gemini transcribed text"); + + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-transcribe:generateContent", + ); + expect(new Headers(init?.headers).get("x-goog-api-key")).toBe("gemini-test-key"); + + // 400 with API_KEY_INVALID reason should report rejected-key + upstream.mockResolvedValueOnce( + new Response(JSON.stringify({ error: { details: [{ reason: "API_KEY_INVALID" }] } }), { + status: 400, + }), + ); + expect(yield* Effect.flip(transcribe())).toMatchObject({ + provider: "gemini", + reason: "rejected-key", + }); + + // 400 without API_KEY_INVALID (e.g. invalid argument) should report unavailable + upstream.mockResolvedValueOnce( + new Response(JSON.stringify({ error: { message: "Invalid payload" } }), { + status: 400, + }), + ); + expect(yield* Effect.flip(transcribe())).toMatchObject({ + provider: "gemini", + reason: "unavailable", + }); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts index 0e9bc4ec93ab5..516aec2c1c1b8 100644 --- a/apps/server/src/speech/CloudSpeech.ts +++ b/apps/server/src/speech/CloudSpeech.ts @@ -266,19 +266,27 @@ export const layer = Layer.effect( reason: current ? "key-changed" : "missing-key", }); const response = yield* execute(request).pipe(Effect.mapError(unavailable)); - if (response.status < 200 || response.status >= 300) + if (response.status < 200 || response.status >= 300) { + const rejected = + [401, 402, 403].includes(response.status) || + (response.status === 400 && + provider === "gemini" && + (yield* response.text.pipe( + Effect.map((body) => body.includes("API_KEY_INVALID")), + Effect.orElseSucceed(() => false), + ))); return yield* new CloudSpeechError({ provider, reason: response.status === 429 ? "rate-limit" - : [401, 402, 403].includes(response.status) || - (response.status === 400 && provider === "gemini") + : rejected ? "rejected-key" : response.status === 404 && provider === "openrouter" ? "model-unavailable" : "unavailable", }); + } const text = provider === "gemini" ? ((yield* HttpClientResponse.schemaBodyJson(GeminiTranscript)(response).pipe( From 9ec4791cad6eba42a7ffa79efb5e40d2fe46ee8d Mon Sep 17 00:00:00 2001 From: azure-swing Date: Sun, 8 Nov 2026 22:50:16 +0800 Subject: [PATCH 3/4] fix(mobile): use geminiIcon in mobile ProviderIcon for cross-platform parity --- apps/mobile/src/components/ProviderIcon.tsx | 12 ++++++++++-- .../src/features/settings/CloudVoiceSection.tsx | 8 +------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index c5d9acacba238..b1d4874c915d4 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -11,7 +11,11 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts"; import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider"; import { AppText as Text } from "./ui/AppText"; import { cn } from "../lib/cn"; -import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; +import { + geminiIcon, + getProviderClient, + openRouterIcon, +} from "@t3tools/client-runtime/provider-clients"; type ProviderIconProps = { readonly provider: string | null | undefined; @@ -100,7 +104,11 @@ export function ProviderIcon(props: ProviderIconProps) { } const packageIcon = - props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon; + props.provider === "openrouter" + ? openRouterIcon + : props.provider === "gemini" + ? geminiIcon + : getProviderClient(props.provider)?.icon; if (packageIcon) { const themeFill = (fill: { readonly light: string; readonly dark: string }) => isDarkMode ? fill.dark : fill.light; diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx index 671f5520c87c2..684f8dbc83769 100644 --- a/apps/mobile/src/features/settings/CloudVoiceSection.tsx +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -279,13 +279,7 @@ function CloudChoices({ environment, active, onClose }: Props & { onClose: () => leading={ From 721b498fb20a54f9a41b4e6182a509ac63b8f00f Mon Sep 17 00:00:00 2001 From: azure-swing Date: Sun, 8 Nov 2026 23:02:05 +0800 Subject: [PATCH 4/4] fix(web): show environment update prompt when speech provider is missing on host --- .../src/components/settings/CloudVoiceConfiguration.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx index 72787936ebfca..0ba315bb18007 100644 --- a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -343,11 +343,10 @@ export function CloudVoiceConfiguration({ - ) : routed && - AsyncResult.isSuccess(presence) && - presence.value && - saved === undefined ? ( -

Update {environment.label} to use OpenRouter dictation.

+ ) : AsyncResult.isSuccess(presence) && presence.value && saved === undefined ? ( +

+ Update {environment.label} to use {name} dictation. +

) : saved === undefined ? (

Checking for an API key…

) : routed ? (