diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 784668082bb6f..97eb9b0cadcab 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -458,6 +458,9 @@ const config: ExpoConfig = { ], extra: { appVariant: APP_VARIANT, + secretsFeatureEnabled: ["1", "true"].includes( + process.env.T3CODE_FEATURE_SECRETS?.trim().toLowerCase() ?? "", + ), iosPersonalTeamBuild: isIosPersonalTeamBuild, relay: { url: repoEnv.T3CODE_RELAY_URL ?? null, diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index cd4574b6f84e5..c5d9acacba238 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -11,7 +11,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts"; import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider"; import { AppText as Text } from "./ui/AppText"; import { cn } from "../lib/cn"; -import { getProviderClient } from "@t3tools/client-runtime/provider-clients"; +import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients"; type ProviderIconProps = { readonly provider: string | null | undefined; @@ -99,7 +99,8 @@ export function ProviderIcon(props: ProviderIconProps) { ); } - const packageIcon = getProviderClient(props.provider)?.icon; + const packageIcon = + props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon; if (packageIcon) { const themeFill = (fill: { readonly light: string; readonly dark: string }) => isDarkMode ? fill.dark : fill.light; diff --git a/apps/mobile/src/features/settings/CloudVoiceSection.tsx b/apps/mobile/src/features/settings/CloudVoiceSection.tsx new file mode 100644 index 0000000000000..684f8dbc83769 --- /dev/null +++ b/apps/mobile/src/features/settings/CloudVoiceSection.tsx @@ -0,0 +1,473 @@ +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { cloudSpeechDisclosure } from "@t3tools/shared/speech"; +import { setCloudSpeechKey } from "@t3tools/client-runtime/voice-input"; +import { + configuredSecretSources, + secretRequestReference, +} from "@t3tools/client-runtime/secret-request"; +import { + AuthSettingsWriteScope, + CLOUD_SPEECH_LABELS, + CLOUD_SPEECH_PROVIDER_NAMES, + CloudSpeechProvider, + CloudSpeechApiKey, + CloudSpeechModelId, + isSecretReference, + type CloudSpeechModel, + type SecretProviderId, + type SettingsSecretValue, + type VoiceTranscriptionDestination, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import Constants from "expo-constants"; +import { AsyncResult } from "effect/reactivity"; +import { useState } from "react"; +import { ScrollView, TextInput, View } from "react-native"; +import { Sheet } from "../../components/ui/Sheet"; +import * as Dialog from "../../components/ui/Dialog"; +import { ProviderIcon } from "../../components/ProviderIcon"; +import { AppText as Text } from "../../components/ui/AppText"; +import { ListActionRow } from "../../components/ui/ListActionRow"; +import { ListChoiceRow } from "../../components/ui/ListChoiceRow"; +import { Chip } from "../../components/ui/Chip"; +import type { EnvironmentPresentation } from "../../state/environments"; +import { useEnvironmentScope } from "../../state/session"; +import { + chooseVoiceDestination, + pendingCloudVoiceChoice, + cloudVoiceConnectionAtom, + cloudVoiceKeysAtom, + cloudVoiceModelsAtom, + runSpeech, +} from "../voice-input/voiceDestination"; +import { SettingsSection } from "./components/SettingsSection"; + +type CloudDestination = Extract; +type Props = { environment: EnvironmentPresentation; active: CloudDestination | null }; +const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +const MODEL_ROWS = 6; + +export function CloudVoiceSection({ environment, active }: Props) { + const [open, setOpen] = useState(false); + return ( + + setOpen(true)} + /> + {open && ( + setOpen(false)} /> + )} + + ); +} + +function ModelPicker({ + models, + value, + current, + disabled, + onChange, +}: { + models: ReadonlyArray; + value: string; + /** The grant's model, flagged when the catalog no longer lists it. */ + current: string | undefined; + disabled: boolean; + onChange: (model: CloudSpeechModelId) => void; +}) { + const [query, setQuery] = useState(""); + const [showAll, setShowAll] = useState(false); + const needle = query.trim().toLowerCase(); + const selected = models.find((model) => model.id === value); + const matches = models.filter((model) => + `${model.name} ${model.id}`.toLowerCase().includes(needle), + ); + const shown = showAll ? matches : matches.slice(0, MODEL_ROWS); + const delisted = current !== undefined && !models.some((model) => model.id === current); + return ( + <> + + {delisted ? ( + + OpenRouter no longer lists {current}. Dictation stays off until you choose another model. + + ) : null} + {selected && !shown.some((model) => model.id === selected.id) ? ( + {}} + /> + ) : null} + {shown.map((model, index) => ( + 0} + disabled={disabled} + onPress={() => onChange(model.id)} + /> + ))} + {matches.length === 0 ? ( + No transcription model matches. + ) : matches.length > shown.length ? ( + setShowAll(true)} + /> + ) : null} + + ); +} + +function CloudChoices({ environment, active, onClose }: Props & { onClose: () => void }) { + const id = environment.environmentId; + const connection = useAtomValue(cloudVoiceConnectionAtom(id)); + const keys = useAtomValue(cloudVoiceKeysAtom(id)); + const reload = useAtomRefresh(cloudVoiceKeysAtom(id)); + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); + const [editing, setEditing] = useState<{ + provider: CloudSpeechProvider; + enable: boolean; + } | null>(null); + const catalogAtom = cloudVoiceModelsAtom(editing?.provider === "openrouter" ? id : null); + const models = useAtomValue(catalogAtom); + const reloadModels = useAtomRefresh(catalogAtom); + const [draft, setDraft] = useState(""); + const [model, setModel] = useState(""); + const [sourceId, setSourceId] = useState(null); + const sources = + Constants.expoConfig?.extra?.secretsFeatureEnabled === true && + environment.serverConfig?.secretReferences + ? configuredSecretSources(environment.serverConfig.settings.secretProviders) + : []; + const source = sources.find((candidate) => candidate.id === sourceId); + const credential = + sourceId !== null + ? source && secretRequestReference(source, draft) + : isApiKey(draft.trim()) + ? draft.trim() + : undefined; + const [saving, setSaving] = useState(false); + const activeModel = active?.provider === "openrouter" ? active.model : undefined; + const enable = (commit: () => void) => { + try { + commit(); + } catch { + Dialog.alert( + "Could not enable cloud dictation", + "Check device storage and choose the provider again.", + ); + } + }; + const edit = (provider: CloudSpeechProvider, enable: boolean) => { + const stored = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider]; + const ref = + isSecretReference(stored) && sources.some((candidate) => candidate.id === stored.provider) + ? stored + : null; + setSourceId(ref?.provider ?? null); + setDraft(ref?.reference ?? ""); + setModel(provider === "openrouter" ? (activeModel ?? "") : ""); + setEditing({ provider, enable }); + }; + const reset = () => { + setDraft(""); + setModel(""); + setEditing(null); + }; + const changeKey = async ( + provider: CloudSpeechProvider, + value: SettingsSecretValue | null, + chosen?: CloudSpeechModelId, + ) => { + if (!connection || !canManage || saving) return; + const commit = + value !== null && (editing?.enable || (chosen !== undefined && chosen !== activeModel)) + ? pendingCloudVoiceChoice(id, provider, chosen) + : null; + setSaving(true); + try { + await runSpeech(setCloudSpeechKey(connection, provider, value)); + } catch { + setSaving(false); + Dialog.alert( + "Could not update the API key", + "Check the environment connection and your settings permissions.", + ); + return; + } + setSaving(false); + if (commit) enable(commit); + reload(); + onClose(); + }; + const catalog = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; + const content = + !AsyncResult.isSuccess(keys) || !keys.value || !connection ? ( + + ) : ( + CloudSpeechProvider.literals.map((provider) => { + if (editing && provider !== editing.provider) return null; + const label = CLOUD_SPEECH_LABELS[provider]; + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const routed = provider === "openrouter"; + const disclosure = cloudSpeechDisclosure(provider, environment.label); + const saved = keys.value![provider]; + const isActive = active?.provider === provider; + const isEditing = editing?.provider === provider; + const modelChosen = + isModelId(model) && (catalog?.some((entry) => entry.id === model) ?? false); + const modelChanged = model !== (activeModel ?? ""); + const original = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider]; + const keyChanged = + credential !== undefined && + !( + isSecretReference(credential) && + isSecretReference(original) && + original.provider === credential.provider && + original.reference === credential.reference + ); + const routedReady = + modelChosen && + (draft.trim() ? credential !== undefined : Boolean(saved)) && + (!isActive || modelChanged || keyChanged); + const saveRouted = () => { + if (!routedReady || !isModelId(model)) return; + const grantChanges = !isActive || modelChanged; + if (credential !== undefined && keyChanged) + return void changeKey(provider, credential, model); + // Model-only changes never write to the environment. + if (grantChanges) enable(() => chooseVoiceDestination(id, provider, model)); + onClose(); + }; + return ( + + {!isEditing && ( + + } + selected={isActive} + disabled={saved === undefined || (!saved && !canManage) || saving} + description={ + saved === undefined + ? "Update the environment to use this provider" + : isActive && routed + ? `${activeModel} · Tap to change model${canManage ? " or key" : ""}` + : isActive && saved && canManage + ? "Tap to manage key" + : saved + ? routed + ? "API key saved · choose a model to use it" + : "API key saved on this environment" + : canManage + ? "Add an API key to use this provider" + : "Ask an environment administrator to add an API key" + } + onPress={() => { + if (routed) return edit(provider, !isActive); + if (!saved) return edit(provider, true); + if (isActive) { + if (canManage) edit(provider, false); + return; + } + Dialog.confirm({ + title: `Use ${label} for dictation?`, + message: `${disclosure} Select This device to stop sharing.`, + confirmText: `Use ${label}`, + onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)), + }); + }} + /> + )} + {isEditing && routed && ( + <> + {disclosure} + {catalog ? ( + + ) : ( + + )} + + )} + {isEditing && (canManage || routed) && ( + <> + {canManage && sources.length > 0 && ( + + { + setDraft(""); + setSourceId(null); + }} + /> + {sources.map((candidate) => ( + { + setDraft(""); + setSourceId(candidate.id); + }} + /> + ))} + + )} + {canManage && ( + { + if (routed) saveRouted(); + else if (credential) void changeKey(provider, credential); + }} + value={draft} + onChangeText={setDraft} + className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground" + /> + )} + {routed && !saved && !canManage ? ( + {}} + /> + ) : ( + { + if (routed) saveRouted(); + else if (credential) void changeKey(provider, credential); + }} + /> + )} + {saved && canManage && ( + + Dialog.confirm({ + title: `Remove the ${label} key from ${environment.label}?`, + message: "Everyone using it for dictation will lose access.", + confirmText: "Remove key", + destructive: true, + onConfirm: () => void changeKey(provider, null), + }) + } + /> + )} + + + )} + + ); + }) + ); + return ( + + + {content} + + + ); +} diff --git a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx index 57ca2cfe5dd40..5f7328214cb16 100644 --- a/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx @@ -30,6 +30,7 @@ import { hasEnvironmentVoiceCapture, } from "../voice-input/voiceDestination"; import { VoiceModelSection } from "./VoiceModelSection"; +import { CloudVoiceSection } from "./CloudVoiceSection"; import type { EnvironmentId } from "@t3tools/contracts"; import { ThreadSwipeable } from "../home/thread-swipe-actions"; import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow"; @@ -241,7 +242,10 @@ export function SettingsVoiceRouteScreen() { ))} {isReady && grant && !grantEnvironment ? ( @@ -255,7 +259,27 @@ export function SettingsVoiceRouteScreen() { /> ) : null} - {grantEnvironment ? ( + {hasEnvironmentVoiceCapture && + environments + .filter( + (env) => + env.serverConfig?.environment.capabilities.cloudVoiceTranscription || + (grant?.destination.kind === "cloud" && + grant.destination.environmentId === env.environmentId), + ) + .map((environment) => ( + + ))} + {grantEnvironment && grant?.destination.kind === "environment" ? ( ({ @@ -46,7 +51,11 @@ vi.mock("../../state/presentation", () => ({ mocks.probe(id); return Atom.make({ connection: { phase: id === "offline" ? "offline" : "connected" }, - serverConfig: { environment: { capabilities: { voiceTranscription: true } } }, + serverConfig: { + environment: { + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true }, + }, + }, }); }), }, @@ -159,6 +168,58 @@ it("keeps local as the default and never reuses consent for an offline or resele ["https://my-server.test/api/voice/status", "error"], ["https://my-server.test/api/voice/transcribe", "error"], ]); + chooseVoiceDestination(host, "openai"); + const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!; + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true })); + await expect(cloud.prepare(options)).rejects.toMatchObject({ + code: "unavailable", + message: expect.stringContaining("Add your OpenAI API key"), + }); + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: true, xai: true })); + const recorded = await cloud.prepare(options); + chooseVoiceDestination(host, "xai"); + await expect(recorded.transcribe("file:///voice.m4a", options)).rejects.toThrow( + "Your transcription choice changed.", + ); + mocks.fetch + .mockResolvedValueOnce(Response.json({ openai: true, xai: true })) + .mockResolvedValueOnce(Response.json({ text: "cloud" })); + const next = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options); + await expect(next.transcribe("file:///voice.m4a", options)).resolves.toBe("cloud"); + mocks.fetch.mockResolvedValueOnce( + Response.json( + { + _tag: "CloudSpeechError", + provider: "xai", + reason: "rejected-key", + }, + { status: 422 }, + ), + ); + await expect(next.transcribe("file:///voice.m4a", options)).rejects.toMatchObject({ + _tag: "VoiceTranscriptionError", + code: "transcription-failed", + message: expect.stringContaining("rejected this API key"), + }); + expect(mocks.fetch.mock.calls.slice(2).map(([url]) => new URL(String(url)).pathname)).toEqual([ + "/api/voice/cloud/keys", + "/api/voice/cloud/keys", + "/api/voice/cloud/keys", + "/api/voice/cloud/xai/transcribe", + "/api/voice/cloud/xai/transcribe", + ]); + // OpenRouter without a model is no destination; with one, the exact id travels with the audio. + chooseVoiceDestination(host, "openrouter"); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + chooseVoiceDestination(host, "openrouter", "acme/transcribe-1"); + mocks.fetch + .mockResolvedValueOnce(Response.json({ openai: true, xai: true, openrouter: true })) + .mockResolvedValueOnce(Response.json({ text: "routed" })); + const routed = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options); + await expect(routed.transcribe("file:///voice.m4a", options)).resolves.toBe("routed"); + const upload = new URL(String(mocks.fetch.mock.lastCall![0])); + expect(upload.pathname).toBe("/api/voice/cloud/openrouter/transcribe"); + expect(upload.searchParams.get("model")).toBe("acme/transcribe-1"); const failWrite = () => { throw new Error("Device storage is unavailable"); }; @@ -172,3 +233,27 @@ it("keeps local as the default and never reuses consent for an offline or resele appAtomRegistry.reset(); } }); + +it("a late cloud key save cannot override a newer destination choice", () => { + const host = EnvironmentId.make("my-server"); + try { + chooseVoiceDestination(null); + const first = pendingCloudVoiceChoice(host, "openai"); + chooseVoiceDestination(null); + first(); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + expect(JSON.parse(mocks.write.mock.lastCall![1])).toBeNull(); + + const older = pendingCloudVoiceChoice(host, "openai"); + const newer = pendingCloudVoiceChoice(host, "xai"); + older(); + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull(); + newer(); + const grant = appAtomRegistry.get(voiceGrantAtom); + older(); + expect(appAtomRegistry.get(voiceGrantAtom)).toBe(grant); + expect(grant?.destination).toEqual({ kind: "cloud", environmentId: host, provider: "xai" }); + } finally { + appAtomRegistry.reset(); + } +}); diff --git a/apps/mobile/src/features/voice-input/voiceDestination.ts b/apps/mobile/src/features/voice-input/voiceDestination.ts index 6f30769c2d3a2..dd27cca674888 100644 --- a/apps/mobile/src/features/voice-input/voiceDestination.ts +++ b/apps/mobile/src/features/voice-input/voiceDestination.ts @@ -1,10 +1,17 @@ import { AuthOrchestrationOperateScope, + type CloudSpeechModelId, + type CloudSpeechProvider, type EnvironmentId, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + type VoiceTranscriptionDestination, } from "@t3tools/contracts"; import { getEnvironmentSpeechModels, + getCloudSpeechKeys, + getCloudSpeechModels, + requireCloudSpeechKey, + transcribeCloudPcm, getEnvironmentSpeechStatus, transcribeEnvironmentPcm, VoiceTranscriptionError, @@ -34,38 +41,74 @@ export const voiceGrantAtom = Atom.make((get) => { return AsyncResult.getOrElse(get(storedGrant), () => null); }); -export function chooseVoiceDestination(environmentId: EnvironmentId | null) { +let choiceRevision = 0; + +type CloudDestination = Extract; +/** OpenRouter has no default model, so a cloud choice without one is no choice at all. */ +function cloudVoiceDestination( + environmentId: EnvironmentId, + provider: CloudSpeechProvider, + model?: CloudSpeechModelId, +): CloudDestination | null { + if (provider === "openrouter") + return model ? { kind: "cloud", environmentId, provider, model } : null; + return { kind: "cloud", environmentId, provider }; +} + +export function chooseVoiceDestination( + environmentId: EnvironmentId | null, + provider?: CloudSpeechProvider, + model?: CloudSpeechModelId, +) { + choiceRevision++; // A failed change must not leave the old destination armed in this session. appAtomRegistry.set(choice, null); - const grant: VoiceGrant | null = + const destination: VoiceTranscriptionDestination | null = environmentId === null ? null - : { - id: uuidv4(), - destination: { kind: "environment", environmentId }, - }; + : provider + ? cloudVoiceDestination(environmentId, provider, model) + : { kind: "environment", environmentId }; + const grant: VoiceGrant | null = destination && { id: uuidv4(), destination }; writeVoiceGrant(grant); appAtomRegistry.set(choice, grant); } +export function pendingCloudVoiceChoice( + id: EnvironmentId, + provider: CloudSpeechProvider, + model?: CloudSpeechModelId, +) { + const revision = ++choiceRevision; + return () => { + if (revision === choiceRevision) chooseVoiceDestination(id, provider, model); + }; +} + const native = requireOptionalNativeModule<{ decodeRecording?: (uri: string) => Promise; }>("T3VoiceTranscription"); export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function"; -export const voiceConnectionAtom = Atom.family((id: EnvironmentId) => - Atom.make((get) => { - const environment = get(environmentPresentations.presentationAtom(id)); - if ( - !hasEnvironmentVoiceCapture || - environment?.connection.phase !== "connected" || - !environment.serverConfig?.environment.capabilities.voiceTranscription || - !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope) - ) - return null; - return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id))); - }), -); +const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") => + Atom.family((id: EnvironmentId) => + Atom.make((get) => { + const environment = get(environmentPresentations.presentationAtom(id)); + if ( + !hasEnvironmentVoiceCapture || + environment?.connection.phase !== "connected" || + !environment.serverConfig?.environment.capabilities[capability] || + !sessionHasScope( + get(environmentSession.sessionStateAtom(id)), + AuthOrchestrationOperateScope, + ) + ) + return null; + return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id))); + }), + ); +export const voiceConnectionAtom = voiceConnections("voiceTranscription"); +export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription"); // React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error". export function runSpeech( @@ -92,10 +135,20 @@ export function runSpeech( { signal }, ); } -function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) { +function environmentTranscriber( + prepared: PreparedConnection, + canSend: () => boolean, + cloud?: CloudDestination, +) { + const provider = cloud?.provider; + const model = cloud?.provider === "openrouter" ? cloud.model : undefined; return createEnvironmentTranscriber({ canSend, prepare: async (signal) => { + if (provider) { + await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend); + return "auto"; + } const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -126,7 +179,21 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo upload: async (pcm, signal) => ( await runSpeech( - transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend), + provider + ? transcribeCloudPcm( + prepared, + provider, + pcm, + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + canSend, + model, + ) + : transcribeEnvironmentPcm( + prepared, + pcm, + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + canSend, + ), signal, canSend, ) @@ -137,9 +204,10 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo const speechAtom = ( id: EnvironmentId, read: (prepared: PreparedConnection) => Effect.Effect, + connectionAtom = voiceConnectionAtom, ) => Atom.make((get) => { - const prepared = get(voiceConnectionAtom(id)); + const prepared = get(connectionAtom(id)); return prepared ? Effect.tryPromise({ try: (signal) => runSpeech(read(prepared), signal), @@ -155,6 +223,14 @@ const speechAtom = ( export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) => speechAtom(id, getEnvironmentSpeechStatus), ); +export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) => + speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom), +); +export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) => + id === null + ? Atom.make(Effect.succeed(null)) + : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom), +); export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) => speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)), ); @@ -167,7 +243,10 @@ export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) => export const voiceRouteAtom = Atom.make((get) => { const grant = get(voiceGrantAtom); if (!grant) return null; - const connectionAtom = voiceConnectionAtom(grant.destination.environmentId); + const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined; + const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)( + grant.destination.environmentId, + ); const connection = get(connectionAtom); const registry = get.registry; return { @@ -176,6 +255,7 @@ export const voiceRouteAtom = Atom.make((get) => { environmentTranscriber( connection, () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection, + cloud, ), }; }); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 3e780298d70a6..6d2c30db16130 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -274,6 +274,7 @@ export const make = Effect.gen(function* () { scheduledTaskThreadModes: true, npmReleaseLookup: true, voiceTranscription: isSpeechPlatformSupported(hostPlatform, hostArchitecture, hostLibc), + cloudVoiceTranscription: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), ...(serverInstallation === null ? {} : { serverInstallation }), ...(serverSelfUpdate === "boot-service" || desktopAppUpdate diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 5c9ab9c433ffc..e7baa0a91164e 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -43,6 +43,7 @@ import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts"; import * as PullRequestHttp from "./pullRequest/http.ts"; import * as SpeechModelStore from "./speech/SpeechModelStore.ts"; import * as SpeechService from "./speech/SpeechService.ts"; +import * as CloudSpeech from "./speech/CloudSpeech.ts"; import { speechHttpApiLayer } from "./speech/http.ts"; import { speechStreamRouteLayer } from "./speech/stream.ts"; import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderRegistry.ts"; @@ -823,6 +824,7 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide(SpeechModelStore.layer), ), ), + Layer.provide(CloudSpeech.layer.pipe(Layer.provide(SecretResolver.layer))), Layer.provide(layerServerSettings), Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index caba93a788b7f..6c470d1f83f05 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1642,6 +1642,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" const openCode = ProviderInstanceId.make("opencode"); const hub = UsageLimitSourceId.make("hub"); const withUsage = yield* serverSettings.updateSettings({ + cloudSpeechCredentials: { openai: usageReference, xai: "literal-fixture" }, usageLimitSources: { [hub]: { kind: "cliproxy", @@ -1687,6 +1688,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" secretReferences: false, }); assert.notInclude(JSON.stringify(legacy), "op://"); + assert.deepEqual(legacy.cloudSpeechCredentials, { + openai: SECRET_REDACTED, + xai: SECRET_REDACTED, + openrouter: "", + }); assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED); assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED); assert.equal(legacy.easDeviceHosts[0]?.accessToken, SECRET_REDACTED); @@ -1696,6 +1702,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" assert.equal(configOf(legacy, antigravity)?.apiKey, SECRET_REDACTED); assert.equal(configOf(legacy, openCode)?.serverPassword, SECRET_REDACTED); const echoed = yield* serverSettings.updateSettings({ + cloudSpeechCredentials: legacy.cloudSpeechCredentials, sourceControlHosts: legacy.sourceControlHosts, usageLimitSources: legacy.usageLimitSources, easDeviceHosts: legacy.easDeviceHosts, @@ -1708,6 +1715,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": "ghp_ghe", }); assert.deepEqual(echoed.usageLimitSources[hub]?.managementKey, usageReference); + assert.deepEqual(echoed.cloudSpeechCredentials, { + openai: usageReference, + xai: "literal-fixture", + openrouter: "", + }); assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference); assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference); assert.deepEqual(configOf(echoed, antigravity)?.apiKey, antigravityReference); @@ -1747,6 +1759,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" bitbucket: { accessToken: personal("Bitbucket/token") }, github: { tokens: { "github.com": personal("GitHub/token"), "ghe.acme.test": work } }, }, + cloudSpeechCredentials: { openai: personal("Voice/key"), xai: work }, usageLimitSources: { [hub]: { kind: "cliproxy", @@ -1791,6 +1804,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "ghe.acme.test": work, }); assert.equal(removed.usageLimitSources[hub]?.managementKey, ""); + assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" }); assert.equal(removed.easDeviceHosts[0]?.accessToken, ""); assert.deepEqual(Object.keys(removed.secretProviders), ["work"]); }).pipe(Effect.provide(layerServerSettingsWithSecrets())), @@ -1800,6 +1814,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" Effect.sync(() => { // Shipped clients decode these as strings. Each must be covered by the older-client case above. assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [ + "cloudSpeechCredentials.openai", + "cloudSpeechCredentials.xai", + "cloudSpeechCredentials.openrouter", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", "usageLimitSources.*.managementKey", diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index ce93b51a8ebaf..ae69a61f1f5dd 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -1,6 +1,7 @@ import { AntigravitySettings, ClaudeSettings, + CloudSpeechCredentials, CodexSettings, EasDeviceHostConfig, type ProviderInstanceConfig, @@ -45,6 +46,7 @@ const providerSecretFields = new Map([ ), ]); const usageLimitSecretFields = secretSettingsFields(UsageLimitSourceConfig); +const voiceSecretFields = secretSettingsFields(CloudSpeechCredentials); const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig); const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig); const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url"); @@ -232,6 +234,11 @@ function mapSecretFieldGroups( return { ...settings, + cloudSpeechCredentials: mapFields( + settings.cloudSpeechCredentials, + voiceSecretFields, + (provider) => `voice-cloud-${provider}`, + ) as ServerSettings["cloudSpeechCredentials"], providerInstances, sourceControlHosts, usageLimitSources, diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts index 10a37b631ecf5..58c84365dfcef 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.test.ts @@ -164,6 +164,7 @@ it.effect("reads a collected reference again once it is invalidated", () => }, ]; cache.set("cliproxy#key", "stale"); + cache.set("voice#key", "stale"); cache.set("expo#token", "stale"); cache.set("simlock#token", "stale"); assert.deepEqual((yield* read()).bitbucket, { @@ -175,12 +176,14 @@ it.effect("reads a collected reference again once it is invalidated", () => resolver, Effect.succeed({ sourceControlHosts: hosts, + cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" }, usageLimitSources, easDeviceHosts, simlockDeviceHosts, }), ); assert.isFalse(cache.has("cliproxy#key")); + assert.isFalse(cache.has("voice#key")); assert.isFalse(cache.has("expo#token")); assert.isFalse(cache.has("simlock#token")); assert.deepEqual((yield* read()).bitbucket, { diff --git a/apps/server/src/sourceControl/sourceControlHostSecrets.ts b/apps/server/src/sourceControl/sourceControlHostSecrets.ts index ba5a868e0799d..94eec45d7c9ea 100644 --- a/apps/server/src/sourceControl/sourceControlHostSecrets.ts +++ b/apps/server/src/sourceControl/sourceControlHostSecrets.ts @@ -118,7 +118,11 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate") settings: Effect.Effect< Pick< ServerSettings, - "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" | "simlockDeviceHosts" + | "sourceControlHosts" + | "usageLimitSources" + | "easDeviceHosts" + | "simlockDeviceHosts" + | "cloudSpeechCredentials" >, E >, @@ -128,6 +132,7 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate") ...collectSourceControlHostSecretReferences(current.sourceControlHosts), ...collectUsageLimitSecretReferences(current.usageLimitSources), ...collectDeviceHostSecretReferences(current), + ...Object.values(current.cloudSpeechCredentials).filter(isSecretReference), ]), Effect.orElseSucceed(() => undefined), ); diff --git a/apps/server/src/speech/CloudSpeech.test.ts b/apps/server/src/speech/CloudSpeech.test.ts new file mode 100644 index 0000000000000..112c7a88d9b3b --- /dev/null +++ b/apps/server/src/speech/CloudSpeech.test.ts @@ -0,0 +1,200 @@ +import { it } from "@effect/vitest"; +import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import { expect, vi } from "vite-plus/test"; +import * as ServerSettings from "../serverSettings.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; + +it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved key", (provider) => + Effect.gen(function* () { + const reference = { + provider: SecretProviderId.make("work"), + reference: "op://Work/Voice/key", + }; + let resolved: string | undefined; + let cached: string | undefined; + const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)])); + const upstream = vi.fn(async () => Response.json({ text: "hello" })); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide( + Layer.mock(SecretResolver.SecretResolver)({ + read, + label: () => Effect.succeed("Work"), + invalidate: () => + Effect.sync(() => { + cached = undefined; + }), + }), + ), + ), + ), + ); + const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) => + service + .transcribe(provider, pcm, { + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + speechLanguage: "en", + speechCustomWords: [{ term: "T3 Code", aliases: [] }], + }) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other")); + expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError"); + yield* service.setKey(provider, Redacted.make("chosen")); + expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false }); + yield* Effect.flip(transcribe(new Uint8Array(3))); + expect(upstream).not.toHaveBeenCalled(); + expect(yield* transcribe()).toBe("hello"); + const [url, init] = upstream.mock.calls[0]!; + expect(String(url)).toBe( + { + openai: "https://api.openai.com/v1/audio/transcriptions", + xai: "https://api.x.ai/v1/stt", + }[provider], + ); + expect(init?.redirect).toBe("error"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen"); + const form = init?.body as FormData; + expect(form.get("model")).toBe( + provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0", + ); + expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]); + expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en"); + expect([...form.keys()].at(-1)).toBe("file"); + const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer())); + expect(wav.getUint32(24, true)).toBe(16_000); + expect(wav.getInt16(46, true)).toBe(16384); + upstream.mockResolvedValueOnce(new Response("sensitive upstream diagnostic", { status: 401 })); + expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" }); + yield* service.setKey(provider, null); + expect((yield* service.configured)[provider]).toBe(false); + yield* Effect.flip(transcribe()); + expect(upstream).toHaveBeenCalledTimes(2); + yield* service.setKey(provider, reference); + expect((yield* service.configured)[provider]).toBe(true); + expect(read).not.toHaveBeenCalled(); + resolved = "resolved-fixture"; + expect(yield* transcribe()).toBe("hello"); + expect(read).toHaveBeenCalledWith([reference]); + expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe( + "Bearer resolved-fixture", + ); + resolved = undefined; + expect(yield* Effect.flip(transcribe())).toMatchObject({ + provider, + reason: "unreadable-key", + }); + expect(upstream).toHaveBeenCalledTimes(3); + }), +); + +it.effect("uploads to OpenRouter only for a listed exact model, as zero-retention JSON WAV", () => + Effect.gen(function* () { + const upstream = vi.fn(async (input) => + new URL(String(input)).pathname.endsWith("/models") + ? Response.json({ data: [{ id: "acme/transcribe-1" }, { id: "acme/transcribe-2" }] }) + : Response.json({ text: "hello" }), + ); + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const uploads = () => + upstream.mock.calls.filter(([url]) => String(url).endsWith("/audio/transcriptions")); + const transcribe = (model: string | undefined) => + service + .transcribe( + "openrouter", + new Uint8Array(new Float32Array([0, 0.5]).buffer), + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" }, + model, + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)); + yield* service.setKey("openrouter", Redacted.make("router-key")); + expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ provider: "openrouter" }); + upstream.mockResolvedValueOnce(new Response(null, { status: 503 })); + expect(yield* Effect.flip(transcribe("acme/transcribe-2"))).toMatchObject({ + reason: "unavailable", + }); + expect(yield* Effect.flip(transcribe("acme/delisted"))).toMatchObject({ + provider: "openrouter", + }); + expect(uploads()).toEqual([]); + expect(yield* transcribe("acme/transcribe-2")).toBe("hello"); + const [[url, init]] = uploads() as [[URL, RequestInit]]; + expect(new URL(String(upstream.mock.calls[0]![0])).searchParams.get("zdr")).toBe("true"); + expect(String(url)).toBe("https://openrouter.ai/api/v1/audio/transcriptions"); + expect(init.redirect).toBe("error"); + const headers = new Headers(init.headers); + expect(headers.get("authorization")).toBe("Bearer router-key"); + expect(headers.get("content-type")).toContain("application/json"); + const body = JSON.parse(yield* Effect.promise(() => new Response(init.body).text())) as { + model: string; + models?: unknown; + provider: { zdr: boolean }; + input_audio: { format: string; data: string }; + }; + expect(body).toMatchObject({ + model: "acme/transcribe-2", + provider: { zdr: true }, + input_audio: { format: "wav" }, + }); + expect(body.models).toBeUndefined(); + const wav = Buffer.from(body.input_audio.data, "base64"); + expect(wav.toString("ascii", 0, 4)).toBe("RIFF"); + expect(wav.readInt16LE(46)).toBe(16384); + }), +); + +it.effect("removing a key while a catalog read is pending prevents the upload", () => + Effect.gen(function* () { + const service = yield* CloudSpeech.CloudSpeech.pipe( + Effect.provide( + CloudSpeech.layer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(ServerSettings.ServerSettingsService.layerTest()), + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})), + ), + ), + ); + const catalogStarted = Promise.withResolvers(); + const catalog = Promise.withResolvers(); + const uploads: string[] = []; + const upstream: typeof fetch = async (input) => { + if (new URL(String(input)).pathname.endsWith("/models")) { + catalogStarted.resolve(); + return catalog.promise; + } + uploads.push(String(input)); + return Response.json({ text: "unexpected" }); + }; + yield* service.setKey("openrouter", Redacted.make("old-key")); + const job = yield* service + .transcribe( + "openrouter", + new Uint8Array(4), + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, + "acme/voice", + ) + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream), Effect.forkChild); + yield* Effect.promise(() => catalogStarted.promise); + yield* service.setKey("openrouter", null); + catalog.resolve(Response.json({ data: [{ id: "acme/voice" }] })); + const exit = yield* Fiber.await(job); + expect(uploads).toEqual([]); + expect(exit._tag).toBe("Failure"); + }), +); diff --git a/apps/server/src/speech/CloudSpeech.ts b/apps/server/src/speech/CloudSpeech.ts new file mode 100644 index 0000000000000..3b7eab6b5242b --- /dev/null +++ b/apps/server/src/speech/CloudSpeech.ts @@ -0,0 +1,258 @@ +import { + CloudSpeechApiKey, + CloudSpeechError, + CloudSpeechModel, + CloudSpeechModelId, + SettingsSecretValue, + isSecretReference, + type SecretReference, + type ServerSettingsError, + type CloudSpeechKeys, + type CloudSpeechProvider, + EnvironmentSpeechTranscriptionResult, + type SpeechTranscriptionOptions, +} from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; +import * as ServerSettings from "../serverSettings.ts"; +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; +import { applySpeechAliases } from "./customWords.ts"; +import { removeSpeechFillerWords } from "./fillerWords.ts"; +import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts"; + +const endpoints = { + openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" }, + xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" }, + openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" }, +}; +const OPENROUTER_CATALOG_URL = + "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true"; +const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +const sameCredential = Schema.toEquivalence(SettingsSecretValue); +const OpenRouterCatalog = Schema.Struct({ + data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })), +}); + +function wave(samples: Float32Array) { + const buffer = new ArrayBuffer(44 + samples.length * 2); + const view = new DataView(buffer); + const tag = (at: number, value: string) => + new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value)); + tag(0, "RIFF"); + view.setUint32(4, buffer.byteLength - 8, true); + tag(8, "WAVE"); + tag(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, 16_000, true); + view.setUint32(28, 32_000, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + tag(36, "data"); + view.setUint32(40, buffer.byteLength - 44, true); + for (let index = 0; index < samples.length; index++) { + const sample = samples[index]!; + view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true); + } + return buffer; +} + +function audioBody( + provider: "openai" | "xai", + audio: ArrayBuffer, + language: string, + options: SpeechTranscriptionOptions, +) { + const form = new FormData(); + form.set("model", endpoints[provider].model); + if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language); + for (const { term } of options.speechCustomWords) { + if (provider === "openai" && /[<>\r\n]/.test(term)) continue; + form.append(provider === "openai" ? "keywords[]" : "keyterm", term); + } + if (provider === "xai") { + if (language !== "auto") form.set("format", "true"); + form.set( + "filler_words", + String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)), + ); + if (!options.speechSilenceFiltering) form.set("vad_threshold", "0"); + } + // xAI ignores options that follow the file part. + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav"); + return HttpClientRequest.bodyFormData(form); +} + +function openRouterBody(model: string, audio: ArrayBuffer, language: string) { + return HttpClientRequest.bodyJsonUnsafe({ + model, + input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" }, + ...(language !== "auto" ? { language } : {}), + provider: { zdr: true }, + }); +} + +export class CloudSpeech extends Context.Service< + CloudSpeech, + { + readonly configured: Effect.Effect; + /** OpenRouter's public transcription catalog, cached briefly on this server. */ + readonly models: Effect.Effect, CloudSpeechError>; + readonly setKey: ( + provider: CloudSpeechProvider, + key: Redacted.Redacted | SecretReference | null, + ) => Effect.Effect; + readonly transcribe: ( + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + model?: string | undefined, + ) => Effect.Effect; + } +>()("t3/speech/CloudSpeech") {} + +export const layer = Layer.effect( + CloudSpeech, + Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const resolver = yield* SecretResolver.SecretResolver; + const http = yield* HttpClient.HttpClient; + const credentials = settings.getSettings.pipe( + Effect.map((value) => value.cloudSpeechCredentials), + ); + const configured = credentials.pipe( + Effect.map(({ openai, xai, openrouter }) => ({ + openai: Boolean(openai), + xai: Boolean(xai), + openrouter: Boolean(openrouter), + })), + ); + const execute = (request: HttpClientRequest.HttpClientRequest) => + http + .execute(request) + .pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provideService(HttpClient.TracerPropagationEnabled, false), + ); + const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () { + const unavailable = () => + new CloudSpeechError({ provider: "openrouter", reason: "unavailable" }); + const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe( + Effect.mapError(unavailable), + ); + if (response.status < 200 || response.status >= 300) return yield* unavailable(); + const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe( + Effect.mapError(unavailable), + ); + return data.flatMap(({ id, name }) => + isModelId(id) ? [{ id, name: name?.trim() || id }] : [], + ); + }); + // The catalog is public, so the cache holds no credential or consent state. + const models = yield* Effect.cachedWithTTL( + fetchCatalog().pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })), + }), + ), + (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0), + ); + const setKey = Effect.fn("CloudSpeech.setKey")(function* ( + provider: CloudSpeechProvider, + key: Redacted.Redacted | SecretReference | null, + ) { + yield* settings.updateSettings({ + cloudSpeechCredentials: { + [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""), + }, + }); + }); + const transcribe = Effect.fn("CloudSpeech.transcribe")( + function* ( + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + model?: string, + ) { + const source = (yield* credentials)[provider]; + if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" }); + if (provider === "openrouter" && (!model || !isModelId(model))) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); + const samples = yield* Effect.try({ + try: () => decodeSpeechPcm(pcm), + catch: () => + new SpeechInvalidAudioError({ + byteLength: pcm.byteLength, + message: "Invalid PCM audio.", + }), + }); + const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage); + if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model)) + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" }); + // A vault that locked since the last recording must not authorize a new upload. + if (isSecretReference(source)) yield* resolver.invalidate([source]); + const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" }); + const key = yield* requireSettingsSecret(resolver, source).pipe( + Effect.mapError(unreadable), + ); + if (!isApiKey(key)) return yield* unreadable(); + const audio = wave(samples); + const request = HttpClientRequest.post(endpoints[provider].url).pipe( + HttpClientRequest.bearerToken(Redacted.make(key)), + provider === "openrouter" + ? openRouterBody(model!, audio, language) + : audioBody(provider, audio, language, options), + ); + const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" }); + // A catalog fetch or vault lookup may have outlived the credential it started with. + const current = (yield* credentials)[provider]; + if (!current || !sameCredential(source, current)) + return yield* new CloudSpeechError({ + provider, + reason: current ? "key-changed" : "missing-key", + }); + const response = yield* execute(request).pipe(Effect.mapError(unavailable)); + if (response.status < 200 || response.status >= 300) + return yield* new CloudSpeechError({ + provider, + reason: + response.status === 429 + ? "rate-limit" + : [401, 402, 403].includes(response.status) + ? "rejected-key" + : response.status === 404 && provider === "openrouter" + ? "model-unavailable" + : "unavailable", + }); + const { text } = yield* HttpClientResponse.schemaBodyJson( + EnvironmentSpeechTranscriptionResult, + )(response).pipe(Effect.mapError(unavailable)); + const corrected = applySpeechAliases(text, options.speechCustomWords); + return ( + options.speechRemoveFillerWords + ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord) + : corrected + ).trim(); + }, + (effect, provider) => + effect.pipe( + Effect.timeoutOrElse({ + duration: "2 minutes", + orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })), + }), + ), + ); + return CloudSpeech.of({ configured, models, setKey, transcribe }); + }), +); diff --git a/apps/server/src/speech/http.test.ts b/apps/server/src/speech/http.test.ts index 169a1e893cef8..f67129be60423 100644 --- a/apps/server/src/speech/http.test.ts +++ b/apps/server/src/speech/http.test.ts @@ -7,6 +7,9 @@ import { expect, vi } from "vite-plus/test"; import { AuthSessionId, AuthOrchestrationOperateScope, + AuthSettingsWriteScope, + type AuthEnvironmentScope, + CloudSpeechError, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES, DEFAULT_SERVER_SETTINGS, @@ -16,6 +19,7 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import * as HttpRouter from "effect/http/HttpRouter"; import * as HttpServer from "effect/http/HttpServer"; @@ -23,6 +27,7 @@ import * as HttpApi from "effect/http-api/HttpApi"; import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import type * as NetAddress from "effect/net/NetAddress"; import * as SpeechService from "./SpeechService.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; @@ -30,7 +35,20 @@ import { speechHttpApiLayer } from "./http.ts"; const decodeSessionId = Schema.decodeUnknownSync(AuthSessionId); +const principal = (...scopes: AuthEnvironmentScope[]) => + Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => + effect.pipe( + Effect.provideService(EnvironmentAuthenticatedPrincipal, { + sessionId: decodeSessionId("voice-test"), + subject: "test", + method: "bearer-access-token", + scopes: new Set(scopes), + }), + ), + ); + const dependencies = Layer.mergeAll( + Layer.mock(CloudSpeech.CloudSpeech)({}), Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }), Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS), @@ -51,21 +69,11 @@ const dependencies = Layer.mergeAll( effectIt.live("rejects oversized chunked audio without invoking transcription", () => Effect.gen(function* () { const transcribe = vi.fn(() => Effect.succeed("unexpected")); - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => - effect.pipe( - Effect.provideService(EnvironmentAuthenticatedPrincipal, { - sessionId: decodeSessionId("voice-test"), - subject: "test", - method: "bearer-access-token", - scopes: new Set([AuthOrchestrationOperateScope]), - }), - ), - ); const routes = HttpApiBuilder.layer( HttpApi.make("environment").add(EnvironmentVoiceHttpApi), ).pipe( Layer.provide(speechHttpApiLayer), - Layer.provide(auth), + Layer.provide(principal(AuthOrchestrationOperateScope)), Layer.provide( Layer.succeed(SpeechService.SpeechService, { status: Effect.succeed({ supported: false as const, reason: "test" }), @@ -97,6 +105,14 @@ effectIt.live("rejects oversized chunked audio without invoking transcription", }), ); expect(select.status).toBe(403); + const changeKey = yield* Effect.promise(() => + fetch(new URL("cloud/xai/key", url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: "fixture" }), + }), + ); + expect(changeKey.status).toBe(403); yield* Effect.promise(() => expect( fetch(url, { @@ -120,6 +136,101 @@ effectIt.live("rejects oversized chunked audio without invoking transcription", }), ); +effectIt.effect( + "keeps keys write-only and returns provider failures without an auth challenge", + () => + Effect.gen(function* () { + let stored = false; + const reference = { provider: "work", reference: "op://Work/Voice/key" }; + const save = vi.fn(); + const transcribe = vi.fn(); + const cloud = Layer.mock(CloudSpeech.CloudSpeech)({ + configured: Effect.sync(() => ({ openai: stored, xai: false })), + models: Effect.succeed([{ id: "acme/transcribe-1", name: "Acme Transcribe" }]), + setKey: (provider, key) => + Effect.sync(() => { + save(provider, key); + stored = + provider === "openai" && + Redacted.isRedacted(key) && + Redacted.value(key) === "fixture"; + }), + transcribe: (provider, _pcm, _options, model) => { + transcribe(provider, model); + return Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" })); + }, + }); + const { handler, dispose } = HttpRouter.toWebHandler( + HttpApiBuilder.layer(HttpApi.make("environment").add(EnvironmentVoiceHttpApi)).pipe( + Layer.provide(speechHttpApiLayer), + Layer.provide(principal(AuthOrchestrationOperateScope, AuthSettingsWriteScope)), + Layer.provide(Layer.mock(SpeechService.SpeechService)({})), + Layer.provide(cloud), + Layer.provide(dependencies), + Layer.provide(HttpServer.layerServices), + Layer.provide(NodeServices.layer), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(dispose)); + const context = yield* Layer.build(NodeServices.layer); + const call = (path: string, init?: RequestInit) => + Effect.promise(() => + handler(new Request(`http://localhost/api/voice/cloud/${path}`, init), context), + ); + const saved = yield* call("openai/key", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: "fixture" }), + }); + expect(saved.status).toBe(204); + expect(yield* Effect.promise(() => saved.text())).toBe(""); + const keys = yield* call("keys"); + expect(yield* Effect.promise(() => keys.json())).toEqual({ + openai: true, + xai: false, + }); + const setReference = (key: unknown) => + call("xai/key", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key }), + }); + expect((yield* setReference(reference)).status).toBe(204); + expect(save).toHaveBeenLastCalledWith("xai", reference); + expect((yield* setReference(reference.reference)).status).toBe(400); + expect(save).toHaveBeenCalledTimes(2); + const failed = yield* call("openai/transcribe", { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + }); + expect(failed.status).toBe(422); + expect(yield* Effect.promise(() => failed.json())).toEqual({ + _tag: "CloudSpeechError", + provider: "openai", + reason: "rejected-key", + }); + expect(transcribe).toHaveBeenLastCalledWith("openai", undefined); + const catalog = yield* call("openrouter/models"); + expect(yield* Effect.promise(() => catalog.json())).toEqual({ + models: [{ id: "acme/transcribe-1", name: "Acme Transcribe" }], + }); + const audio = { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), + }; + expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe( + 400, + ); + expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe( + 422, + ); + expect(transcribe).toHaveBeenLastCalledWith("openrouter", "acme/transcribe-1"); + }).pipe(Effect.scoped), +); + effectIt.effect.each([ { error: new SpeechService.SpeechInvalidAudioError({ byteLength: 3, message: "invalid PCM" }), @@ -151,16 +262,6 @@ effectIt.effect.each([ ({ error, reason, ...options }) => Effect.gen(function* () { // Only authentication and the native service are replaced; exercise the actual HTTP handlers and codecs. - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) => - effect.pipe( - Effect.provideService(EnvironmentAuthenticatedPrincipal, { - sessionId: decodeSessionId("voice-test"), - subject: "test", - method: "bearer-access-token", - scopes: new Set([AuthOrchestrationOperateScope]), - }), - ), - ); const transcribe = vi.fn(() => Effect.fail(error)); const service = Layer.succeed(SpeechService.SpeechService, { status: Effect.succeed({ supported: false as const, reason: "test" }), @@ -179,7 +280,7 @@ effectIt.effect.each([ const { handler, dispose } = HttpRouter.toWebHandler( HttpApiBuilder.layer(api).pipe( Layer.provide(speechHttpApiLayer), - Layer.provide(auth), + Layer.provide(principal(AuthOrchestrationOperateScope)), Layer.provide(service), Layer.provide(HttpServer.layerServices), Layer.provide(dependencies), diff --git a/apps/server/src/speech/http.ts b/apps/server/src/speech/http.ts index 2e1d1ae71fced..eb4a7a01b88fb 100644 --- a/apps/server/src/speech/http.ts +++ b/apps/server/src/speech/http.ts @@ -21,6 +21,7 @@ import { requireEnvironmentScope, } from "../auth/http.ts"; import * as SpeechPostProcessing from "./postProcessing.ts"; +import * as CloudSpeech from "./CloudSpeech.ts"; import * as SpeechService from "./SpeechService.ts"; const MAX_TRANSCRIBE_BODY_BYTES = SpeechService.MAX_SPEECH_BYTES + SPEECH_MAX_OPTIONS_BYTES + 4; @@ -57,6 +58,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group( "voice", Effect.fnUntraced(function* (handlers) { const speech = yield* SpeechService.SpeechService; + const cloud = yield* CloudSpeech.CloudSpeech; const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing; const guard = Effect.fnUntraced(function* ( endpoint: string, @@ -66,6 +68,53 @@ export const speechHttpApiLayer = HttpApiBuilder.group( yield* requireEnvironmentScope(scope); }); return handlers + .handle( + "cloudKeys", + Effect.fn("environment.voice.cloudKeys")(function* (args) { + yield* guard(args.endpoint.name); + return yield* cloud.configured.pipe( + Effect.catch((error) => failEnvironmentInternal("internal_error", error)), + ); + }), + ) + .handle( + "setCloudKey", + Effect.fn("environment.voice.setCloudKey")(function* (args) { + yield* guard(args.endpoint.name, AuthSettingsWriteScope); + yield* cloud + .setKey(args.params.provider, args.payload.key) + .pipe(Effect.catch((error) => failEnvironmentInternal("internal_error", error))); + }), + ) + .handle( + "cloudModels", + Effect.fn("environment.voice.cloudModels")(function* (args) { + yield* guard(args.endpoint.name); + return { models: yield* cloud.models }; + }), + ) + .handle( + "transcribeCloud", + Effect.fn("environment.voice.transcribeCloud")(function* (args) { + yield* guard(args.endpoint.name); + const decoded = parseAudio(args.payload); + if (Option.isNone(decoded)) return yield* failEnvironmentInvalidRequest("invalid_audio"); + const text = yield* cloud + .transcribe( + args.params.provider, + decoded.value.pcm, + decoded.value.options, + args.query.model, + ) + .pipe( + Effect.catchTags({ + ServerSettingsError: (error) => failEnvironmentInternal("internal_error", error), + SpeechInvalidAudioError: () => failEnvironmentInvalidRequest("invalid_audio"), + }), + ); + return { text }; + }), + ) .handle( "status", Effect.fn("environment.voice.status")(function* (args) { diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx new file mode 100644 index 0000000000000..0295603ed2038 --- /dev/null +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx @@ -0,0 +1,227 @@ +import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection"; +import { EnvironmentId } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import { expect, it, vi } from "vite-plus/test"; +import { page, userEvent } from "vite-plus/test/browser"; +import { render } from "../../../test/render"; +import "~/index.css"; +import { chooseAudioDestination } from "../../speech/voiceConsent"; +import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; +import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; +import { SettingsScopeProvider } from "./SettingsScopeContext"; + +vi.mock(import("../../state/session"), { spy: true }); +vi.mock("../../env", async (original) => ({ + ...(await original()), + isSecretsFeatureEnabled: true, +})); +vi.mock("../../state/environments", async (importOriginal) => ({ + ...(await importOriginal()), + useEnvironments: () => ({ + environments: [ + { + environmentId: "voice-host", + label: "Laptop", + connection: { phase: "available" }, + serverConfig: { + secretReferences: true, + settings: { + secretProviders: { + work: { kind: "one_password", label: "Work vault", config: { account: "work" } }, + }, + }, + environment: { + platform: { os: "linux", arch: "x64" }, + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true }, + }, + }, + }, + ], + }), +})); + +it("enables the chosen provider only after a successful setup and keeps its grant when editing the key", async () => { + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination({ kind: "environment", environmentId: id }); + const writes: unknown[] = []; + let rejectNextWrite = true; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: writes.length > 0, xai: true })); + if (url.pathname === "/api/voice/cloud/openai/key") { + if (rejectNextWrite) { + rejectNextWrite = false; + return Promise.resolve(new Response(null, { status: 403 })); + } + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + } + return fetch(request, init); + }); + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await expect.element(destination).toHaveTextContent("Laptop"); + await destination.click(); + await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect.element(destination).toHaveTextContent("Laptop"); + expect(writes).toEqual([]); + await destination.click(); + await page.getByRole("option", { name: "OpenAI via Laptop" }).click(); + await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click(); + await page.getByRole("option", { name: "Work vault" }).click(); + await dialog.getByLabelText("OpenAI API key reference").fill("op://Work/Voice/key"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog.getByRole("alert")).toBeVisible(); + expect(writes).toEqual([]); + await expect + .element(dialog.getByRole("button", { name: "Save and enable" })) + .toBeEnabled(); + await dialog.getByLabelText("OpenAI API key reference").click(); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenAI via Laptop"); + expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]); + const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1"); + await screen.getByRole("button", { name: "Manage key" }).click(); + await dialog.getByLabelText("OpenAI API key", { exact: true }).fill("replacement"); + await userEvent.keyboard("{Enter}"); + await expect.element(dialog).not.toBeInTheDocument(); + expect(writes.at(-1)).toEqual({ key: "replacement" }); + expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant); + await destination.click(); + await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click(); + await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click(); + await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop"); + await destination.click(); + await page.getByRole("option", { name: "Off", exact: true }).click(); + await expect.element(destination).toHaveTextContent("Off"); + }, + ); + }, + ); +}); + +it("OpenRouter needs an exact model, and changing it writes no credential", async () => { + const id = EnvironmentId.make("voice-host"); + const target = new PrimaryConnectionTarget({ + environmentId: id, + label: "Laptop", + httpBaseUrl: window.location.origin, + wsBaseUrl: "ws://localhost", + }); + const prepared = Option.some({ + ...target, + target, + socketUrl: target.wsBaseUrl, + httpAuthorization: null, + }); + using cleanup = new DisposableStack(); + cleanup.defer(() => chooseAudioDestination(null)); + chooseAudioDestination(null); + const writes: unknown[] = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => { + const url = new URL(request instanceof Request ? request.url : request, window.location.href); + if (url.pathname === "/api/voice/cloud/keys") + return Promise.resolve(Response.json({ openai: false, xai: false, openrouter: true })); + if (url.pathname === "/api/voice/cloud/openrouter/models") + return Promise.resolve( + Response.json({ + models: [ + { id: "acme/transcribe-1", name: "Acme: Transcribe 1" }, + { id: "other/listen-2", name: "Other: Listen 2" }, + ], + }), + ); + if (url.pathname === "/api/voice/cloud/openrouter/key") + return new Request(request, init).json().then((body) => { + writes.push(body); + return new Response(null, { status: 204 }); + }); + return fetch(request, init); + }); + const grantKey = "t3code:voice-audio-grant:v1"; + const grantModel = () => { + const raw = window.localStorage.getItem(grantKey); + const parsed = raw ? (JSON.parse(raw) as { destination?: { model?: string } }) : null; + return parsed?.destination?.model ?? null; + }; + await vi.mocked(useEnvironmentScope).withImplementation( + () => true, + async () => { + await vi.mocked(usePreparedConnection).withImplementation( + () => prepared, + async () => { + await using screen = await render( + {}}> + + , + ); + const destination = screen.getByRole("combobox", { name: "Transcribe with" }); + await destination.click(); + await page.getByRole("option", { name: "OpenRouter via Laptop" }).click(); + const dialog = page.getByRole("dialog"); + const save = dialog.getByRole("button", { name: "Save and enable" }); + await expect.element(save).toBeDisabled(); + const picker = dialog.getByRole("combobox", { name: "OpenRouter model" }); + await expect.element(picker).toBeVisible(); + await picker.click(); + await userEvent.keyboard("listen"); + await expect.element(page.getByRole("option", { name: /Other: Listen 2/ })).toBeVisible(); + await page.getByRole("option", { name: /Other: Listen 2/ }).click(); + await expect.element(save).toBeEnabled(); + const key = dialog.getByLabelText("OpenRouter API key", { exact: true }); + await key.fill("op://not-a-literal-key"); + await expect.element(save).toBeDisabled(); + await key.fill(""); + await save.click(); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + expect(grantModel()).toBe("other/listen-2"); + expect(writes).toEqual([]); + await screen.getByRole("button", { name: "Manage OpenRouter" }).click(); + const current = dialog.getByRole("combobox", { name: "OpenRouter model" }); + await expect.element(current).toHaveValue("Other: Listen 2"); + await userEvent.clear(current); + await userEvent.type(current, "acme"); + await page.getByRole("option", { name: /Acme: Transcribe 1/ }).click(); + await dialog.getByRole("button", { name: "Save" }).click(); + await expect.element(dialog).not.toBeInTheDocument(); + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop"); + expect(grantModel()).toBe("acme/transcribe-1"); + expect(writes).toEqual([]); + }, + ); + }, + ); +}); diff --git a/apps/web/src/components/settings/CloudVoiceConfiguration.tsx b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx new file mode 100644 index 0000000000000..eafde3d65bbea --- /dev/null +++ b/apps/web/src/components/settings/CloudVoiceConfiguration.tsx @@ -0,0 +1,471 @@ +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; +import { cloudSpeechDisclosure } from "@t3tools/shared/speech"; +import type { PreparedConnection } from "@t3tools/client-runtime/connection"; +import { + configuredSecretSources, + secretRequestReference, +} from "@t3tools/client-runtime/secret-request"; +import { + getCloudSpeechKeys, + getCloudSpeechModels, + setCloudSpeechKey, +} from "@t3tools/client-runtime/voice-input"; +import { + AuthOrchestrationOperateScope, + AuthSettingsWriteScope, + CloudSpeechApiKey, + CloudSpeechModelId, + CLOUD_SPEECH_LABELS, + CLOUD_SPEECH_PROVIDER_NAMES, + isSecretReference, + type CloudSpeechModel, + type CloudSpeechProvider, + type SecretProviderId, + type SettingsSecretValue, + type VoiceTranscriptionDestination, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import type { HttpClient } from "effect/http"; +import { Atom, AsyncResult } from "effect/reactivity"; +import { useId, useState } from "react"; +import { isSecretsFeatureEnabled } from "../../env"; +import { + inEnvironmentSpeechRuntime, + runEnvironmentSpeech, +} from "../../speech/runEnvironmentSpeech"; +import { + chooseAudioDestination, + pendingCloudAudioChoice, + useAudioGrant, +} from "../../speech/voiceConsent"; +import type { EnvironmentPresentation } from "../../state/environments"; +import { useEnvironmentScope, usePreparedConnection } from "../../state/session"; +import { Button } from "../ui/button"; +import { + Combobox, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, + ComboboxPopup, + useComboboxFilter, +} from "../ui/combobox"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; +import { Input } from "../ui/input"; +import { Label } from "../ui/label"; +import { SecretInput } from "./SecretInput"; +import { SecretSourceSelect } from "./SecretSourceInput"; + +// Per dialog instance, not an Atom.family: key presence must be read fresh each time it opens. +function useCloudRead( + prepared: PreparedConnection | null, + read: (prepared: PreparedConnection) => Effect.Effect, +) { + const make = (connection: PreparedConnection | null) => ({ + prepared: connection, + atom: Atom.make( + connection ? inEnvironmentSpeechRuntime(read(connection)) : Effect.succeed(null), + ), + }); + const [current, setCurrent] = useState(() => make(prepared)); + if (current.prepared !== prepared) setCurrent(make(prepared)); + return [useAtomValue(current.atom), useAtomRefresh(current.atom)] as const; +} +const isApiKey = Schema.is(CloudSpeechApiKey); +const isModelId = Schema.is(CloudSpeechModelId); +type CloudDestination = Extract; + +function ModelPicker({ + models, + value, + onChange, + disabled, + current, +}: { + models: ReadonlyArray; + value: string; + onChange: (next: string) => void; + disabled: boolean; + /** The grant's model, flagged when the catalog no longer lists it. */ + current: string | undefined; +}) { + const { contains } = useComboboxFilter(); + const selected = models.find((model) => model.id === value) ?? null; + const delisted = current !== undefined && !models.some((model) => model.id === current); + return ( +
+ model.name} + itemToStringValue={(model: CloudSpeechModel) => model.id} + filter={(model: CloudSpeechModel, query) => + contains(model.name, query) || contains(model.id, query) + } + value={selected} + onValueChange={(model) => onChange(model?.id ?? "")} + autoHighlight + disabled={disabled} + > + + + No transcription model matches. + + {(model: CloudSpeechModel) => ( + + + {model.name} + {model.id} + + + )} + + + + {delisted ? ( +

+ OpenRouter no longer lists {current}. Dictation stays off until you choose another model. +

+ ) : selected ? ( +

{selected.id}

+ ) : null} +
+ ); +} + +export function CloudVoiceConfiguration({ + environment, + provider, + active, + close, +}: { + environment: Pick; + provider: CloudSpeechProvider; + active: boolean; + close: () => void; +}) { + const id = environment.environmentId; + const name = CLOUD_SPEECH_LABELS[provider]; + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider]; + const routed = provider === "openrouter"; + const formId = useId(); + const prepared = Option.getOrNull(usePreparedConnection(id)); + const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope); + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope); + const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys); + const [models, refreshModels] = useCloudRead( + routed && canUse ? prepared : null, + getCloudSpeechModels, + ); + const grant = useAudioGrant(); + const currentModel = + active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter" + ? grant.destination.model + : undefined; + const [remove, setRemove] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [model, setModel] = useState(currentModel ?? ""); + const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined; + const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider]; + const sources = + isSecretsFeatureEnabled && environment.serverConfig?.secretReferences + ? configuredSecretSources(environment.serverConfig.settings?.secretProviders ?? {}) + : []; + const reference = + isSecretReference(existing) && sources.some((s) => s.id === existing.provider) + ? existing + : null; + const [sourceId, setSourceId] = useState(reference?.provider ?? null); + const [draft, setDraft] = useState(reference?.reference ?? ""); + const source = sources.find((s) => s.id === sourceId); + const credential = + sourceId !== null + ? source && secretRequestReference(source, draft) + : isApiKey(draft.trim()) + ? draft.trim() + : undefined; + const keyChanged = + credential !== undefined && + !( + isSecretReference(credential) && + reference?.provider === credential.provider && + reference.reference === credential.reference + ); + const connected = canUse && prepared !== null; + // Fixed providers edit the key only when nothing is saved or the device already uses them. + const editingKey = canManage && saved !== undefined && (routed || active || !saved); + const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null; + const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false); + const keyReady = draft.trim() ? credential !== undefined : Boolean(saved); + const modelChanged = model !== (currentModel ?? ""); + const destination = (): CloudDestination => + routed && isModelId(model) + ? { kind: "cloud", provider: "openrouter", environmentId: id, model } + : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id }; + const enable = () => chooseAudioDestination(destination()); + const canSubmit = routed + ? modelChosen && keyReady && (!active || modelChanged || keyChanged) + : credential !== undefined; + + const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => { + if (!connected || !canManage || busy) return; + setError(""); + setBusy(true); + try { + const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null; + await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value)); + refresh(); + commit?.(); + close(); + } catch { + setError("Couldn’t save this change. Check your connection and try again."); + } + setBusy(false); + }; + const submit = () => { + if (!canSubmit || busy) return; + const grantChanges = !active || modelChanged; + if (credential !== undefined && (!routed || keyChanged)) + return void saveKey(credential, grantChanges); + // Model-only changes never write to the environment. + if (grantChanges) enable(); + close(); + }; + + const keyField = ( + <> + +
+ {sources.length > 0 && ( + { + setSourceId(next); + setDraft(""); + setError(""); + }} + /> + )} +
+ {sourceId !== null ? ( + setDraft(event.target.value)} + /> + ) : ( + setDraft(event.target.value)} + /> + )} +
+
+

+ {draft.trim() && !credential + ? (source?.invalidReferenceMessage ?? "Enter an API key, not a URL or secret reference.") + : active && !routed + ? "Updating the key keeps dictation enabled." + : routed && saved + ? "Leave blank to keep the saved key." + : `Anyone with dictation access to ${environment.label} can bill this key.`} +

+ + ); + + return ( + { + if (!open && !busy) close(); + }} + > + + + + {remove + ? `Remove the ${company} key?` + : active + ? routed + ? "Manage OpenRouter dictation" + : `Manage ${company} key` + : `Use ${name} for dictation?`} + + + {remove + ? `Everyone using ${name} for dictation on ${environment.label} will lose access.` + : active && !routed + ? `Anyone with dictation access to ${environment.label} can use this API key.` + : cloudSpeechDisclosure(provider, environment.label)} + + + {!remove && ( + + {!connected ? ( +

Connect to {environment.label} with dictation access to continue.

+ ) : AsyncResult.isFailure(presence) && saved === undefined ? ( + + ) : routed && + AsyncResult.isSuccess(presence) && + presence.value && + saved === undefined ? ( +

Update {environment.label} to use OpenRouter dictation.

+ ) : saved === undefined ? ( +

Checking for an API key…

+ ) : routed ? ( +
{ + event.preventDefault(); + submit(); + }} + > +
+ + {listedModels ? ( + + ) : AsyncResult.isFailure(models) ? ( + + ) : ( +

+ Loading transcription models… +

+ )} +
+ {editingKey ? ( +
{keyField}
+ ) : !saved ? ( +

Ask an administrator on {environment.label} to add an API key.

+ ) : ( +

+ Uses the API key saved on {environment.label}. This choice applies only to this + device. +

+ )} +
+ ) : editingKey ? ( +
{ + event.preventDefault(); + submit(); + }} + > + {keyField} +
+ ) : !saved ? ( +

Ask an administrator on {environment.label} to add an API key.

+ ) : ( +

+ This choice applies only to this device. Choose Off to stop sharing. +

+ )} +
+ )} + {error && ( +

+ {error} +

+ )} + + {connected && canManage && saved && (active || routed) && !remove && ( +
+ +
+ )} + + {connected && + (remove ? ( + + ) : routed ? ( + saved !== undefined && (saved || canManage) ? ( + + ) : null + ) : editingKey ? ( + + ) : saved && !active ? ( + + ) : null)} +
+
+
+ ); +} diff --git a/apps/web/src/components/settings/SecretsSettings.logic.test.ts b/apps/web/src/components/settings/SecretsSettings.logic.test.ts index 1a0ff29e11cfb..1480844f58baf 100644 --- a/apps/web/src/components/settings/SecretsSettings.logic.test.ts +++ b/apps/web/src/components/settings/SecretsSettings.logic.test.ts @@ -31,6 +31,7 @@ const settings = Schema.decodeSync(ServerSettings)({ easDeviceHosts: [ { kind: "eas", id: "eas", label: "EAS Simulator", accessToken: personal("op://expo") }, ], + cloudSpeechCredentials: { openai: personal("op://voice"), xai: "" }, }); it.each([ @@ -42,6 +43,7 @@ it.each([ "GitHub · github.com", "https://hub.test · Management Key", "EAS Simulator · Access Token", + "Voice · OpenAI", ], ], ["work", ["Antigravity Work · OTHER_TOKEN", "GitHub · ghe.acme.test"]], diff --git a/apps/web/src/components/settings/SecretsSettings.logic.ts b/apps/web/src/components/settings/SecretsSettings.logic.ts index 3ae528aed162a..a85bc9e330143 100644 --- a/apps/web/src/components/settings/SecretsSettings.logic.ts +++ b/apps/web/src/components/settings/SecretsSettings.logic.ts @@ -1,4 +1,6 @@ import { + CLOUD_SPEECH_LABELS, + CloudSpeechProvider, EasDeviceHostConfig, isSecretReference, secretReferencesInSettings, @@ -12,7 +14,11 @@ import { deriveProviderSettingsFields, type SettingsFormDefinition } from "./Pro type SecretUseSettings = Pick< ServerSettings, - "providerInstances" | "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" + | "providerInstances" + | "sourceControlHosts" + | "usageLimitSources" + | "easDeviceHosts" + | "cloudSpeechCredentials" >; interface SecretProviderUse { @@ -86,5 +92,11 @@ export function secretProviderUses( ...settings.easDeviceHosts.flatMap((host) => uses({ settingsSchema: EasDeviceHostConfig }, `eas:${host.id}`, host.label, host), ), + ...CloudSpeechProvider.literals.flatMap((provider) => { + const credential = settings.cloudSpeechCredentials[provider]; + return isSecretReference(credential) && credential.provider === providerId + ? [{ id: `voice:${provider}`, label: `Voice · ${CLOUD_SPEECH_LABELS[provider]}` }] + : []; + }), ]; } diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index e62e7180fe073..c8e81874ce4b2 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -375,7 +375,7 @@ it.each([ ); await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite)); await expect - .element(screen.getByRole("combobox", { name: "Send microphone audio to" })) + .element(screen.getByRole("combobox", { name: "Transcribe with" })) .toHaveTextContent("Off"); await expect .element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" })) @@ -394,7 +394,7 @@ it("keeps a missing destination selected without claiming it cannot transcribe", , ); await expect - .element(screen.getByRole("combobox", { name: "Send microphone audio to" })) + .element(screen.getByRole("combobox", { name: "Transcribe with" })) .toHaveTextContent("Chosen environment (unavailable)"); await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument(); }); diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index fc3517831268e..5d2a8fbed227e 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -9,13 +9,17 @@ import { updateEnvironmentSpeechModelUnloadTimeout, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; +import { openRouterIcon } from "@t3tools/client-runtime/provider-clients"; import { AuthSettingsWriteScope, + CLOUD_SPEECH_LABELS, + CloudSpeechProvider, DEFAULT_SPEECH_POST_PROCESSING_PROMPT, type EnvironmentId, type EnvironmentSpeechModel, type EnvironmentSpeechStatus, type ProjectId, + ProviderDriverKind, type SpeechAcceleration, type SpeechCustomWords, type SpeechModelUnloadTimeout, @@ -23,11 +27,15 @@ import { } from "@t3tools/contracts"; import { createModelSelection } from "@t3tools/shared/model"; import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings"; +import { cloudSpeechLanguages } from "@t3tools/shared/speech"; import { AudioLinesIcon, CheckIcon, DownloadIcon, LanguagesIcon, + KeyRoundIcon, + MonitorIcon, + MicOffIcon, MicIcon, RefreshCwIcon, StarIcon, @@ -60,6 +68,8 @@ import { } from "../../speech/useEnvironmentSpeechInput"; import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton"; import { ProviderModelPicker } from "../chat/ProviderModelPicker"; +import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon"; +import { ProviderPackageIcon } from "../chat/ProviderPackageIcon"; import { TraitsPicker } from "../chat/TraitsPicker"; import { VoiceInputSetup } from "../chat/VoiceInputSetup"; import { Badge } from "../ui/badge"; @@ -79,11 +89,20 @@ import { MenuSeparator, MenuTrigger, } from "../ui/menu"; -import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select"; +import { + Select, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "../ui/select"; import { Switch } from "../ui/switch"; import { Textarea } from "../ui/textarea"; import { toastManager } from "../ui/toast"; import { MicrophoneTest } from "./MicrophoneTest"; +import { CloudVoiceConfiguration } from "./CloudVoiceConfiguration"; import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison"; import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect"; import { filterSpeechModels, type SpeechModelSort } from "./speechModelBrowser"; @@ -102,6 +121,17 @@ import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; const OFF = "off"; +function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { + if (provider === "openrouter") + return ; + return ( + + ); +} const SYSTEM_DEFAULT = "system-default"; const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`; const SHORTCUT_MODE_LABELS: Record = { @@ -1063,9 +1093,14 @@ export function VoiceSettingsPanel() { const settings = useClientSettings(); const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); + const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope); + const [pendingCloud, setPendingCloud] = useState<{ + provider: CloudSpeechProvider; + environmentId: EnvironmentId; + } | null>(null); const { environments } = useEnvironments(); const [microphones, setMicrophones] = useState([]); - const prepared = destination.prepared; + const prepared = destination.cloudProvider ? null : destination.prepared; const catalog = useVoiceModels(prepared); const refreshMicrophones = useStableCallback(async () => { @@ -1098,11 +1133,39 @@ export function VoiceSettingsPanel() { ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ?? "Selected microphone (unavailable)") : "System default"; - const destinationValue = destination.environmentId ?? OFF; + const environmentChoices = environments.map((environment) => ({ + environment, + value: JSON.stringify(["environment", environment.environmentId]), + })); + const cloudGroups = environments + .filter( + (environment) => environment.serverConfig?.environment.capabilities.cloudVoiceTranscription, + ) + .map((environment) => ({ + environment, + choices: CloudSpeechProvider.literals.map((provider) => ({ + environment, + provider, + value: JSON.stringify(["cloud", environment.environmentId, provider]), + })), + })); + const cloudChoices = cloudGroups.flatMap((group) => group.choices); + const destinationValue = destination.cloudProvider + ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) + : destination.environmentId + ? JSON.stringify(["environment", destination.environmentId]) + : OFF; + const cloudEnvironment = environments.find( + (env) => env.environmentId === pendingCloud?.environmentId, + ); + // OpenRouter users without settings access still manage their own model choice. + const manageLabel = + destination.cloudProvider === "openrouter" ? "Manage OpenRouter" : "Manage key"; const chosenEnvironment = environments.find( (environment) => environment.environmentId === destination.environmentId, ); const chosenUnsupported = + !destination.cloudProvider && chosenEnvironment?.serverConfig != null && chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true; const model = prepared && modelStatus?.supported ? modelStatus : null; @@ -1110,6 +1173,8 @@ export function VoiceSettingsPanel() { return ( - - Microphone audio is sent to this environment to transcribe, then discarded. Choose Off - to turn voice input off on this device. This choice is saved on this device only. - - } - control={ - - } + className="@container/destination px-3 py-3 sm:px-4" > +
+

+ Transcribe with + Use +

+
+ + {destination.cloudProvider && + chosenEnvironment && + (canManageKey || destination.cloudProvider === "openrouter") && ( + + { + if (destination.cloudProvider) + setPendingCloud({ + provider: destination.cloudProvider, + environmentId: chosenEnvironment.environmentId, + }); + }} + > + + + } + /> + {manageLabel} + + )} +
+
{destination.environmentId && chosenUnsupported ? (

- {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use - voice input. + {chosenEnvironment.label} can’t transcribe audio locally. Choose a different + destination.

) : null} -
+ - } - /> + {pendingCloud && cloudEnvironment && ( + setPendingCloud(null)} + /> + )} + {(model || destination.cloudProvider) && ( + + } + /> + )} {model?.supportsTranslation ? ( void updateClientSettings({ speechSilenceFiltering: checked }) @@ -1341,7 +1524,7 @@ export function VoiceSettingsPanel() { boolean; readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; readonly getMicrophoneId: () => string; @@ -26,25 +35,6 @@ export function createBrowserVoiceInputPlatform(input: { }) { return createBrowserPcmCapture(input, async (options) => { const recordingOptions = input.getTranscriptionOptions(); - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); - throwIfVoiceTranscriptionAborted(options.signal); - if (!status.supported) - throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); - const language = status.languages - ? effectiveSpeechLanguage( - { - languages: status.languages, - supportsLanguageDetection: status.supportsLanguageDetection ?? false, - }, - recordingOptions.speechLanguage, - ) - : "auto"; - const locale = - !status.supportsStreaming && - status.supportsTranslation && - recordingOptions.speechTranslateToEnglish - ? "en" - : language; const transcribeRecording = async ( uri: string, { signal: transcriptionSignal }: VoiceTranscriptionOptions, @@ -53,22 +43,70 @@ export function createBrowserVoiceInputPlatform(input: { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); const result = await runEnvironmentSpeech( - transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent), - { - signal: transcriptionSignal, - }, + input.cloudProvider + ? transcribeCloudPcm( + input.prepared, + input.cloudProvider, + pcm, + recordingOptions, + input.hasAudioConsent, + input.cloudModel, + ) + : transcribeEnvironmentPcm( + input.prepared, + pcm, + recordingOptions, + input.hasAudioConsent, + ), + { signal: transcriptionSignal }, ); throwIfVoiceTranscriptionAborted(transcriptionSignal); return result.text; } catch (cause) { throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (isVoiceTranscriptionError(cause)) throw cause; throw new VoiceTranscriptionError({ code: "transcription-failed", - message: "Voice transcription on this environment failed.", + message: "Voice transcription failed. Check the destination in Settings → Voice.", cause, }); } }; + if (input.cloudProvider) { + if (input.cloudProvider === "openrouter" && !input.cloudModel) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: "Choose an OpenRouter model in Settings → Voice.", + }); + await runEnvironmentSpeech( + requireCloudSpeechKey(input.prepared, input.cloudProvider), + options, + ); + throwIfVoiceTranscriptionAborted(options.signal); + return { + locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage), + transcribe: transcribeRecording, + }; + } + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); + throwIfVoiceTranscriptionAborted(options.signal); + if (!status.supported) + throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); + const language = status.languages + ? effectiveSpeechLanguage( + { + languages: status.languages, + supportsLanguageDetection: status.supportsLanguageDetection ?? false, + }, + recordingOptions.speechLanguage, + ) + : "auto"; + const locale = + !status.supportsStreaming && + status.supportsTranslation && + recordingOptions.speechTranslateToEnglish + ? "en" + : language; if (!status.supportsStreaming && !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.ts index bc6d1db747c78..f9635774d030e 100644 --- a/apps/web/src/speech/runEnvironmentSpeech.ts +++ b/apps/web/src/speech/runEnvironmentSpeech.ts @@ -6,14 +6,17 @@ import { connectionAtomRuntime } from "../connection/runtime"; import { appAtomRegistry } from "../rpc/atomRegistry"; // Relay (DPoP) environments authorize through services only the connection runtime provides. +export function inEnvironmentSpeechRuntime( + effect: Effect.Effect, +) { + return AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.flatMap((context) => Effect.provide(effect, context)), + ); +} + export function runEnvironmentSpeech( effect: Effect.Effect, options?: Effect.RunOptions, ): Promise { - return Effect.runPromise( - AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe( - Effect.flatMap((context) => Effect.provide(effect, context)), - ), - options, - ); + return Effect.runPromise(inEnvironmentSpeechRuntime(effect), options); } diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index 04534c3e0cfdc..8fb4722282747 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -6,7 +6,7 @@ import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import { DEFAULT_CLIENT_SETTINGS, DEFAULT_SERVER_SETTINGS, - type EnvironmentId, + EnvironmentId, type ProjectId, type SpeechCustomWords, type SpeechPostProcessingOptions, @@ -14,8 +14,14 @@ import { } from "@t3tools/contracts"; import { act, render } from "../../test/render"; +import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput"; -import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent"; +import { + chooseAudioDestination, + chooseTranscriptCleanup, + bindAudioGrant, + pendingCloudAudioChoice, +} from "./voiceConsent"; import { writeBrowserClientSettings } from "../clientPersistenceStorage"; const mocks = vi.hoisted(() => ({ @@ -23,6 +29,7 @@ const mocks = vi.hoisted(() => ({ settings: {} as Record, connected: new Set(), voiceCapable: new Set(), + cloudVoiceCapable: new Set(), canDictate: new Set(), prepared: new Map(), primaryEnvironmentId: "primary-env", @@ -63,7 +70,10 @@ vi.mock("../state/environments", () => ({ environmentId, serverConfig: { environment: { - capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) }, + capabilities: { + voiceTranscription: mocks.voiceCapable.has(environmentId), + cloudVoiceTranscription: mocks.cloudVoiceCapable.has(environmentId), + }, }, }, } @@ -124,12 +134,10 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ return { text: `Cleaned ${transcript}` }; }), })); -// Capture and the wire live behind the platform; its `prepared` is where audio would go. vi.mock("./browserVoiceInput", () => ({ - createBrowserVoiceInputPlatform: (input: { - prepared: PreparedConnection; - getTranscriptionOptions?: () => SpeechTranscriptionOptions; - }) => { + createBrowserVoiceInputPlatform: ( + input: Parameters[0], + ) => { const environmentId = input.prepared.environmentId; mocks.platforms.push(environmentId); return { @@ -143,12 +151,15 @@ vi.mock("./browserVoiceInput", () => ({ }, transcriber: { prepare: async () => { - const options = input.getTranscriptionOptions?.(); - if (options) mocks.recordingOptions.push(options); + mocks.recordingOptions.push(input.getTranscriptionOptions()); return { locale: "en", transcribe: async () => { - mocks.transcribeCalls.push(environmentId); + mocks.transcribeCalls.push( + input.cloudProvider + ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` + : environmentId, + ); const next = mocks.transcripts.shift() ?? "dictated"; if (next instanceof Error) throw next; return next; @@ -171,6 +182,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.primaryEnvironmentId = "primary-env"; mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]); + mocks.cloudVoiceCapable.clear(); mocks.canDictate = new Set(mocks.voiceCapable); mocks.prepared.clear(); mocks.statusRequests = []; @@ -367,7 +379,15 @@ it("revokes across windows even before the storage event and survives stale pref const grant = JSON.parse(storage.getItem(key)!) as { id: string }; const canSend = bindAudioGrant(grant.id); expect(canSend()).toBe(true); + const saved = pendingCloudAudioChoice({ + kind: "cloud", + provider: "openai", + environmentId: EnvironmentId.make("env-a"), + }); + expect(canSend()).toBe(true); storage.removeItem(key); + saved(); + expect(storage.getItem(key)).toBeNull(); expect(canSend()).toBe(false); writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false }); await vi.waitFor(() => expect(mocks.voice.available).toBe(false)); @@ -381,15 +401,89 @@ it("revokes across windows even before the storage event and survives stale pref } }); -it("cleans the transcript only after cleanup is separately enabled", async () => { +it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => { + setup(); + mocks.cloudVoiceCapable.add("env-b"); + mocks.modelState = "missing-model"; + window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json"); + const save = pendingCloudAudioChoice({ + kind: "cloud", + environmentId: EnvironmentId.make("env-b"), + provider: "xai", + }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + await act(() => mocks.voice.start()); + expectNoAudioSent(); + await act(save); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual(["xai via env-b"]); + expect(mocks.statusRequests).toEqual([]); +}); + +it("sends audio to OpenRouter only under a device grant naming the exact model", async () => { + setup(); + mocks.cloudVoiceCapable.add("env-b"); + const key = "t3code:voice-audio-grant:v1"; + const destination = { kind: "cloud", environmentId: "env-b", provider: "openrouter" }; + window.localStorage.setItem(key, JSON.stringify({ id: "no-model", destination })); + { + await using _screen = await render(); + await act(() => mocks.voice.start()); + expectNoAudioSent(); + } + const model = "acme/transcribe-1"; + window.localStorage.setItem( + key, + JSON.stringify({ id: "model", destination: { ...destination, model } }), + ); + await using _screen = await render(); + await act(() => mocks.voice.start()); + await act(() => mocks.voice.stop()); + expect(mocks.transcribeCalls).toEqual([`openrouter:${model} via env-b`]); + expect(mocks.statusRequests).toEqual([]); +}); + +it("uses neither destination when the cloud save marker cannot be read", async () => { + setup({ destination: "env-a" }); + const key = "t3code:voice-audio-grant:v1"; + const previous = JSON.parse(window.localStorage.getItem(key)!) as { id: string }; + const canSendToPrevious = bindAudioGrant(previous.id); + const save = pendingCloudAudioChoice({ + kind: "cloud", + environmentId: EnvironmentId.make("env-b"), + provider: "openai", + }); + save(); + const getItem = Storage.prototype.getItem; + using _blocked = vi.spyOn(Storage.prototype, "getItem").mockImplementation(function ( + this: Storage, + k, + ) { + if (k.startsWith(`${key}:saved:`)) + throw new DOMException("Storage is blocked", "SecurityError"); + return getItem.call(this, k); + }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + expect(canSendToPrevious()).toBe(false); + await act(() => mocks.voice.start()); + expectNoAudioSent(); +}); + +it("cleans text on a cloud-only draft environment only while cleanup is separately enabled", async () => { setup({ destination: "env-a" }); + mocks.voiceCapable.delete(THREAD_ENV); + mocks.cloudVoiceCapable.add(THREAD_ENV); chooseTranscriptCleanup(true); const nearby = "x".repeat(1_024); mocks.draft = "unrelated private paragraph " + nearby; await using screen = await render(); await act(() => mocks.voice.start()); await act(() => mocks.voice.stop()); - expect(mocks.cleanupCalls).toHaveLength(1); + expect(mocks.transcribeCalls).toEqual(["env-a"]); + expect(mocks.cleanupCalls).toEqual([THREAD_ENV]); expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({ text: nearby, selection: { start: nearby.length, end: nearby.length }, @@ -414,6 +508,7 @@ it.each([ "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized", async ({ cleanup, authorized }) => { setup({ destination: "env-a" }); + mocks.cloudVoiceCapable.add(THREAD_ENV); chooseTranscriptCleanup(true); mocks.settings.speechCorrectionWord = "sorry"; if (!authorized) mocks.canDictate.delete(THREAD_ENV); diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 0d1853a5dcc89..279bfcfa8e88d 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -11,7 +11,7 @@ import { type VoiceInputState, } from "@t3tools/client-runtime/voice-input"; import type { PreparedConnection } from "@t3tools/client-runtime/connection"; -import { AuthOrchestrationOperateScope } from "@t3tools/contracts"; +import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts"; import type { EnvironmentId, EnvironmentSpeechModel, @@ -109,11 +109,22 @@ export function useVoiceTranscriptionDestination() { const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); - const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true; + const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; + const provider = cloud?.provider; + const model = cloud?.provider === "openrouter" ? cloud.model : undefined; + const supported = + environment?.serverConfig?.environment.capabilities[ + provider ? "cloudVoiceTranscription" : "voiceTranscription" + ] === true; return { environmentId, + ...(provider ? { cloudProvider: provider } : {}), + ...(model ? { cloudModel: model } : {}), grantId: grant?.id ?? null, - label: environment?.label ?? null, + label: + provider && environment + ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` + : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, }; } @@ -122,12 +133,15 @@ export function useEnvironmentSpeechInput(input: HookInput) { const destination = useVoiceTranscriptionDestination(); const prepared = destination.prepared; const grantId = destination.grantId; - const originatingEnvironment = useEnvironment(input.environmentId); + const cloudProvider = destination.cloudProvider; + const cloudModel = destination.cloudModel; + const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment + .capabilities; const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope); const cleanupPrepared = canCleanUp && - originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true + (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription) ? originatingConnection : null; const clientSettings = useClientSettings(); @@ -231,6 +245,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { }; const platform = createBrowserVoiceInputPlatform({ prepared, + cloudProvider, + cloudModel, hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -344,11 +360,11 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId]); + }, [prepared, grantId, cloudProvider, cloudModel]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { - if (!prepared) return; + if (!prepared || cloudProvider) return; let disposed = false; void probeSpeechStatus(prepared) .then((value) => { @@ -360,8 +376,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { return () => { disposed = true; }; - }, [prepared]); - const currentStatus = status?.prepared === prepared ? status.value : null; + }, [prepared, cloudProvider]); + const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null; // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download useEffect(() => { @@ -425,6 +441,11 @@ export function useEnvironmentSpeechInput(input: HookInput) { const controller = controllerRef.current; const owner = latestRef.current.input.ownerKey; if (latestRef.current.input.disabled || !controller || !prepared) return; + if (cloudProvider) { + setLevel(0); + await controller.start(); + return; + } const stillCurrent = () => request === startRequestRef.current && controllerRef.current === controller && @@ -511,8 +532,8 @@ export function useEnvironmentSpeechInput(input: HookInput) { return { available: prepared !== null && - status?.prepared === prepared && - status.value?.supported !== false && + (cloudProvider !== undefined || + (status?.prepared === prepared && status.value?.supported !== false)) && typeof navigator !== "undefined" && Boolean(navigator.mediaDevices?.getUserMedia) && typeof AudioWorkletNode !== "undefined", diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 81da7cd6adb3d..726717a583ebe 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -1,8 +1,11 @@ import { VoiceTranscriptionDestination } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import { useSyncExternalStore } from "react"; import { randomUUID } from "../lib/utils"; import { + getLocalStorageItem, readLocalStorageValue, + subscribeLocalStorageKey, useLocalStorage, writeLocalStorageValue, } from "../hooks/useLocalStorage"; @@ -10,33 +13,91 @@ import { // Separate keys: a stale window saving ordinary preferences must not restore either grant. const AUDIO_KEY = "t3code:voice-audio-grant:v1"; const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1"; -const AudioGrant = Schema.NullOr( - Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }), +const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }); +const AudioGrant = Schema.NullOr(Grant); +const sameGrant = Schema.toEquivalence(AudioGrant); +const AudioChoice = Schema.NullOr( + Schema.Union([Grant, Schema.Struct({ pending: Grant, previous: AudioGrant })]), ); +type AudioChoice = typeof AudioChoice.Type; +const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`; +const pendingGrant = (choice: AudioChoice) => + choice && "pending" in choice ? choice.pending : null; const CleanupGrant = Schema.NullOr(Schema.String); -const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant)); +const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice)); +const readChoice = () => readLocalStorageValue(AUDIO_KEY, null, AudioChoice); + +function readSaveMarker(id: string | undefined) { + try { + return id ? window.localStorage.getItem(savedKey(id)) : null; + } catch { + return undefined; + } +} + +function activeGrant(choice: AudioChoice, marker: string | null | undefined) { + if (!choice || !("pending" in choice)) return choice; + // Only a confirmed missing marker means the previous grant still applies. + if (marker === null) return choice.previous; + return marker === "true" ? choice.pending : null; +} export function useAudioGrant() { - return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0]; + const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice); + const id = pendingGrant(choice)?.id; + const marker = useSyncExternalStore( + (notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}), + () => readSaveMarker(id), + () => undefined, + ); + return activeGrant(choice, marker); +} + +function replaceChoice(choice: AudioChoice, previous = readChoice()) { + const old = pendingGrant(previous); + writeLocalStorageValue(AUDIO_KEY, choice, null, AudioChoice); + if (old && pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id !== old.id) + window.localStorage.removeItem(savedKey(old.id)); } export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) { - writeLocalStorageValue( - AUDIO_KEY, - destination && { id: randomUUID(), destination }, - null, - AudioGrant, + replaceChoice(destination && { id: randomUUID(), destination }); +} + +export function pendingCloudAudioChoice( + destination: Extract, +) { + const choice = readChoice(); + const pending = { id: randomUUID(), destination }; + replaceChoice( + { pending, previous: activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)) }, + choice, ); + // A late response can acknowledge only its own choice, never write the destination another tab picked. + return () => { + if (pendingGrant(readChoice())?.id === pending.id) + writeLocalStorageValue(savedKey(pending.id), true, false, Schema.Boolean); + }; } // Bind this recording to this act of consent, even if the user later reselects the same destination. export function bindAudioGrant(id: string) { try { - const raw = window.localStorage.getItem(AUDIO_KEY); - if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false; + let cachedRaw: string | null = null; + let choice: AudioChoice = null; + const current = () => { + const raw = window.localStorage.getItem(AUDIO_KEY); + if (raw !== cachedRaw) { + choice = raw ? decodeChoice(raw) : null; + cachedRaw = raw; + } + return activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)); + }; + const bound = current(); + if (bound?.id !== id) return () => false; return () => { try { - return window.localStorage.getItem(AUDIO_KEY) === raw; + return sameGrant(current(), bound); } catch { return false; } diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index fb84458a03a67..2bf73af0ab36d 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -2,8 +2,9 @@ Transcription edits a composer draft. It does not submit an agent turn. Audio is temporary client input, and only normal message submission sends the resulting -text. Mobile defaults to on-device transcription; any client that sends audio off -the device needs a grant for one environment. +text to the agent. Mobile defaults to on-device transcription. Sending audio off +the device requires an explicit grant naming one environment and, for cloud +transcription, one provider. The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts) owns the operation while the client supplies capture and transcription. Preparation @@ -15,28 +16,42 @@ capture and `finish` replaces the file-based `transcribe`. ## Where audio may go -Only an explicit audio grant authorizes off-device transcription. On mobile the -grant is stored in device-bound secure storage, outside preference backups. The -recorder is the same for both destinations; only a finished recording can be -uploaded, and its grant is checked again after native decoding and before dispatch. +On mobile, on-device transcription needs no audio grant. Environment and cloud +transcription share a recorder and upload only finished recordings. The client +keeps the grant in device-bound secure storage, outside preference backups, and +rechecks it after native decoding and before upload. + The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another -window saving unrelated settings cannot resurrect a revoked grant. Each new -choice gets a distinct grant, even if the destination was previously selected. The -[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a -recording controller to the prepared connection of that destination and to -nothing else. When the destination changes, disconnects, or stops advertising the -`voiceTranscription` capability, the controller is disposed: a recording in -progress is discarded, a failed recording kept for retry is deleted, and nothing -falls back to another environment. Every sender in the web app, including the -settings page's transcription test, goes through that hook. The microphone loopback +window saving unrelated settings cannot resurrect a revoked grant. Each new choice +gets a distinct grant, even if the destination was previously selected. The +[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds the +controller to that grant and the selected environment's connection. It requires +`voiceTranscription` to use the environment's model or `cloudVoiceTranscription` +to forward audio to the selected provider. Changing the grant or losing the +connection, permission, or required capability discards both the active recording +and any recording retained for retry. It never chooses a fallback destination. +The settings page's transcription test uses the same hook. The microphone loopback test captures separately for local playback and sends nothing. The transport refuses redirects so credentials and audio cannot follow one to another host. Transcript cleanup requires its own grant, off by default. It sends text, never audio, to the text generation provider of the draft's -own environment, because that is where the provider credentials live. A later -backend adds its own destination kind, disclosure, and transcriber; capture and -the controller stay as they are. +own environment, because that is where the provider credentials live. + +A cloud destination binds both the forwarding environment and the named provider. +It uses a separate, batch-only endpoint: changing a server key cannot upgrade +an environment-local grant, and a cloud recording never uses the streaming +socket or falls back to another provider or the environment's native model. + +OpenRouter routes to many upstreams, so its grant also names the exact model and +the model travels with every upload as a query parameter. The server accepts only +ids present in OpenRouter's public ZDR-eligible transcription catalog, +which it caches for ten minutes after a successful read; +router aliases such as `openrouter/auto` or `:nitro` fail schema validation before +that. Requests set `provider.zdr` and carry no `models` fallback list. OpenRouter +applies the account's safety-retention policy when choosing an endpoint. The catalog +holds no credential or consent state, which is why it may be cached while key +presence is read fresh on every request. ## Server side diff --git a/docs/user/composer.md b/docs/user/composer.md index 13f451e2caa73..22b499ebe7b0c 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -203,6 +203,8 @@ choose the environment, and confirm. Finished recordings go only to that environ even when a draft belongs to another one. This also works on phones without local speech recognition. If the chosen environment disconnects, dictation stops; it never switches hosts. Choose **This device** to revoke access and discard pending audio. +For [cloud dictation](./voice-input.md#choose-where-audio-goes), open **Cloud providers** +under the environment in **Settings → Voice** and confirm which provider may receive recordings. On Android, keep T3 Code open while recording and transcribing. On-device dictation may need a network connection to download a speech model the diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index e204b28660626..877a1a38a20f0 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -12,9 +12,9 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o ## Choose where audio goes -On web and desktop, transcription runs on one of your T3 Code environments. +On web and desktop, you choose where transcription runs. Nothing is sent until you choose one: open **Settings → Voice** and set -**Send microphone audio to** the environment that should transcribe for you. +**Transcribe with** to the environment that should transcribe for you. Microphone audio recorded on this device is sent to that environment over your authenticated connection, transcribed there, and discarded. The choice is saved on this device only, so every computer and browser you use asks for it once. @@ -23,10 +23,33 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. +For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment +you want to use. If it needs a key, enter one and choose **Save and enable**. +You can also choose a [secret provider](secrets.md) instead of **Value** and enter a +reference. Unlock the vault on that environment before dictating. Anyone with +dictation access to the environment can use the key, but only this device opts in. +If a key is already saved, just confirm the provider. After +you stop, the environment forwards your recording and any vocabulary hints to +the selected provider, which bills its API account. No local model download or chat +subscription is needed. Switch back to **This device** on mobile or **Off** on +web and desktop to stop sharing. Removing the key stops that provider for everyone +on the environment. + +**OpenRouter** works the same way but has no default model: pick one from its live +speech-to-text catalog when you enable it, or later under **Manage OpenRouter**. +The model choice is yours alone and needs no settings access, so anyone with +dictation access can pick a different model without touching the saved key. +OpenRouter chooses who runs the model. T3 requests zero data retention; your +OpenRouter account can also allow safety retention. If no endpoint qualifies, +dictation fails instead of switching models. +Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied +after transcription. If OpenRouter stops listing your model, dictation stops +until you choose another. + ## Set up a model -The first time you dictate, T3 Code offers to download the recommended English -model to the chosen environment. In **Settings → Voice → Transcription**, select +If you choose environment transcription, T3 Code offers to download the recommended +English model to that environment. In **Settings → Voice → Transcription**, select the languages you dictate in to compare models that support all of them. Accuracy shows how often the model got words or characters wrong in each language; speed is the time to transcribe 30 seconds on a reference GPU. Choose diff --git a/packages/client-runtime/src/providerClients.ts b/packages/client-runtime/src/providerClients.ts index e23200653b963..3759cecbc7c7d 100644 --- a/packages/client-runtime/src/providerClients.ts +++ b/packages/client-runtime/src/providerClients.ts @@ -6,7 +6,7 @@ import { ProviderDriverKind, } from "@t3tools/contracts"; import { acpRegistryClient } from "@t3tools/provider-acp-registry/client"; -import { makeProviderClientRegistry } from "@t3tools/provider-core/client"; +import { makeProviderClientRegistry, type ProviderIcon } from "@t3tools/provider-core/client"; import { modelShortName } from "@t3tools/provider-core/modelShortName"; import { cursorClient } from "@t3tools/provider-cursor/client"; import { fxClient } from "@t3tools/provider-fx/client"; @@ -16,6 +16,17 @@ import { museClient } from "@t3tools/provider-muse/client"; import { openCodeClient } from "@t3tools/provider-opencode/client"; import { piClient } from "@t3tools/provider-pi/client"; +// OpenRouter's Ink and Cloud glyphs from https://openrouter.ai/brand. +export const openRouterIcon: ProviderIcon = { + viewBox: "0 0 1024 730", + fill: { light: "#03080A", dark: "#FCFCFE" }, + paths: [ + { + d: "M795.893 0C915.776 0 1012.95 97.9963 1012.95 218.88C1012.95 339.764 915.776 437.76 795.893 437.76L1011.2 654.869C1038.55 682.447 1019.18 729.6 980.504 729.6H361.77C161.97 729.6 0 566.273 0 364.8C0 163.327 161.97 0 361.77 0L795.893 0ZM361.77 145.92C241.89 145.92 144.708 243.916 144.708 364.8C144.708 485.684 241.89 583.68 361.77 583.68C481.649 583.68 578.831 485.684 578.831 364.8C578.831 243.916 481.649 145.92 361.77 145.92Z", + }, + ], +}; + export const providerClients = makeProviderClientRegistry([ { driverKind: ProviderDriverKind.make("codex"), diff --git a/packages/client-runtime/src/secretRequest.ts b/packages/client-runtime/src/secretRequest.ts index d9612411db09d..26a32e735bcfb 100644 --- a/packages/client-runtime/src/secretRequest.ts +++ b/packages/client-runtime/src/secretRequest.ts @@ -95,6 +95,12 @@ export function secretRequestSources( providers: ServerSettings["secretProviders"], ): ReadonlyArray { if (config?.secretRequestReferences !== true) return NO_SOURCES; + return configuredSecretSources(providers); +} + +export function configuredSecretSources( + providers: ServerSettings["secretProviders"], +): ReadonlyArray { return (Object.keys(providers) as Array).flatMap((id) => { const provider = providers[id]; const definition = diff --git a/packages/client-runtime/src/voice-input/environment.ts b/packages/client-runtime/src/voice-input/environment.ts index 21166d8b9a084..52fa5a0d1fe5b 100644 --- a/packages/client-runtime/src/voice-input/environment.ts +++ b/packages/client-runtime/src/voice-input/environment.ts @@ -1,8 +1,12 @@ import { encodeSpeechPcmRequest } from "@t3tools/shared/speech"; import * as Effect from "effect/Effect"; import * as Redacted from "effect/Redacted"; -import { SPEECH_STREAM_PATH } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts"; import type { + CloudSpeechModelId, + CloudSpeechProvider, + SettingsSecretValue, SpeechAcceleration, SpeechModelUnloadTimeout, SpeechTranscriptionOptions, @@ -17,8 +21,10 @@ import { makeEnvironmentHttpApiUrlBuilder, } from "../rpc/http.ts"; import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts"; +import { VoiceTranscriptionError } from "./transcription.ts"; const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000; +const isCloudSpeechError = Schema.is(CloudSpeechError); type EnvironmentApiClient = Effect.Success< ReturnType> >; @@ -65,6 +71,94 @@ export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) => run: ({ client, headers }) => client.status({ headers }), }); +export const getCloudSpeechKeys = (prepared: PreparedConnection) => + request({ + group: "voice", + prepared, + method: "GET", + path: (urls) => urls.cloudKeys(), + run: ({ client, headers }) => client.cloudKeys({ headers }), + }); + +export const getCloudSpeechModels = (prepared: PreparedConnection) => + request({ + group: "voice", + prepared, + method: "GET", + path: (urls) => urls.cloudModels(), + run: ({ client, headers }) => client.cloudModels({ headers }), + }); + +export const requireCloudSpeechKey = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, +) => + getCloudSpeechKeys(prepared).pipe( + Effect.flatMap((keys) => + keys[provider] + ? Effect.void + : Effect.fail( + new VoiceTranscriptionError({ + code: "unavailable", + message: new CloudSpeechError({ provider, reason: "missing-key" }).message, + }), + ), + ), + ); + +export const setCloudSpeechKey = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, + key: SettingsSecretValue | null, +) => + request({ + group: "voice", + prepared, + method: "POST", + path: (urls) => urls.setCloudKey({ params: { provider } }), + run: ({ client, headers }) => + client.setCloudKey({ + headers, + params: { provider }, + payload: { key: typeof key === "string" ? Redacted.make(key) : key }, + }), + }); + +export const transcribeCloudPcm = ( + prepared: PreparedConnection, + provider: CloudSpeechProvider, + pcm: Uint8Array, + options: SpeechTranscriptionOptions, + canSend: () => boolean, + model?: CloudSpeechModelId, +) => { + const query = model === undefined ? {} : { model }; + return request({ + group: "voice", + prepared, + method: "POST", + canSend, + timeoutMs: 130_000, + path: (urls) => urls.transcribeCloud({ params: { provider }, query }), + run: ({ client, headers }) => + client.transcribeCloud({ + headers, + params: { provider }, + query, + payload: encodeSpeechPcmRequest(pcm, options), + }), + }).pipe( + Effect.mapError((error) => + error._tag === "RemoteEnvironmentAuthFetchError" && isCloudSpeechError(error.cause) + ? new VoiceTranscriptionError({ + code: "transcription-failed", + message: error.cause.message, + }) + : error, + ), + ); +}; + export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => { let endpoint = prepared.httpBaseUrl; return request({ diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts index 06a5399d6813c..5843267800f67 100644 --- a/packages/client-runtime/src/voice-input/index.ts +++ b/packages/client-runtime/src/voice-input/index.ts @@ -14,6 +14,11 @@ export { cancelEnvironmentSpeechModelDownload, downloadEnvironmentSpeechModel, getEnvironmentSpeechModels, + getCloudSpeechKeys, + getCloudSpeechModels, + requireCloudSpeechKey, + setCloudSpeechKey, + transcribeCloudPcm, getEnvironmentSpeechStatus, getEnvironmentSpeechStreamUrl, postProcessEnvironmentTranscript, diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 627b9529a6cc5..96b4537478ecc 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -174,6 +174,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ scheduledTaskThreadModes: Schema.optionalKey(Schema.Boolean), /** Server can transcribe 16 kHz mono PCM audio on the environment machine. */ voiceTranscription: Schema.optionalKey(Schema.Boolean), + cloudVoiceTranscription: Schema.optionalKey(Schema.Boolean), }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 8dd67640a0bc5..46dfcde197ac2 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -49,6 +49,12 @@ import { TrimmedNonEmptyString, } from "./baseSchemas.ts"; import { + CloudSpeechApiKey, + CloudSpeechError, + CloudSpeechKeys, + CloudSpeechModelId, + CloudSpeechModels, + CloudSpeechProvider, EnvironmentSpeechAccelerationRequest, EnvironmentSpeechModelRequest, EnvironmentSpeechModels, @@ -58,6 +64,7 @@ import { EnvironmentSpeechStatus, EnvironmentSpeechTranscriptionResult, } from "./speech.ts"; +import { SecretReference } from "./secretReference.ts"; import { OrchestrationV2ShellSnapshot, OrchestrationV2ThreadBoundedSnapshot, @@ -775,6 +782,50 @@ export class EnvironmentVoiceBodyLimit extends HttpApiMiddleware.Service()( + "CloudSpeechError", + { + provider: CloudSpeechProvider, + reason: Schema.Literals([ + "missing-key", + "key-changed", + "unreadable-key", + "rejected-key", + "rate-limit", + "model-unavailable", + "unavailable", + ]), + }, + { httpApiStatus: 422 }, +) { + override get message() { + const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider]; + switch (this.reason) { + case "missing-key": + return `Add your ${name} API key in Settings → Voice.`; + case "key-changed": + return `The ${name} key changed before your recording was sent. Try again.`; + case "unreadable-key": + return `Couldn’t read the ${name} API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent to ${name}.`; + case "rejected-key": + return `${name} rejected this API key. Check its permissions and billing in Settings → Voice.`; + case "rate-limit": + return `${name} reached a usage limit. Check the account's quota or try again later.`; + case "model-unavailable": + return `The chosen model isn’t available on ${name}. Choose another model in Settings → Voice.`; + case "unavailable": + return `${name} couldn’t transcribe this recording. Try again later.`; + } + } +} export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160)); export type SpeechModelId = typeof SpeechModelId.Type; diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts index fcf74f10187b8..0fb077452bb3a 100644 --- a/packages/shared/src/speech.ts +++ b/packages/shared/src/speech.ts @@ -1,4 +1,6 @@ import { + CLOUD_SPEECH_PROVIDER_NAMES, + type CloudSpeechProvider, SPEECH_MAX_OPTIONS_BYTES, type SpeechCustomWords, SpeechTranscriptionOptions, @@ -8,6 +10,21 @@ import * as Schema from "effect/Schema"; const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); +export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) { + return provider === "openrouter" + ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.` + : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`; +} + +// Only offer explicit hints understood by all cloud APIs; other languages can still be detected. +export const cloudSpeechLanguages = { + supportsLanguageDetection: true, + languages: + "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split( + " ", + ), +}; + /** Resolve the language the model can produce from the requested language. */ export function effectiveSpeechLanguage( model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean },