diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx index ab1c0187c2806..e62e7180fe073 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx @@ -1,4 +1,5 @@ -import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection"; +import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection"; +import type * as Effect from "effect/Effect"; import { EnvironmentId, type EnvironmentSpeechModel, @@ -19,6 +20,53 @@ import { SpeechLanguageSelect } from "./SpeechLanguageSelect"; import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true }); +// The real connection runtime authorizes relay requests with a signed-in cloud session. +vi.mock("../../connection/runtime", async () => { + const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([ + import("@t3tools/client-runtime/relay"), + import("@t3tools/client-runtime/rpc"), + import("effect/reactivity"), + import("effect/Context"), + import("effect/Effect"), + import("effect/Layer"), + ]); + type AuthorizedHttp = + import("@t3tools/client-runtime/authorization").AuthorizedRemoteHttpEnvironment; + // Services resolve by key, and client-runtime does not export this one. + class RemoteEnvironmentAuthorization extends Context.Service< + RemoteEnvironmentAuthorization, + { + readonly authorizeDpopHttp: (input: { + readonly expectedEnvironmentId: AuthorizedHttp["environmentId"]; + }) => Effect.Effect; + } + // @effect-diagnostics-next-line deterministicKeys:off + >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {} + return { + connectionAtomRuntime: Atom.runtime( + Layer.mergeAll( + rpc.layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)), + Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, { + thumbprint: Effect.succeed("thumbprint"), + createProof: () => Effect.succeed("proof"), + }), + Layer.succeed(RemoteEnvironmentAuthorization, { + authorizeDpopHttp: ({ expectedEnvironmentId }) => + Effect.succeed({ + environmentId: expectedEnvironmentId, + label: "MacBook Pro", + httpBaseUrl: window.location.origin, + httpAuthorization: { + _tag: "Dpop", + accessToken: "relay-access-token", + expiresAtEpochMs: 4_102_444_800_000, + }, + }), + }), + ), + ), + }; +}); it("preserves the model test transcript when its model is filtered out", async () => { const target = new PrimaryConnectionTarget({ @@ -157,6 +205,81 @@ it("preserves the model test transcript when its model is filtered out", async ( ); }); +it("loads the models of a relay-connected destination", async () => { + const target = new RelayConnectionTarget({ + environmentId: EnvironmentId.make("voice-relay"), + label: "MacBook Pro", + }); + const prepared = { + environmentId: target.environmentId, + label: target.label, + httpBaseUrl: window.location.origin, + socketUrl: "ws://localhost", + httpAuthorization: { + _tag: "Dpop" as const, + accessToken: "stale-access-token", + expiresAtEpochMs: 4_102_444_800_000, + }, + target, + }; + const model: EnvironmentSpeechModel = { + id: "tiny", + name: "Tiny", + description: "", + size: 100, + languages: ["en"], + accuracy: 1, + speed: 1, + recommended: false, + supportsStreaming: false, + supportsTranslation: false, + supportsLanguageDetection: false, + active: false, + state: "downloadable", + }; + const status = { + supported: true, + state: "missing-model", + modelId: model.id, + model: model.name, + size: model.size, + supportsStreaming: false, + supportsTranslation: false, + acceleration: "cpu", + modelUnloadTimeout: "never", + gpuDevices: [], + } satisfies EnvironmentSpeechStatus; + const authorizations: Array = []; + const fetch = globalThis.fetch; + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { + const url = new URL(input instanceof Request ? input.url : input, window.location.href); + if (url.pathname === "/api/voice/status") { + authorizations.push(new Headers(init?.headers).get("authorization")); + return Promise.resolve(Response.json(status)); + } + if (url.pathname === "/api/voice/models") + return Promise.resolve(Response.json({ models: [model] })); + return fetch(input, init); + }); + using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); + // Restored even when an assertion fails, unlike `withImplementation`. + using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({ + environmentId: target.environmentId, + label: target.label, + grantId: "test", + prepared, + })); + await using screen = await render( + {}}> + + , + ); + await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0)); + expect(notify).not.toHaveBeenCalled(); + expect(authorizations).toEqual(["DPoP relay-access-token"]); + await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible(); +}); + it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => { const target = new PrimaryConnectionTarget({ environmentId: EnvironmentId.make("voice-progress"), diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index 596e3ea295ad7..7ca234047a3df 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -44,7 +44,7 @@ import { useCleanupGrant, } from "../../speech/voiceConsent"; import { updateClientSettings, useClientSettings } from "../../hooks/useSettings"; -import { runtime } from "../../lib/runtime"; +import { runEnvironmentSpeech } from "../../speech/runEnvironmentSpeech"; import { getCustomModelOptionsByInstance, resolveAppModelSelectionState, @@ -533,8 +533,8 @@ function useVoiceModels(prepared: PreparedConnection | null) { const revision = ++refreshRevision.current; try { const [nextStatus, nextModels] = await Promise.all([ - runtime.runPromise(getEnvironmentSpeechStatus(connection)), - runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")), + runEnvironmentSpeech(getEnvironmentSpeechStatus(connection)), + runEnvironmentSpeech(getEnvironmentSpeechModels(connection, mode === "foreground")), ]); if (current !== epoch.current || revision !== refreshRevision.current) return; setStatus(nextStatus); @@ -640,7 +640,7 @@ function AdvancedSpeechSettings({ onValueChange={(value) => { if (value) void run("Could not change transcription hardware", () => - runtime.runPromise( + runEnvironmentSpeech( updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration), ), ); @@ -679,7 +679,7 @@ function AdvancedSpeechSettings({ onValueChange={(value) => { if (value) void run("Could not change when the model unloads", () => - runtime.runPromise( + runEnvironmentSpeech( updateEnvironmentSpeechModelUnloadTimeout( prepared, value as SpeechModelUnloadTimeout, @@ -838,22 +838,22 @@ function SpeechModelBrowser(props: { void run( "Could not download the model", () => - runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)), + runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, model.id)), model.id, ), onSelect: () => void run("Could not select the model", () => - runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)), + runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, model.id)), ), onCancel: () => void run("Could not cancel the download", () => - runtime.runPromise( + runEnvironmentSpeech( cancelEnvironmentSpeechModelDownload(prepared, model.id), ), ), onDelete: () => void run("Could not delete the model", () => - runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)), + runEnvironmentSpeech(removeEnvironmentSpeechModel(prepared, model.id)), ), }; return model.active ? ( diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index 1d04f96bba0e2..5690773a61025 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -11,7 +11,7 @@ import { import type { PreparedConnection } from "@t3tools/client-runtime/connection"; import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts"; -import { runtime } from "../lib/runtime"; +import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; export function createBrowserVoiceInputPlatform(input: { @@ -26,7 +26,7 @@ export function createBrowserVoiceInputPlatform(input: { }) { return createBrowserPcmCapture(input, async (options) => { const recordingOptions = input.getTranscriptionOptions(); - const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options); + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options); throwIfVoiceTranscriptionAborted(options.signal); if (!status.supported) throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason }); @@ -52,7 +52,7 @@ export function createBrowserVoiceInputPlatform(input: { try { const response = await fetch(uri, { signal: transcriptionSignal }); const pcm = new Uint8Array(await response.arrayBuffer()); - const result = await runtime.runPromise( + const result = await runEnvironmentSpeech( transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent), { signal: transcriptionSignal, @@ -72,7 +72,7 @@ export function createBrowserVoiceInputPlatform(input: { if (!status.supportsStreaming && !status.supportsBufferedStreaming) { return { locale, transcribe: transcribeRecording }; } - const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options); + const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options); return { locale, transcribe: transcribeRecording, diff --git a/apps/web/src/speech/runEnvironmentSpeech.ts b/apps/web/src/speech/runEnvironmentSpeech.ts new file mode 100644 index 0000000000000..bc6d1db747c78 --- /dev/null +++ b/apps/web/src/speech/runEnvironmentSpeech.ts @@ -0,0 +1,19 @@ +import * as Effect from "effect/Effect"; +import type { HttpClient } from "effect/http"; +import { AtomRegistry } from "effect/reactivity"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; + +// Relay (DPoP) environments authorize through services only the connection runtime provides. +export function runEnvironmentSpeech( + effect: Effect.Effect, + options?: Effect.RunOptions, +): Promise { + return Effect.runPromise( + AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe( + Effect.flatMap((context) => Effect.provide(effect, context)), + ), + options, + ); +} diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index a41f359f68ef4..04534c3e0cfdc 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -88,7 +88,7 @@ vi.mock("../hooks/useSettings", () => ({ ), }), })); -vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } })); +vi.mock("./runEnvironmentSpeech", () => ({ runEnvironmentSpeech: Effect.runPromise })); vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } })); vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ ...(await importOriginal()), diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 644edb303aef2..0d1853a5dcc89 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -33,10 +33,10 @@ import { useClientSettingsHydrated, useEnvironmentSettings, } from "../hooks/useSettings"; -import { runtime } from "../lib/runtime"; import { useEnvironment } from "../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../state/session"; import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; +import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent"; const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; @@ -54,7 +54,7 @@ const statusProbes = new WeakMap { if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared); @@ -296,18 +296,16 @@ export function useEnvironmentSpeechInput(input: HookInput) { return Promise.reject( new Error("The draft's environment is unavailable for transcript cleanup."), ); - return runtime - .runPromise( - postProcessEnvironmentTranscript( - target, - transcript, - cleanupContext(options.draft), - preferences.cleanup, - () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(), - ), - options, - ) - .then((result) => result.text); + return runEnvironmentSpeech( + postProcessEnvironmentTranscript( + target, + transcript, + cleanupContext(options.draft), + preferences.cleanup, + () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(), + ), + options, + ).then((result) => result.text); }; controller = new VoiceInputController({ retryFailedRecording: true, @@ -374,8 +372,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const timer = window.setInterval(() => { if (refreshing) return; refreshing = true; - void runtime - .runPromise(getEnvironmentSpeechModels(prepared)) + void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared)) .then((result) => { if (disposed) return; const model = result.models.find((candidate) => candidate.id === modelId) ?? null; @@ -471,7 +468,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const request = ++startRequestRef.current; setSetup((current) => ({ ...current, downloading: true, error: null })); try { - await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId)); + await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId)); if (request !== startRequestRef.current) return; const next = await probeSpeechStatus(prepared, true); if (request !== startRequestRef.current) return; @@ -500,7 +497,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { startRequestRef.current += 1; setSetup((current) => ({ ...current, open: false, downloading: false })); try { - await runtime.runPromise( + await runEnvironmentSpeech( cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId), ); } catch (error) {