Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export function PreviewAnnotationSpeech({
projectName: project?.title,
ownerKey: JSON.stringify(["preview-annotation", threadRef, tabId, config.sessionId]),
draftText: text,
offerSetup: false,
readDraft: () => ({
text: draft.current.text,
selection: { start: draft.current.cursor, end: draft.current.selectionEnd },
Expand Down
5 changes: 3 additions & 2 deletions apps/web/src/components/settings/CloudVoiceConfiguration.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import {
import {
chooseAudioDestination,
pendingCloudAudioChoice,
useAudioGrant,
useAudioChoice,
} from "../../speech/voiceConsent";
import type { EnvironmentPresentation } from "../../state/environments";
import { useEnvironmentScope, usePreparedConnection } from "../../state/session";
Expand Down Expand Up @@ -172,7 +172,8 @@ export function CloudVoiceConfiguration({
routed && canUse ? prepared : null,
getCloudSpeechModels,
);
const grant = useAudioGrant();
const choice = useAudioChoice();
const grant = choice === "off" ? null : choice;
const currentModel =
active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter"
? grant.destination.model
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@ import {
useVoiceTranscriptionDestination,
} from "../../speech/useEnvironmentSpeechInput";
import { chooseAudioDestination, chooseTranscriptCleanup } from "../../speech/voiceConsent";
import { useEnvironmentScope } from "../../state/session";
import { toastManager } from "../ui/toast";
import { SettingsScopeProvider } from "./SettingsScopeContext";
import { SpeechLanguageSelect } from "./SpeechLanguageSelect";
import { VoiceSettingsPanel } from "./VoiceSettingsPanel";

vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });
vi.mock(import("../../state/session"), { spy: true });

// The real connection runtime authorizes relay requests with a signed-in cloud session.
vi.mock("../../connection/runtime", async () => {
const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([
Expand Down Expand Up @@ -68,7 +71,7 @@ vi.mock("../../connection/runtime", async () => {
};
});

it("preserves the model test transcript when its model is filtered out", async () => {
it("tests the selected model after download and preserves its transcript across filters", async () => {
const target = new PrimaryConnectionTarget({
environmentId: EnvironmentId.make("voice-filter"),
label: "Test environment",
Expand Down Expand Up @@ -103,15 +106,23 @@ it("preserves the model test transcript when its model is filtered out", async (
modelUnloadTimeout: "never",
gpuDevices: [],
} satisfies EnvironmentSpeechStatus;
let installed = false;
const fetch = globalThis.fetch;
using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
const url = new URL(input instanceof Request ? input.url : input, window.location.href);
if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status));
if (url.pathname === "/api/voice/status")
return Promise.resolve(
Response.json(installed ? status : { ...status, state: "missing-model" }),
);
if (url.pathname === "/api/voice/models/download") {
installed = true;
return Promise.resolve(Response.json({ models: [model] }));
}
if (url.pathname === "/api/voice/models")
return Promise.resolve(
Response.json({
models: [
model,
installed ? model : { ...model, state: "downloadable", active: false },
{
...model,
id: "both",
Expand All @@ -136,6 +147,7 @@ it("preserves the model test transcript when its model is filtered out", async (
return fetch(input, init);
});
const transcript = "Keep this transcript.";
using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true);
await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
() => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
async () => {
Expand Down Expand Up @@ -177,7 +189,13 @@ it("preserves the model test transcript when its model is filtered out", async (
</SettingsScopeProvider>,
);
const testModel = screen.getByRole("button", { name: "Test model", exact: true });
const download = screen.getByRole("button", { name: "Download", exact: true });
await expect.element(download).toBeVisible();
expect(testModel.query()).toBeNull();
expect(screen.getByRole("button", { name: "Set up voice input" }).query()).toBeNull();
await userEvent.click(download);
await expect.element(testModel).toBeVisible();
expect(screen.getByRole("button", { name: "Set up voice input" }).query()).toBeNull();
await userEvent.click(testModel);
const result = screen.getByRole("status", { name: "Test transcript" });
await expect.element(result).toHaveTextContent(transcript);
Expand Down
62 changes: 38 additions & 24 deletions apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ const THREAD_ENV = "thread-env";
function setup(options: { readonly destination?: string | null } = {}) {
mocks.settings = {};
setDestination(options.destination ?? null);
if (!options.destination) localStorage.removeItem("t3code:voice-audio-grant:v1");
chooseTranscriptCleanup(false);
mocks.primaryEnvironmentId = "primary-env";
mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
Expand Down Expand Up @@ -243,15 +244,22 @@ it.each([
["nothing is configured", {}],
// A transcription host picked by an earlier PoC build is not consent.
["a legacy transcription host is configured", { voiceTranscriptionEnvironmentId: "env-a" }],
])("stays local-only when %s, even with voice-capable environments", async (_, legacy) => {
])("opens destination setup without sending audio when %s", async (_, legacy) => {
setup();
mocks.settings = { ...legacy, speechSilenceFiltering: true };
await using _screen = await render(<Probe />);
await using screen = await render(<Probe />);
// The composer mic and the dictation shortcut both gate on this.
expect(mocks.voice.available).toBe(true);
await act(() => mocks.voice.start());
expect(screen.router?.state.location).toMatchObject({
pathname: "/settings/voice",
hash: "voice-transcription-destination",
});
expectNoAudioSent();
expect(mocks.committed).not.toHaveBeenCalled();
expect(mocks.voice.available).toBe(false);
expect(mocks.voice.state.phase).toBe("idle");
expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
expect(localStorage.getItem("t3code:voice-audio-grant:v1")).toBeNull();
});

it.each([true, false])(
Expand Down Expand Up @@ -302,6 +310,7 @@ it.each([
expect(mocks.voice.state.phase).toBe("recording");
setDestination(next);
await screen.rerender(<Probe />);
if (next === null) expect(mocks.voice.available).toBe(false);
expect(mocks.voice.state.phase).toBe("idle");
await act(() => mocks.voice.stop());
expect(mocks.transcribeCalls).toEqual([]);
Expand Down Expand Up @@ -390,7 +399,7 @@ it("revokes across windows even before the storage event and survives stale pref
expect(storage.getItem(key)).toBeNull();
expect(canSend()).toBe(false);
writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
await vi.waitFor(() => expect(mocks.voice.transcriptionEnvironmentId).toBeNull());
expect(mocks.voice.state.phase).toBe("idle");
await act(() => mocks.voice.stop());
expect(mocks.transcribeCalls).toEqual([]);
Expand All @@ -401,26 +410,30 @@ it("revokes across windows even before the storage event and survives stale pref
}
});

it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => {
setup();
mocks.cloudVoiceCapable.add("env-b");
mocks.modelState = "missing-model";
window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json");
const save = pendingCloudAudioChoice({
kind: "cloud",
environmentId: EnvironmentId.make("env-b"),
provider: "xai",
});
await using _screen = await render(<Probe />);
expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
await act(() => mocks.voice.start());
expectNoAudioSent();
await act(save);
await act(() => mocks.voice.start());
await act(() => mocks.voice.stop());
expect(mocks.transcribeCalls).toEqual(["xai via env-b"]);
expect(mocks.statusRequests).toEqual([]);
});
it.each(["not-json", JSON.stringify("off")])(
"does not enable a pending provider over %s before the save completes",
async (previous) => {
setup();
mocks.cloudVoiceCapable.add("env-b");
mocks.modelState = "missing-model";
window.localStorage.setItem("t3code:voice-audio-grant:v1", previous);
const save = pendingCloudAudioChoice({
kind: "cloud",
environmentId: EnvironmentId.make("env-b"),
provider: "xai",
});
await using _screen = await render(<Probe />);
expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
if (previous === JSON.stringify("off")) expect(mocks.voice.available).toBe(false);
await act(() => mocks.voice.start());
expectNoAudioSent();
await act(save);
await act(() => mocks.voice.start());
await act(() => mocks.voice.stop());
expect(mocks.transcribeCalls).toEqual(["xai via env-b"]);
expect(mocks.statusRequests).toEqual([]);
},
);

it("sends audio to OpenRouter only under a device grant naming the exact model", async () => {
setup();
Expand Down Expand Up @@ -468,6 +481,7 @@ it("uses neither destination when the cloud save marker cannot be read", async (
await using _screen = await render(<Probe />);
expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
expect(canSendToPrevious()).toBe(false);
expect(mocks.voice.available).toBe(false);
await act(() => mocks.voice.start());
expectNoAudioSent();
});
Expand Down
31 changes: 25 additions & 6 deletions apps/web/src/speech/useEnvironmentSpeechInput.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import type {
} from "@t3tools/contracts";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
import { useNavigate } from "@tanstack/react-router";
import * as Option from "effect/Option";
import { useEffect, useRef, useState } from "react";

Expand All @@ -37,7 +38,7 @@ import { useEnvironment } from "../state/environments";
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";
import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";

const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
Expand Down Expand Up @@ -88,6 +89,11 @@ type HookInput = {
readonly ownerKey: string;
readonly draftText: string;
readonly disabled?: boolean | undefined;
/**
* Until a destination is chosen, voice input is offered and its first gesture opens Voice
* settings. False hides it instead, for editors whose draft would not survive the navigation.
*/
readonly offerSetup?: boolean | undefined;
/** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */
readonly cleanup?: boolean | undefined;
readonly readDraft: () => DraftInput;
Expand All @@ -104,7 +110,8 @@ type HookInput = {
*/
export function useVoiceTranscriptionDestination() {
const hydrated = useClientSettingsHydrated();
const grant = useAudioGrant();
const choice = useAudioChoice();
const grant = choice === "off" ? null : choice;
const environmentId = hydrated && grant ? grant.destination.environmentId : null;
const environment = useEnvironment(environmentId);
const connection = Option.getOrNull(usePreparedConnection(environmentId));
Expand All @@ -117,6 +124,7 @@ export function useVoiceTranscriptionDestination() {
provider ? "cloudVoiceTranscription" : "voiceTranscription"
] === true;
return {
...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}),
environmentId,
...(provider ? { cloudProvider: provider } : {}),
...(model ? { cloudModel: model } : {}),
Expand All @@ -130,6 +138,7 @@ export function useVoiceTranscriptionDestination() {
}

export function useEnvironmentSpeechInput(input: HookInput) {
const navigate = useNavigate();
const destination = useVoiceTranscriptionDestination();
const prepared = destination.prepared;
const grantId = destination.grantId;
Expand Down Expand Up @@ -437,10 +446,16 @@ export function useEnvironmentSpeechInput(input: HookInput) {
});

const start = useStableCallback(async () => {
if (latestRef.current.input.disabled) return;
if (!grantId) {
if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
return;
}
const request = ++startRequestRef.current;
const controller = controllerRef.current;
const owner = latestRef.current.input.ownerKey;
if (latestRef.current.input.disabled || !controller || !prepared) return;
if (!controller || !prepared) return;
if (cloudProvider) {
setLevel(0);
await controller.start();
Expand Down Expand Up @@ -529,11 +544,15 @@ export function useEnvironmentSpeechInput(input: HookInput) {
}
});

const destinationReady =
grantId === null
? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false
: prepared !== null &&
(cloudProvider !== undefined ||
(status?.prepared === prepared && status.value?.supported !== false));
return {
available:
prepared !== null &&
(cloudProvider !== undefined ||
(status?.prepared === prepared && status.value?.supported !== false)) &&
destinationReady &&
typeof navigator !== "undefined" &&
Boolean(navigator.mediaDevices?.getUserMedia) &&
typeof AudioWorkletNode !== "undefined",
Expand Down
1 change: 1 addition & 0 deletions apps/web/src/speech/useTextFieldSpeech.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export function useTextFieldSpeech(input: {
ownerKey: input.ownerKey,
draftText: input.text,
disabled: input.disabled,
offerSetup: false,
readDraft: () => {
const field = latest.current.textareaRef.current;
const cursor = field?.selectionStart ?? latest.current.text.length;
Expand Down
27 changes: 15 additions & 12 deletions apps/web/src/speech/voiceConsent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1";
const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination });
const AudioGrant = Schema.NullOr(Grant);
const sameGrant = Schema.toEquivalence(AudioGrant);
const AudioChoice = Schema.NullOr(
Schema.Union([Grant, Schema.Struct({ pending: Grant, previous: AudioGrant })]),
);
const SavedChoice = Schema.Union([AudioGrant, Schema.Literal("off")]);
const AudioChoice = Schema.Union([
SavedChoice,
Schema.Struct({ pending: Grant, previous: SavedChoice }),
]);
type AudioChoice = typeof AudioChoice.Type;
const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`;
const pendingGrant = (choice: AudioChoice) =>
choice && "pending" in choice ? choice.pending : null;
choice && choice !== "off" && "pending" in choice ? choice.pending : null;
const CleanupGrant = Schema.NullOr(Schema.String);
const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice));
const readChoice = () => readLocalStorageValue(AUDIO_KEY, null, AudioChoice);
Expand All @@ -35,22 +37,22 @@ function readSaveMarker(id: string | undefined) {
}
}

function activeGrant(choice: AudioChoice, marker: string | null | undefined) {
if (!choice || !("pending" in choice)) return choice;
function activeChoice(choice: AudioChoice, marker: string | null | undefined) {
if (!choice || choice === "off" || !("pending" in choice)) return choice;
// Only a confirmed missing marker means the previous grant still applies.
if (marker === null) return choice.previous;
return marker === "true" ? choice.pending : null;
return marker === "true" ? choice.pending : ("off" as const);
}

export function useAudioGrant() {
export function useAudioChoice() {
const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice);
const id = pendingGrant(choice)?.id;
const marker = useSyncExternalStore(
(notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}),
() => readSaveMarker(id),
() => undefined,
);
return activeGrant(choice, marker);
return activeChoice(choice, marker);
}

function replaceChoice(choice: AudioChoice, previous = readChoice()) {
Expand All @@ -61,7 +63,7 @@ function replaceChoice(choice: AudioChoice, previous = readChoice()) {
}

export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) {
replaceChoice(destination && { id: randomUUID(), destination });
replaceChoice(destination ? { id: randomUUID(), destination } : "off");
}

export function pendingCloudAudioChoice(
Expand All @@ -70,7 +72,7 @@ export function pendingCloudAudioChoice(
const choice = readChoice();
const pending = { id: randomUUID(), destination };
replaceChoice(
{ pending, previous: activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)) },
{ pending, previous: activeChoice(choice, readSaveMarker(pendingGrant(choice)?.id)) },
choice,
);
// A late response can acknowledge only its own choice, never write the destination another tab picked.
Expand All @@ -91,7 +93,8 @@ export function bindAudioGrant(id: string) {
choice = raw ? decodeChoice(raw) : null;
cachedRaw = raw;
}
return activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id));
const active = activeChoice(choice, readSaveMarker(pendingGrant(choice)?.id));
return active === "off" ? null : active;
};
const bound = current();
if (bound?.id !== id) return () => false;
Expand Down
Loading
Loading