Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 99 additions & 1 deletion apps/web/src/components/ChatView.browser.test.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createMemoryHistory } from "@tanstack/react-router";
import { scopeProjectRef } from "@t3tools/client-runtime/environment";
import { ThreadId } from "@t3tools/contracts";
import { ThreadId, type VoiceTranscriptionDestination } from "@t3tools/contracts";
import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts/settings";
import type * as Context from "effect/Context";
import { expect, it, vi } from "vite-plus/test";
Expand All @@ -13,13 +13,16 @@ import "~/components/ChatView";
import { DraftId, useComposerDraftStore } from "~/composerDraftStore";
import { __setClientSettingsForTests } from "~/hooks/useSettings";
import { getRouter } from "~/router";
import { chooseAudioDestination } from "~/speech/voiceConsent";
import {
ENVIRONMENT_ID,
PROJECT_ID,
installFakeEnvironment,
type FakeDriver,
} from "../../test/fakeEnvironment";
import { render } from "../../test/render";
import { useComposerDictation } from "~/dictation/useComposerDictation";
import { useEnvironmentSpeechInput } from "~/speech/useEnvironmentSpeechInput";

// The socket is the only fake layer; see test/appScenarios.renders.tsx.
vi.mock("../../../../packages/client-runtime/src/connection/driver.ts", async (importOriginal) => {
Expand All @@ -31,6 +34,8 @@ vi.mock("../../../../packages/client-runtime/src/connection/driver.ts", async (i
return { ...actual, layer: Layer.succeed(actual.ConnectionDriver, fakeDriver) };
});

vi.mock(import("~/dictation/useComposerDictation"), { spy: true });
vi.mock(import("~/speech/useEnvironmentSpeechInput"), { spy: true });
vi.mock(import("~/rpc/requestLatencyState"), async (original) => ({
...(await original()),
trackRpcRequestSent: () => {},
Expand Down Expand Up @@ -65,3 +70,96 @@ it("a new draft does not load its reserved thread before the server lists it", a
// A 404 here marks the thread deleted, and the view never loads it once the send creates it.
expect(threadRequests).toEqual([]);
});

const MICROPHONE_BUTTONS = /^(Dictate|Set up dictation|Start voice input)$/;

async function renderDraftComposer(name: string) {
__setClientSettingsForTests({
...DEFAULT_CLIENT_SETTINGS,
onboardingCompletedAt: "2026-01-01T00:00:00.000Z",
});
const draftId = DraftId.make(`draft-microphone-${name}`);
useComposerDraftStore
.getState()
.setProjectDraftThreadId(scopeProjectRef(ENVIRONMENT_ID, PROJECT_ID), draftId, {
threadId: ThreadId.make(`thread-microphone-${name}`),
});
const router = getRouter(createMemoryHistory({ initialEntries: [`/draft/${draftId}`] }));
await router.load();
// AppRoot brings its own router.
return render(<AppRoot router={router} />, { router: false });
}

it.each<{
readonly state: string;
readonly destination: VoiceTranscriptionDestination | null;
readonly microphone: string;
}>([
{ state: "no Voice destination", destination: null, microphone: "Set up dictation" },
{
state: "a chosen Voice destination",
destination: { kind: "cloud", environmentId: ENVIRONMENT_ID, provider: "openai" },
microphone: "Start voice input",
},
])("the composer shows one microphone with $state", async ({ destination, microphone }) => {
using _environment = installFakeEnvironment([]);
using _voiceChoice = {
[Symbol.dispose]: () => window.localStorage.removeItem("t3code:voice-audio-grant:v1"),
};
if (destination) chooseAudioDestination(destination);
await using _app = await renderDraftComposer(destination?.kind ?? "none");
await expect.element(page.getByRole("button", { name: microphone, exact: true })).toBeVisible();

const microphones = page.getByRole("button", { name: MICROPHONE_BUTTONS }).elements();
expect(microphones.map((button) => button.getAttribute("aria-label"))).toEqual([microphone]);
});

const isMac = /Mac/.test(navigator.platform);

it.each([
{
running: "Voice input",
chord: { key: " ", code: "Space", altKey: isMac, ctrlKey: !isMac, shiftKey: !isMac },
visibleControl: "Finish voice input",
},
{
running: "Dictation",
chord: { key: "d", code: "KeyD", metaKey: isMac, ctrlKey: !isMac, shiftKey: true },
visibleControl: "Stop and insert dictation",
},
])("a running $running recording blocks the other shortcut", async (row) => {
const { useEnvironmentSpeechInput: realVoice } = await vi.importActual<
typeof import("~/speech/useEnvironmentSpeechInput")
>("~/speech/useEnvironmentSpeechInput");
const { useComposerDictation: realDictation } = await vi.importActual<
typeof import("~/dictation/useComposerDictation")
>("~/dictation/useComposerDictation");
const recording = { phase: "recording", error: null, errorAction: null } as const;
const voiceRunning = row.running === "Voice input";
const startVoice = vi.fn(async () => {});
const startDictation = vi.fn();
using _environment = installFakeEnvironment([]);
using _voice = vi.mocked(useEnvironmentSpeechInput).mockImplementation((input) => ({
...realVoice(input),
available: true,
start: startVoice,
...(voiceRunning ? { state: recording, blocksSubmission: true } : {}),
}));
using _dictation = vi.mocked(useComposerDictation).mockImplementation((input) => ({
...realDictation(input),
start: startDictation,
...(voiceRunning ? {} : { state: recording, active: true, readPhase: () => "recording" }),
}));
await using _app = await renderDraftComposer(`busy-${row.running}`);
await expect
.element(page.getByRole("button", { name: row.visibleControl, exact: true }))
.toBeVisible();

window.dispatchEvent(new KeyboardEvent("keydown", { ...row.chord, cancelable: true }));

expect(startVoice).not.toHaveBeenCalled();
expect(startDictation).not.toHaveBeenCalled();
await expect
.element(page.getByRole("button", { name: row.visibleControl, exact: true }))
.toBeVisible();
});
21 changes: 13 additions & 8 deletions apps/web/src/components/chat/ChatComposer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1941,6 +1941,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
draftText: prompt,
disabled: dictationDisabled,
offerSetup: false,
readDraft: () => {
const snapshot = readComposerSnapshot();
return {
Expand All @@ -1963,13 +1964,15 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
window.requestAnimationFrame(() => composerEditorRef.current?.focusAt(cursor));
},
});
// Dictation and Voice input are separate systems that share one composer microphone.
const voiceInputOwnsMicrophone = speechInput.available && !dictation.active;
Comment thread
eyeveil marked this conversation as resolved.
Resolved
const { blocksSubmission: voiceInputActive, submitAfterDictation } = speechInput;
const voiceSendDisabledReason =
sendDisabledReason ?? (speechInput.preparing ? "Preparing voice input" : null);
const dictationShortcutLabel = useDictationShortcut({
keybindings,
speech: speechInput,
disabled: dictationDisabled,
disabled: dictationDisabled || dictation.active,
terminalOpen,
modelPickerOpen: isComposerModelPickerOpen,
...(backgroundThread ? { targetRef: composerFormRef } : {}),
Expand Down Expand Up @@ -2695,7 +2698,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)

useDictationHoldShortcut({
dictation,
enabled: !composerTextUnavailable,
enabled: !composerTextUnavailable && !voiceInputActive,
isDictateShortcut: (event) =>
!isCommandPaletteOpen() &&
(backgroundThread
Expand Down Expand Up @@ -3718,11 +3721,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
>
{composerControls}
</div>
<ComposerDictationControl
dictation={dictation}
disabled={composerTextUnavailable}
shortcutLabel={shortcutLabelForCommand(keybindings, "composer.dictate")}
/>
{voiceInputOwnsMicrophone ? null : (
<ComposerDictationControl
dictation={dictation}
disabled={composerTextUnavailable}
shortcutLabel={shortcutLabelForCommand(keybindings, "composer.dictate")}
/>
)}
<ComposerFooterPrimaryActions
compact={isComposerPrimaryActionsCompact}
canOperateThread={canOperateThread}
Expand Down Expand Up @@ -3782,7 +3787,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
? { onCompactContext: compactThreadContextStable }
: {})}
>
{speechInput.available ? (
{voiceInputOwnsMicrophone ? (
<>
<ComposerSpeechRecordingPill
state={speechInput.state}
Expand Down
2 changes: 1 addition & 1 deletion apps/web/test/fakeEnvironment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ const descriptor = {
// A matching version, since a mismatch adds an update banner a typical user never sees.
serverVersion: APP_VERSION,
orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION,
capabilities: { repositoryIdentity: true },
capabilities: { repositoryIdentity: true, cloudVoiceTranscription: true },
};
const auth = {
policy: "loopback-browser",
Expand Down
3 changes: 3 additions & 0 deletions docs/user/composer.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,9 @@ Linux while T3 Code is focused, and release to insert. `Esc` cancels. Recordings
five minutes long. Rebind the shortcut as **Composer: Hold to Dictate** in
[keybindings](keybindings.md).

After you choose a destination in **Settings → Voice**, the composer's microphone uses
[Voice input](voice-input.md) instead. The hold shortcut still runs Dictation.

The machine that owns the thread transcribes when its dictation is ready; otherwise another
ready machine does. With two or more machines connected, you can pick one for this device in
**Settings → Connections → Dictation machine**. Your audio goes only to that machine.
Expand Down
3 changes: 2 additions & 1 deletion docs/user/voice-input.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o
On web and desktop, you choose where transcription runs.
Nothing is sent until you choose one: open **Settings → Voice** and set
**Transcribe with** to the environment that should transcribe for you.
On a new device, the microphone or dictation shortcut opens that setting from the composer.
Until you choose one, the composer's microphone runs
[Dictation](./composer.md#dictation-on-web-and-desktop) instead.
Microphone audio recorded on this device is sent to that environment over your
authenticated connection, transcribed there, and discarded. The choice is saved
on this device only, so every computer and browser you use asks for it once.
Expand Down
Loading