Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ import {
discardScheduledTaskEditor,
markScheduledTaskEditorSaved,
scheduledTaskEditorSessionAtom,
subscribeScheduledTaskEditorReset,
} from "./scheduled-task-editor-state";
import { appAtomRegistry } from "../../state/atom-registry";
import {
Expand Down Expand Up @@ -518,6 +519,7 @@ function SettingsScheduledTaskEditorScreen({ title }: { readonly title: string }
label: editor?.draft.title.trim() || title,
subscribeToDraftChanges: (onChange) =>
appAtomRegistry.subscribe(scheduledTaskEditorSessionAtom, onChange),
subscribeToDraftReset: subscribeScheduledTaskEditorReset,
selection,
readDraftMessage: () => {
const current = readEditor();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,14 @@ type ScheduledTaskEditorSession = {
export const scheduledTaskEditorSessionAtom = Atom.make<ScheduledTaskEditorSession | null>(
null,
).pipe(Atom.keepAlive);
const scheduledTaskEditorResetAtom = Atom.make(0).pipe(Atom.keepAlive);
let nextEditorId = 0;
let newTaskLinkAwaitsDiscard = false;

export function subscribeScheduledTaskEditorReset(onReset: () => void): () => void {
return appAtomRegistry.subscribe(scheduledTaskEditorResetAtom, onReset);
}

export function scheduledTaskEditorHasChanges(
initial: ScheduledTaskEditor | null,
current: ScheduledTaskEditor | null,
Expand Down Expand Up @@ -68,6 +73,7 @@ export function updateScheduledTaskEditor(
*/
export function markScheduledTaskEditorSaved(): void {
newTaskLinkAwaitsDiscard = false;
appAtomRegistry.set(scheduledTaskEditorResetAtom, ++nextEditorId);
const session = appAtomRegistry.get(scheduledTaskEditorSessionAtom);
if (session) {
appAtomRegistry.set(scheduledTaskEditorSessionAtom, { ...session, initial: session.current });
Expand Down Expand Up @@ -96,6 +102,7 @@ export function takeScheduledTaskNewLink(
if (session && scheduledTaskEditorHasChanges(session.initial, session.current)) {
newTaskLinkAwaitsDiscard = true;
} else {
appAtomRegistry.set(scheduledTaskEditorResetAtom, ++nextEditorId);
appAtomRegistry.set(scheduledTaskEditorSessionAtom, null);
}
}
Expand All @@ -106,6 +113,7 @@ export function takeScheduledTaskNewLink(
export function discardScheduledTaskEditor(): void {
if (newTaskLinkAwaitsDiscard) {
newTaskLinkAwaitsDiscard = false;
appAtomRegistry.set(scheduledTaskEditorResetAtom, ++nextEditorId);
appAtomRegistry.set(scheduledTaskEditorSessionAtom, null);
return;
}
Expand Down
2 changes: 2 additions & 0 deletions apps/mobile/src/features/threads/ThreadComposer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import {
getComposerDraftSnapshot,
composerDraftsAtom,
setComposerDraftText,
subscribeComposerDraftReset,
composerContextImportsAtom,
countComposerDraftAttachmentsAfterSelection,
useComposerDraft,
Expand Down Expand Up @@ -576,6 +577,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
label: props.selectedThread.title || "Untitled thread",
readDraftMessage: () => getComposerDraftSnapshot(composerDraftKey).text,
subscribeToDraftChanges: (onChange) => appAtomRegistry.subscribe(composerDraftsAtom, onChange),
subscribeToDraftReset: (onReset) => subscribeComposerDraftReset(composerDraftKey, onReset),
selection: composerMenu.selection,
onChangeDraftMessage: (text) => setComposerDraftText(composerDraftKey, text),
onChangeSelection: composerMenu.onSelectionChange,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export function useVoiceInputController(input: {
readonly label: string;
readonly readDraftMessage: () => string | null;
readonly subscribeToDraftChanges: (onChange: () => void) => () => void;
readonly subscribeToDraftReset?: (onReset: () => void) => () => void;
readonly selection: ComposerEditorSelection;
readonly disabled?: boolean;
readonly onChangeDraftMessage: (value: string) => void;
Expand Down Expand Up @@ -59,6 +60,7 @@ export function useVoiceInputController(input: {
},
captured.selection,
captured.subscribeToDraftChanges,
captured.subscribeToDraftReset,
),
label: captured.label,
});
Expand Down
121 changes: 118 additions & 3 deletions apps/mobile/src/features/voice-input/voiceInputSession.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
import { beforeEach, describe, expect, it, onTestFinished, vi } from "vite-plus/test";
import type { PreparedVoiceTranscription } from "@t3tools/client-runtime/voice-input";
import { EnvironmentId } from "@t3tools/contracts";
import { Atom, AtomRegistry } from "effect/reactivity";
import { resetVoiceInputGlobalsForTests } from "../../../../../packages/client-runtime/src/voice-input/controller";
import {
Expand All @@ -26,6 +27,16 @@ import {
type VoiceInputTarget,
type VoiceRecordingSample,
} from "./voiceInputSession";
import {
discardScheduledTaskEditor,
markScheduledTaskEditorSaved,
scheduledTaskEditorSessionAtom,
subscribeScheduledTaskEditorReset,
takeScheduledTaskNewLink,
updateScheduledTaskEditor,
} from "../settings/scheduled-task-editor-state";
import { createDraft } from "../settings/scheduledTaskDraft";
import { appAtomRegistry } from "../../state/atom-registry";

function createTarget(
ownerKey: string,
Expand All @@ -47,6 +58,7 @@ function createSession(
const releaseKeepAwake = vi.fn();
const keepAwake = vi.fn(() => releaseKeepAwake);
const publishAudioLevels = vi.fn();
const deleteRecording = vi.fn();
const recorder = {
uri: "file:///voice.m4a",
prepareToRecordAsync: vi.fn(async () => {}),
Expand All @@ -63,18 +75,120 @@ function createSession(
requestPermission: async () => ({ granted: true, canAskAgain: true }),
configureRecording: async () => {},
releaseRecording: vi.fn(async () => {}),
deleteRecording: vi.fn(),
deleteRecording,
sampleRecording: () => null,
publishAudioLevels,
keepAwake,
...overrides,
});
return { session, recorder, prepare, keepAwake, releaseKeepAwake, publishAudioLevels };
return {
session,
recorder,
prepare,
keepAwake,
releaseKeepAwake,
publishAudioLevels,
deleteRecording,
};
}

describe("global voice input", () => {
beforeEach(() => resetVoiceInputGlobalsForTests());

it.each([
"recover",
"reset",
"remove",
"new-task-link",
"task-saved",
"discard-new-task-link",
] as const)("handles retained batch audio on %s", async (action) => {
const registry = AtomRegistry.make();
onTestFinished(() => registry.dispose());
appAtomRegistry.set(scheduledTaskEditorSessionAtom, null);
onTestFinished(() => appAtomRegistry.set(scheduledTaskEditorSessionAtom, null));
const draft = Atom.make<string | null>("draft");
const reset = Atom.make(0);
const unsubscribeReset = vi.fn();
const { session, recorder, prepare, deleteRecording, keepAwake } = createSession();
const transcribe = vi
.fn<PreparedVoiceTranscription["transcribe"]>()
.mockRejectedValueOnce(new Error("Upload failed"))
.mockRejectedValueOnce(new Error("Transcription failed"))
.mockResolvedValueOnce("recovered text");
prepare.mockResolvedValue({ locale: "en-US", transcribe });
const commit = vi.fn();
await session.start(
createVoiceInputTarget(
"first",
() => registry.get(draft),
commit,
{ start: 5, end: 5 },
(onChange) => registry.subscribe(draft, onChange),
(onReset) => {
const stop = ["new-task-link", "task-saved", "discard-new-task-link"].includes(action)
? subscribeScheduledTaskEditorReset(onReset)
: registry.subscribe(reset, onReset);
return () => {
stop();
unsubscribeReset();
};
},
),
);
await session.controller.stop();
expect(session.controller.currentState.phase).toBe("error");
expect(deleteRecording).not.toHaveBeenCalled();
expect(commit).not.toHaveBeenCalled();

await session.retry();
expect(session.controller.currentState.phase).toBe("error");
expect(deleteRecording).not.toHaveBeenCalled();
if (action === "recover") {
await session.retry();
} else {
if (action === "new-task-link") {
takeScheduledTaskNewLink({ routes: [{ name: "SettingsScheduledTaskNew" }] });
} else if (action === "discard-new-task-link") {
const editor = {
environmentId: EnvironmentId.make("environment"),
environmentLabel: "Environment",
draft: createDraft(null, null),
};
updateScheduledTaskEditor(
{ ...editor, draft: { ...editor.draft, prompt: "typed" } },
editor,
);
takeScheduledTaskNewLink({ routes: [{ name: "SettingsScheduledTaskNew" }] });
expect(session.controller.currentState.phase).toBe("error");
expect(deleteRecording).not.toHaveBeenCalled();
discardScheduledTaskEditor();
} else if (action === "task-saved") {
markScheduledTaskEditorSaved();
} else if (action === "reset") registry.set(reset, 1);
else registry.set(draft, null);
expect(session.controller.currentState.phase).toBe("idle");
expect(deleteRecording).toHaveBeenCalledExactlyOnceWith("file:///voice.m4a");
expect(commit).not.toHaveBeenCalled();
await session.start(createTarget("first", () => "draft", commit, { start: 5, end: 5 }));
await session.controller.stop();
}
expect(session.controller.currentState.phase).toBe("idle");
expect(transcribe.mock.calls.map(([uri]) => uri)).toEqual([
"file:///voice.m4a",
"file:///voice.m4a",
"file:///voice.m4a",
]);
expect(recorder.record).toHaveBeenCalledTimes(action === "recover" ? 1 : 2);
expect(keepAwake).toHaveBeenCalledTimes(action === "recover" ? 1 : 2);
expect(commit).toHaveBeenCalledExactlyOnceWith("draft recovered text", {
start: 20,
end: 20,
});
expect(deleteRecording).toHaveBeenCalledTimes(action === "recover" ? 1 : 2);
expect(unsubscribeReset).toHaveBeenCalledTimes(action === "recover" ? 3 : 2);
});

it.each(["retry", "takeover"])(
"handles failed deactivation with %s without notifying a released owner",
async (action) => {
Expand Down Expand Up @@ -294,6 +408,7 @@ describe("global voice input", () => {
await session.controller.stop();
expect(commit).not.toHaveBeenCalled();
expect(session.controller.currentState.error).toContain("draft changed");
session.dispose();
expect(unsubscribe).toHaveBeenCalledTimes(1);
registry.dispose();
});
Expand Down
22 changes: 17 additions & 5 deletions apps/mobile/src/features/voice-input/voiceInputSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export type VoiceInputTarget = {
readonly label?: string;
readonly readDraft: () => VoiceDraftSnapshot | null;
readonly commitDraft: VoiceInputControllerDependencies["commitDraft"];
readonly subscribe: () => () => void;
readonly subscribe: (onReset: () => void) => () => void;
};

export type VoiceInputSnapshot = {
Expand Down Expand Up @@ -53,6 +53,7 @@ export function createVoiceInputTarget(
commitDraft: VoiceInputTarget["commitDraft"],
selection: VoiceDraftSnapshot["selection"],
subscribeToChanges: (onChange: () => void) => () => void,
subscribeToReset?: (onReset: () => void) => () => void,
): VoiceInputTarget {
let revision = 0;
return {
Expand All @@ -63,15 +64,24 @@ export function createVoiceInputTarget(
return { ownerKey, text, selection, revision };
},
commitDraft,
subscribe: () => {
subscribe: (onReset) => {
let previousText = readText();
return subscribeToChanges(() => {
const unsubscribeChanges = subscribeToChanges(() => {
const text = readText();
if (text === null) {
onReset();
return;
}
if (text !== previousText) {
previousText = text;
revision += 1;
}
});
const unsubscribeReset = subscribeToReset?.(onReset);
return () => {
unsubscribeChanges();
unsubscribeReset?.();
};
},
};
}
Expand Down Expand Up @@ -100,6 +110,7 @@ export class VoiceInputSession {
constructor(dependencies: VoiceInputSessionDependencies) {
this.dependencies = dependencies;
this.controller = new VoiceInputController({
retryFailedRecording: true,
Comment thread
Lucenx9 marked this conversation as resolved.
Resolved
...dependencies,
onStateChange: (state) => this.handleStateChange(state),
readDraft: () => this.target?.readDraft() ?? null,
Expand Down Expand Up @@ -140,8 +151,9 @@ export class VoiceInputSession {

start(target: VoiceInputTarget): Promise<void> {
if (voiceInputBlocksSubmission(this.controller.currentState)) return Promise.resolve();
this.unsubscribe?.();
this.target = target;
this.unsubscribe = target.subscribe();
this.unsubscribe = target.subscribe(() => this.cancel(target.ownerKey));
return this.controller.start();
}

Expand All @@ -154,7 +166,7 @@ export class VoiceInputSession {
}

private handleStateChange(state: VoiceInputState): void {
if (!voiceInputBlocksSubmission(state)) {
if (state.phase === "idle") {
this.unsubscribe?.();
this.unsubscribe = null;
}
Expand Down
4 changes: 4 additions & 0 deletions apps/mobile/src/state/use-composer-drafts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,10 @@ function markComposerDraftReset(draftKey: string): void {
appAtomRegistry.set(composerDraftResetAtom(draftKey), nextComposerDraftReset());
}

export function subscribeComposerDraftReset(draftKey: string, onReset: () => void): () => void {
return appAtomRegistry.subscribe(composerDraftResetAtom(draftKey), onReset);
}

/**
* Raised when a draft's text is replaced outright (sent, discarded), or when the editor switches
* drafts. iOS editor events read it synchronously, so an edit the editor made before applying the
Expand Down
9 changes: 7 additions & 2 deletions docs/user/composer.md
Original file line number Diff line number Diff line change
Expand Up @@ -240,8 +240,13 @@ microphone on iOS. Swipe a device left to forget it. Android may use another inp
when the preferred microphone cannot be routed. Devices with the same name may share
a saved preference, but simultaneous same-name inputs remain separately selectable.

T3 Code deletes the temporary recording from your phone after transcription or
cancellation. The transcript stays in your draft until you send the message to the agent.
If transcription on a paired environment fails, retry dictation to reuse the same
recording. Sending or discarding the draft, dismissing the error, or choosing another
transcription destination discards that recording.
T3 Code deletes the temporary recording from your phone after successful transcription
Comment thread
Lucenx9 marked this conversation as resolved.
Resolved
or cancellation. On-device iPhone transcription also deletes it after a failed attempt,
so retrying requires a new recording. The transcript stays in your draft until you send
the message to the agent.

## Queued messages

Expand Down
Loading