diff --git a/apps/desktop/scripts/build-apple-speech.mjs b/apps/desktop/scripts/build-apple-speech.mjs new file mode 100644 index 0000000000000..efef5a452190f --- /dev/null +++ b/apps/desktop/scripts/build-apple-speech.mjs @@ -0,0 +1,74 @@ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import * as NodeUtil from "node:util"; + +// oxlint-disable-next-line t3code/no-global-process-runtime -- The Swift compiler targets the actual host; this script has no Effect runtime. +const hostArch = process.arch; +// oxlint-disable-next-line t3code/no-global-process-runtime -- The Speech framework exists only in the macOS SDK. +const hostPlatform = process.platform; + +const { values } = NodeUtil.parseArgs({ + options: { output: { type: "string" }, arch: { type: "string", default: hostArch } }, +}); +// Packaging passes --output and must not ship without the helper; dev builds only lose the option. +const required = values.output !== undefined; + +if (hostPlatform !== "darwin") { + if (required) throw new Error("The Apple Speech helper can only be built on macOS."); +} else { + const targets = { arm64: ["arm64"], x64: ["x86_64"], universal: ["arm64", "x86_64"] }[ + values.arch + ]; + if (targets === undefined) throw new Error(`Unsupported macOS architecture: ${values.arch}`); + const root = NodeURL.fileURLToPath(new URL("../../../native/apple-speech/", import.meta.url)); + const source = NodePath.resolve(root, "AppleSpeech.swift"); + const output = values.output ?? NodePath.resolve(root, "build", values.arch, "t3-apple-speech"); + let current = false; + try { + current = + NodeFS.statSync(output).mtimeMs >= + Math.max( + NodeFS.statSync(source).mtimeMs, + NodeFS.statSync(NodeURL.fileURLToPath(import.meta.url)).mtimeMs, + ); + } catch {} + if (!current) { + NodeFS.mkdirSync(NodePath.dirname(output), { recursive: true }); + const temporary = `${output}.${process.pid}.tmp`; + const slices = targets.map((target) => `${temporary}.${target}`); + try { + targets.forEach((target, index) => + NodeChildProcess.execFileSync( + "xcrun", + [ + "swiftc", + "-O", + "-swift-version", + "5", + "-parse-as-library", + "-target", + `${target}-apple-macos26.0`, + source, + "-o", + slices[index], + ], + { stdio: "inherit" }, + ), + ); + NodeChildProcess.execFileSync("xcrun", ["lipo", "-create", ...slices, "-output", temporary], { + stdio: "inherit", + }); + NodeFS.renameSync(temporary, output); + } catch (cause) { + if (required) + throw new Error("Building the Apple Speech helper requires Xcode 26 or later.", { cause }); + console.warn( + "Skipping the Apple Speech helper (needs Xcode 26 or later); on-device Mac dictation is unavailable in this build.", + ); + } finally { + for (const file of [temporary, ...slices]) NodeFS.rmSync(file, { force: true }); + } + } +} diff --git a/apps/desktop/scripts/dev-electron.mjs b/apps/desktop/scripts/dev-electron.mjs index 122da3f0512e5..aca0bf3ea26f2 100644 --- a/apps/desktop/scripts/dev-electron.mjs +++ b/apps/desktop/scripts/dev-electron.mjs @@ -60,6 +60,11 @@ NodeChildProcess.execFileSync( [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")], { stdio: "inherit" }, ); +NodeChildProcess.execFileSync( + process.execPath, + [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")], + { stdio: "inherit" }, +); await waitForResources({ baseDir: desktopDir, diff --git a/apps/desktop/scripts/start-electron.mjs b/apps/desktop/scripts/start-electron.mjs index 5dde034121b8e..8cabe3e5069fe 100644 --- a/apps/desktop/scripts/start-electron.mjs +++ b/apps/desktop/scripts/start-electron.mjs @@ -8,6 +8,11 @@ NodeChildProcess.execFileSync( [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")], { stdio: "inherit" }, ); +NodeChildProcess.execFileSync( + process.execPath, + [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")], + { stdio: "inherit" }, +); const childEnv = { ...process.env }; delete childEnv.ELECTRON_RUN_AS_NODE; diff --git a/apps/desktop/src/app/DesktopApp.test.ts b/apps/desktop/src/app/DesktopApp.test.ts index 10e0d1cd16c9b..980df0095777b 100644 --- a/apps/desktop/src/app/DesktopApp.test.ts +++ b/apps/desktop/src/app/DesktopApp.test.ts @@ -71,6 +71,7 @@ import * as ElectronTheme from "../electron/ElectronTheme.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as MacPermissions from "../permissions/MacPermissions.ts"; import * as PreviewManager from "../preview/Manager.ts"; +import * as AppleSpeech from "../speech/AppleSpeech.ts"; it.effect.each([false, true])( "stops backends together before flushing history and cancels Linux registration, stop defect=%s", @@ -235,6 +236,7 @@ it.effect.each([false, true])( Layer.mock(ElectronTheme.ElectronTheme, {}), Layer.mock(ElectronWindow.ElectronWindow, {}), Layer.mock(MacPermissions.MacPermissions, {}), + Layer.mock(AppleSpeech.AppleSpeech, {}), Layer.mock(PreviewManager.PreviewManager, { isBrowserPartition: () => { throw new Error("unexpected browser partition lookup"); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 9f2f68be61520..cd32423e6c894 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -1,6 +1,11 @@ import * as Effect from "effect/Effect"; import { receiveProviderAuthCallback, cancelProviderAuthCallback } from "./methods/providerAuth.ts"; +import { + appleSpeechAvailable, + appleSpeechPrepare, + appleSpeechTranscribe, +} from "./methods/appleSpeech.ts"; import * as DesktopIpc from "./DesktopIpc.ts"; import { installNotificationBadge } from "./methods/notificationBadge.ts"; import { getClientSettings, setClientSettings } from "./methods/clientSettings.ts"; @@ -155,6 +160,9 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(cancelProviderAuthCallback); yield* ipc.handle(openSystemSettings); yield* ipc.handle(checkSystemPermission); + yield* ipc.handle(appleSpeechAvailable); + yield* ipc.handle(appleSpeechPrepare); + yield* ipc.handle(appleSpeechTranscribe); yield* ipc.handle(pasteAsText); yield* ipc.handle(revealWindow); yield* ipc.handle(syncHtmlFullscreen); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 80053c78caee4..60e35f4c11da1 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -127,6 +127,9 @@ export const PREVIEW_FIND_RESULT_CHANNEL = "desktop:preview-find-result"; export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper"; export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission"; +export const APPLE_SPEECH_AVAILABLE_CHANNEL = "desktop:apple-speech-available"; +export const APPLE_SPEECH_PREPARE_CHANNEL = "desktop:apple-speech-prepare"; +export const APPLE_SPEECH_TRANSCRIBE_CHANNEL = "desktop:apple-speech-transcribe"; export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input"; diff --git a/apps/desktop/src/ipc/methods/appleSpeech.ts b/apps/desktop/src/ipc/methods/appleSpeech.ts new file mode 100644 index 0000000000000..8396db6f29b3d --- /dev/null +++ b/apps/desktop/src/ipc/methods/appleSpeech.ts @@ -0,0 +1,37 @@ +import { + DesktopAppleSpeechResultSchema, + DesktopAppleSpeechTranscribeInputSchema, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import * as AppleSpeech from "../../speech/AppleSpeech.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +export const appleSpeechAvailable = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, + payload: Schema.Void, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.appleSpeech.available")(function* () { + return yield* (yield* AppleSpeech.AppleSpeech).isAvailable; + }), +}); + +export const appleSpeechPrepare = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, + payload: Schema.String.check(Schema.isMaxLength(64)), + result: DesktopAppleSpeechResultSchema, + handler: Effect.fn("desktop.ipc.appleSpeech.prepare")(function* (locale) { + return yield* (yield* AppleSpeech.AppleSpeech).prepare(locale); + }), +}); + +export const appleSpeechTranscribe = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, + payload: DesktopAppleSpeechTranscribeInputSchema, + result: DesktopAppleSpeechResultSchema, + handler: Effect.fn("desktop.ipc.appleSpeech.transcribe")(function* ({ locale, pcm }) { + return yield* (yield* AppleSpeech.AppleSpeech).transcribe(locale, pcm); + }), +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 35156d645dc0a..ee20b0dc3b495 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -72,6 +72,7 @@ import * as DesktopUpdatePolicy from "./updates/DesktopUpdatePolicy.ts"; import * as DesktopUpdates from "./updates/DesktopUpdates.ts"; import * as BrowserImport from "./preview/BrowserImport/BrowserImport.ts"; import * as LinuxBrowserSecret from "./preview/BrowserImport/LinuxBrowserSecret.ts"; +import * as AppleSpeech from "./speech/AppleSpeech.ts"; import * as BrowserSession from "./preview/BrowserSession.ts"; import * as DesktopBrowserHost from "./preview/DesktopBrowserHost.ts"; import * as PreviewManager from "./preview/Manager.ts"; @@ -156,6 +157,7 @@ const layerDesktopFoundation = Layer.mergeAll( DesktopPrimaryPaste.layer, DesktopObservability.layer, DesktopRendererHistory.layer, + AppleSpeech.layer, ).pipe(Layer.provideMerge(layerDesktopEnvironment)); const layerDesktopSsh = layerDesktopSshEnvironment.pipe( diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index a538b821ff77c..219accdf178bc 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -255,6 +255,13 @@ contextBridge.exposeInMainWorld("desktopBridge", { readMedia: (url: string) => ipcRenderer.invoke(IpcChannels.READ_MEDIA_CHANNEL, url), checkSystemPermission: (pane: string) => ipcRenderer.invoke(IpcChannels.CHECK_SYSTEM_PERMISSION_CHANNEL, pane), + appleSpeech: { + isAvailable: () => ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined), + prepare: (locale: string) => + ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, locale), + transcribe: (input: { locale: string; pcm: Uint8Array }) => + ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, input), + }, openSystemSettings: (pane: string) => ipcRenderer.invoke(IpcChannels.OPEN_SYSTEM_SETTINGS_CHANNEL, pane), probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined), diff --git a/apps/desktop/src/speech/AppleSpeech.test.ts b/apps/desktop/src/speech/AppleSpeech.test.ts new file mode 100644 index 0000000000000..54eba0706e7af --- /dev/null +++ b/apps/desktop/src/speech/AppleSpeech.test.ts @@ -0,0 +1,189 @@ +import { assert, describe, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; +import * as AppleSpeech from "./AppleSpeech.ts"; + +const encoder = new TextEncoder(); +const text = (value: string) => (value ? Stream.make(encoder.encode(value)) : Stream.empty); + +type HelperOutput = { + stdout?: string; + stderr?: string; + exitCode: number; + exit?: Deferred.Deferred; +}; + +function makeProcess(result: HelperOutput) { + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: text(result.stdout ?? ""), + stderr: text(result.stderr ?? ""), + all: Stream.empty, + exitCode: (result.exit ? Deferred.await(result.exit) : Effect.void).pipe( + Effect.as(ChildProcessSpawner.ExitCode(result.exitCode)), + ), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); +} + +function setup(result: HelperOutput) { + const commands: Array> = []; + const written: Uint8Array[] = []; + const layer = AppleSpeech.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(DesktopEnvironment.DesktopEnvironment, { + platform: "darwin", + isPackaged: true, + resourcesPath: "/app/Resources", + path: { join: (...parts: string[]) => parts.join("/") }, + } as unknown as DesktopEnvironment.DesktopEnvironment["Service"]), + FileSystem.layerNoop({ + exists: () => Effect.succeed(true), + makeTempFileScoped: () => Effect.succeed("/tmp/recording.wav"), + writeFile: (_path, data) => Effect.sync(() => void written.push(data)), + }), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => { + const { command: executable, args } = command as unknown as { + command: string; + args: ReadonlyArray; + }; + commands.push([executable, ...args]); + return Effect.succeed(makeProcess(result)); + }), + ), + ), + ), + ); + return { commands, written, layer }; +} + +describe("AppleSpeech", () => { + it.effect.each([ + [ + { stdout: '{"value":"en-US"}', exitCode: 0 }, + { ok: true, value: "en-US" }, + ], + [ + { stderr: "Locale not supported: xx\n", exitCode: 2 }, + { ok: false, code: "unsupported-locale", message: "Locale not supported: xx" }, + ], + [{ exitCode: 3 }, { ok: false, code: "unavailable" }], + [ + { stderr: "boom", exitCode: 1 }, + { ok: false, code: "failed", message: "boom" }, + ], + [ + { stdout: "not json", exitCode: 0 }, + { ok: false, code: "failed" }, + ], + ] as const)( + "maps helper output %j to %j, preparing again only after a failure", + ([output, expected]) => { + const { layer, commands } = setup(output); + return Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + assert.deepInclude(yield* speech.prepare("en-US"), expected); + assert.deepInclude(yield* speech.prepare("en-US"), expected); + const command = ["/app/Resources/apple-speech/t3-apple-speech", "prepare", "en-US"]; + assert.deepEqual(commands, expected.ok ? [command] : [command, command]); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect("shares one helper between prepares that start while the first is running", () => + Effect.gen(function* () { + const exit = yield* Deferred.make(); + const { layer, commands } = setup({ stdout: '{"value":"en-US"}', exitCode: 0, exit }); + return yield* Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + const first = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + const second = yield* speech + .prepare("en-US") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.succeed(exit, undefined); + assert.deepEqual(yield* Fiber.join(first), { ok: true, value: "en-US" }); + assert.deepEqual(yield* Fiber.join(second), { ok: true, value: "en-US" }); + assert.lengthOf(commands, 1); + }).pipe(Effect.provide(layer)); + }), + ); + + it.effect.each([ + [0, true, 1], + [3, false, 1], + [1, false, 2], + ] as const)( + "isAvailable with helper exit %i is %s and probes %i time(s) for two calls", + ([exitCode, expected, probes]) => { + const { layer, commands } = setup({ stdout: '{"value":""}', exitCode }); + return Effect.gen(function* () { + const speech = yield* AppleSpeech.AppleSpeech; + assert.equal(yield* speech.isAvailable, expected); + assert.equal(yield* speech.isAvailable, expected); + assert.lengthOf(commands, probes); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect("transcribes Float32 PCM through a 16-bit WAV file", () => { + const { layer, commands, written } = setup({ stdout: '{"value":"hello"}', exitCode: 0 }); + // An unaligned Buffer view: Buffer.slice and subarray do not copy. + const pcm = new Uint8Array(17); + new DataView(pcm.buffer).setFloat32(1, 0, true); + new DataView(pcm.buffer).setFloat32(5, 1, true); + new DataView(pcm.buffer).setFloat32(9, -1, true); + new DataView(pcm.buffer).setFloat32(13, 2, true); + return Effect.gen(function* () { + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe( + "en-US", + Buffer.from(pcm.buffer).subarray(1), + ); + assert.deepEqual(result, { ok: true, value: "hello" }); + assert.deepEqual(commands, [ + [ + "/app/Resources/apple-speech/t3-apple-speech", + "transcribe", + "en-US", + "/tmp/recording.wav", + ], + ]); + const wav = new DataView(written[0]!.buffer); + assert.equal(new TextDecoder().decode(written[0]!.subarray(0, 4)), "RIFF"); + assert.equal(wav.getUint16(20, true), 1); + assert.equal(wav.getUint32(24, true), 16_000); + assert.equal(wav.getUint16(34, true), 16); + assert.equal(wav.getUint32(40, true), 8); + assert.deepEqual( + [0, 1, 2, 3].map((index) => wav.getInt16(44 + index * 2, true)), + [0, 32767, -32767, 32767], + ); + }).pipe(Effect.provide(layer)); + }); + + it.effect("rejects PCM that is not whole Float32 samples without running the helper", () => { + const { layer, commands } = setup({ exitCode: 0 }); + return Effect.gen(function* () { + const result = yield* (yield* AppleSpeech.AppleSpeech).transcribe("en-US", new Uint8Array(3)); + assert.deepInclude(result, { ok: false, code: "failed" }); + assert.lengthOf(commands, 0); + }).pipe(Effect.provide(layer)); + }); +}); diff --git a/apps/desktop/src/speech/AppleSpeech.ts b/apps/desktop/src/speech/AppleSpeech.ts new file mode 100644 index 0000000000000..b8ab710be3fff --- /dev/null +++ b/apps/desktop/src/speech/AppleSpeech.ts @@ -0,0 +1,231 @@ +import type { DesktopAppleSpeechResult } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; + +const SAMPLE_RATE = 16_000; +const MAX_PCM_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * 5 * 60; +// The first prepare for a language can download Apple's model. +const PREPARE_TIMEOUT = Duration.minutes(10); +// Matches the longest accepted recording, so analysis can never outlast its deadline. +const TRANSCRIBE_TIMEOUT = Duration.minutes(5); + +const decodeOutput = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ value: Schema.String })), +); + +/** 16 kHz mono Float32 PCM to a 16-bit WAV that AVAudioFile reads. */ +function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array { + // Float32Array needs a 4-byte-aligned offset; copy a view that is not (Buffer.slice is a view). + const aligned = + pcmBytes.byteOffset % Float32Array.BYTES_PER_ELEMENT === 0 + ? pcmBytes + : new Uint8Array(pcmBytes); + const samples = new Float32Array( + aligned.buffer, + aligned.byteOffset, + aligned.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + const wav = new Uint8Array(44 + samples.length * 2); + const view = new DataView(wav.buffer); + const ascii = (offset: number, text: string) => { + for (let index = 0; index < text.length; index++) + view.setUint8(offset + index, text.charCodeAt(index)); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + samples.length * 2, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, SAMPLE_RATE, true); + view.setUint32(28, SAMPLE_RATE * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, samples.length * 2, true); + samples.forEach((sample, index) => { + const clamped = Number.isFinite(sample) ? Math.max(-1, Math.min(1, sample)) : 0; + view.setInt16(44 + index * 2, Math.round(clamped * 0x7fff), true); + }); + return wav; +} + +export class AppleSpeech extends Context.Service< + AppleSpeech, + { + readonly isAvailable: Effect.Effect; + readonly prepare: (locale: string) => Effect.Effect; + readonly transcribe: ( + locale: string, + pcm: Uint8Array, + ) => Effect.Effect; + } +>()("@t3tools/desktop/speech/AppleSpeech") {} + +const unavailable: DesktopAppleSpeechResult = { + ok: false, + code: "unavailable", + message: "Apple Speech needs the T3 Code desktop app on macOS 26 or later.", +}; + +export const layer = Layer.effect( + AppleSpeech, + Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const hostEnvironment = yield* HostProcess.Environment; + + let helper: string | undefined; + if (environment.platform === "darwin") { + const relative = environment.path.join("apple-speech", "t3-apple-speech"); + const candidates = environment.isPackaged + ? [environment.path.join(environment.resourcesPath, relative)] + : [ + environment.path.join( + environment.rootDir, + "native", + "apple-speech", + "build", + environment.processArch, + "t3-apple-speech", + ), + ]; + for (const candidate of candidates) { + if (yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false))) { + helper = candidate; + break; + } + } + } + + const run = (args: readonly string[], timeout: Duration.Input) => + Effect.scoped( + Effect.gen(function* () { + if (helper === undefined) return unavailable; + const handle = yield* spawner.spawn( + ChildProcess.make(helper, args, { stdin: "ignore", env: hostEnvironment }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + handle.stdout.pipe(Stream.decodeText(), Stream.mkString), + handle.stderr.pipe(Stream.decodeText(), Stream.mkString), + handle.exitCode, + ], + { concurrency: "unbounded" }, + ); + const message = stderr.trim() || "Apple Speech failed."; + switch (Number(exitCode)) { + case 0: + return { ok: true, value: (yield* decodeOutput(stdout)).value } as const; + case 2: + return { ok: false, code: "unsupported-locale", message } as const; + case 3: + return unavailable; + default: + return { ok: false, code: "failed", message } as const; + } + }), + ).pipe( + // Leaving the scope kills a helper that outlives its timeout. + Effect.timeoutOrElse({ + duration: timeout, + orElse: () => + Effect.succeed({ + ok: false, + code: "failed", + message: "Apple Speech did not respond.", + }), + }), + Effect.catch((cause) => + Effect.succeed({ + ok: false, + code: "failed", + message: cause instanceof Error ? cause.message : "Apple Speech failed.", + }), + ), + ); + + // Only a definite answer is kept; a slow or failed probe is asked again next time. + let available: boolean | undefined; + const isAvailable = Effect.suspend(() => + available === undefined + ? run(["available"], Duration.seconds(10)).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (result.ok || result.code === "unavailable") available = result.ok; + }), + ), + Effect.map((result) => result.ok), + ) + : Effect.succeed(available), + ); + + // Each dictation start awaits prepare before the microphone opens, so a locale is + // prepared once. Callers share an in-flight prepare (one asset download), and a + // failure is dropped so the next start retries it. + const prepared = new Map>(); + const claimPermit = yield* Semaphore.make(1); + // One native analysis at a time, even if IPC callers overlap. + const transcribePermit = yield* Semaphore.make(1); + const claimPrepare = Effect.fn("AppleSpeech.claimPrepare")(function* (locale: string) { + const existing = prepared.get(locale); + if (existing !== undefined) return existing; + const shared = yield* Effect.cached( + run(["prepare", locale], PREPARE_TIMEOUT).pipe( + Effect.tap((result) => + Effect.sync(() => { + if (!result.ok) prepared.delete(locale); + }), + ), + ), + ); + prepared.set(locale, shared); + return shared; + }, claimPermit.withPermits(1)); + + return AppleSpeech.of({ + isAvailable, + prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) { + return yield* yield* claimPrepare(locale); + }), + transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) { + if ( + pcm.byteLength === 0 || + pcm.byteLength > MAX_PCM_BYTES || + pcm.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0 + ) + return { + ok: false, + code: "failed", + message: "The recording is empty or too long.", + } as const; + return yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fileSystem.makeTempFileScoped({ suffix: ".wav" }); + yield* fileSystem.writeFile(file, encodeSpeechWav(pcm)); + return yield* run(["transcribe", locale, file], TRANSCRIBE_TIMEOUT); + }), + ).pipe( + Effect.orElseSucceed((): DesktopAppleSpeechResult => ({ + ok: false, + code: "failed", + message: "Could not save the recording for Apple Speech.", + })), + transcribePermit.withPermits(1), + ); + }), + }); + }), +); diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 455901299edc3..4575be5f4b1cb 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -31,7 +31,7 @@ export default defineConfig({ tasks: { build: { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && vp pack", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && vp pack", dependsOn: ["t3#build"], cache: false, }, @@ -43,13 +43,13 @@ export default defineConfig({ }, dev: { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", dependsOn: ["t3#build"], cache: false, }, "dev:bundle": { command: - "node scripts/build-browser-secret.mjs && node scripts/build-preview-annotation-css.mjs && vp pack --watch", + "node scripts/build-browser-secret.mjs && node scripts/build-apple-speech.mjs && node scripts/build-preview-annotation-css.mjs && vp pack --watch", cache: false, }, "dev:electron": { diff --git a/apps/web/src/components/settings/VoiceSettingsPanel.tsx b/apps/web/src/components/settings/VoiceSettingsPanel.tsx index cfddfce72d154..57de26a44b346 100644 --- a/apps/web/src/components/settings/VoiceSettingsPanel.tsx +++ b/apps/web/src/components/settings/VoiceSettingsPanel.tsx @@ -38,6 +38,7 @@ import { DownloadIcon, LanguagesIcon, KeyRoundIcon, + LaptopIcon, MonitorIcon, MicOffIcon, MicIcon, @@ -52,6 +53,7 @@ import { import { useEffect, useRef, useState, type ReactNode } from "react"; import { useStableCallback } from "~/hooks/useStableCallback"; +import { useAppleSpeechAvailable } from "../../speech/appleSpeech"; import { cn } from "~/lib/utils"; import { listAudioInputs, @@ -137,6 +139,7 @@ import { searchableSetting } from "./settingsSearch"; import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext"; import { useScopedModelDisabledReason } from "./useScopedModelAvailability"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; +const DEVICE = "device"; const OFF = "off"; function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) { @@ -1192,6 +1195,7 @@ export function VoiceSettingsPanel() { const settings = useClientSettings(); const settingsScope = useOptionalSettingsScope(); const destination = useVoiceTranscriptionDestination(); + const appleSpeechAvailable = useAppleSpeechAvailable(); const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope); const [pendingCloud, setPendingCloud] = useState<{ provider: CloudSpeechProvider; @@ -1248,11 +1252,13 @@ export function VoiceSettingsPanel() { })), })); const cloudChoices = cloudGroups.flatMap((group) => group.choices); - const destinationValue = destination.cloudProvider - ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) - : destination.environmentId - ? JSON.stringify(["environment", destination.environmentId]) - : OFF; + const destinationValue = destination.device + ? DEVICE + : destination.cloudProvider + ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider]) + : destination.environmentId + ? JSON.stringify(["environment", destination.environmentId]) + : OFF; const cloudEnvironment = environments.find( (env) => env.environmentId === pendingCloud?.environmentId, ); @@ -1272,7 +1278,9 @@ export function VoiceSettingsPanel() { choice.value === value); if (value === OFF) chooseAudioDestination(null); + else if (value === DEVICE) chooseAudioDestination({ kind: "device" }); else if (chosen) chooseAudioDestination({ kind: "environment", @@ -1390,7 +1399,9 @@ export function VoiceSettingsPanel() { > - {destination.cloudProvider ? ( + {destination.device ? ( + + ) : destination.cloudProvider ? ( ) : destination.environmentId ? ( @@ -1398,11 +1409,15 @@ export function VoiceSettingsPanel() { )} - {destination.cloudProvider - ? CLOUD_SPEECH_LABELS[destination.cloudProvider] - : destination.environmentId - ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)") - : "Off"} + {destination.device + ? appleSpeechAvailable + ? "Apple Speech" + : "Apple Speech (unavailable)" + : destination.cloudProvider + ? CLOUD_SPEECH_LABELS[destination.cloudProvider] + : destination.environmentId + ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)") + : "Off"} {destination.cloudProvider && ( {" via "} @@ -1420,6 +1435,17 @@ export function VoiceSettingsPanel() { Off + {(appleSpeechAvailable || destination.device) && ( + + On this Mac + + + + {appleSpeechAvailable ? "Apple Speech" : "Apple Speech (unavailable)"} + + + + )} {environmentChoices.length > 0 && ( On an environment @@ -1501,20 +1527,24 @@ export function VoiceSettingsPanel() { close={() => setPendingCloud(null)} /> )} - {(model || destination.cloudProvider) && ( + {(model || destination.cloudProvider || destination.device) && ( } /> @@ -1538,14 +1568,16 @@ export function VoiceSettingsPanel() { {...searchableSetting("speech-silence-filtering")} title="Silence filtering" description={ - destination.cloudProvider === "openai" - ? "OpenAI handles silence itself." - : "Skip silence and background noise before transcribing." + destination.device + ? "Apple Speech handles silence itself." + : destination.cloudProvider === "openai" + ? "OpenAI handles silence itself." + : "Skip silence and background noise before transcribing." } control={ void updateClientSettings({ speechSilenceFiltering: checked }) @@ -1556,10 +1588,15 @@ export function VoiceSettingsPanel() { void updateClientSettings({ speechRemoveFillerWords: checked }) diff --git a/apps/web/src/speech/appleSpeech.test.ts b/apps/web/src/speech/appleSpeech.test.ts new file mode 100644 index 0000000000000..b8f7fcfc2e124 --- /dev/null +++ b/apps/web/src/speech/appleSpeech.test.ts @@ -0,0 +1,33 @@ +import { expect, it, vi } from "vite-plus/test"; + +it("probes Apple Speech again for a subscriber after a 'not available' answer", async () => { + const original = Object.getOwnPropertyDescriptor(globalThis, "window"); + const isAvailable = vi + .fn<() => Promise>() + .mockResolvedValueOnce(false) + .mockResolvedValue(true); + Object.defineProperty(globalThis, "window", { + value: { desktopBridge: { appleSpeech: { isAvailable } } }, + configurable: true, + }); + vi.useFakeTimers(); + let unsubscribe = () => {}; + try { + const { appleSpeechAvailability } = await import("./appleSpeech"); + const notify = vi.fn(); + unsubscribe = appleSpeechAvailability.subscribe(notify); + + await vi.advanceTimersByTimeAsync(0); + expect(notify).toHaveBeenCalledTimes(1); + expect(appleSpeechAvailability.getSnapshot()).toBe(false); + await vi.advanceTimersByTimeAsync(15_000); + expect(isAvailable).toHaveBeenCalledTimes(2); + expect(notify).toHaveBeenCalledTimes(2); + expect(appleSpeechAvailability.getSnapshot()).toBe(true); + } finally { + unsubscribe(); + vi.useRealTimers(); + if (original) Object.defineProperty(globalThis, "window", original); + else Reflect.deleteProperty(globalThis, "window"); + } +}); diff --git a/apps/web/src/speech/appleSpeech.ts b/apps/web/src/speech/appleSpeech.ts new file mode 100644 index 0000000000000..5d592a5d9d185 --- /dev/null +++ b/apps/web/src/speech/appleSpeech.ts @@ -0,0 +1,53 @@ +import { useSyncExternalStore } from "react"; + +let available = false; +let probe: Promise | undefined; +let retries = 0; +const listeners = new Set<() => void>(); +// The main process keeps only a definite answer, so a "no" may be a slow first probe. +const MAX_RETRIES = 3; +const RETRY_DELAY_MS = 15_000; + +function startProbe() { + const bridge = window.desktopBridge?.appleSpeech; + probe = (bridge?.isAvailable() ?? Promise.resolve(false)) + .catch(() => false) + .then((value) => { + available = value; + listeners.forEach((notify) => notify()); + if (value || !bridge || retries >= MAX_RETRIES) return; + retries++; + setTimeout(() => { + probe = undefined; + if (listeners.size > 0) startProbe(); + }, RETRY_DELAY_MS); + }); +} + +/** The store behind `useAppleSpeechAvailable`. */ +export const appleSpeechAvailability = { + subscribe(listener: () => void) { + listeners.add(listener); + if (!probe) startProbe(); + return () => listeners.delete(listener); + }, + getSnapshot: () => available, +}; + +/** Apple Speech runs only in the macOS 26 desktop app; the main process caches the answer. */ +export function useAppleSpeechAvailable() { + return useSyncExternalStore( + appleSpeechAvailability.subscribe, + appleSpeechAvailability.getSnapshot, + () => false, + ); +} + +/** The packaged app pins `navigator.language` to en-US, so "auto" asks the OS. */ +export function appleSpeechLocale(speechLanguage: string) { + if (speechLanguage !== "auto") return speechLanguage; + const bridge = window.desktopBridge; + return ( + bridge?.getPreferredSystemLanguages?.()[0] ?? bridge?.getSystemLocale?.() ?? navigator.language + ); +} diff --git a/apps/web/src/speech/browserVoiceInput.ts b/apps/web/src/speech/browserVoiceInput.ts index e6d164db59f3c..fc03ebdac8af1 100644 --- a/apps/web/src/speech/browserVoiceInput.ts +++ b/apps/web/src/speech/browserVoiceInput.ts @@ -15,6 +15,7 @@ import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/cont import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; +import { appleSpeechLocale } from "./appleSpeech"; import { createBrowserPcmCapture } from "./browserPcmCapture"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; @@ -125,3 +126,53 @@ export function createBrowserVoiceInputPlatform(input: { }; }); } + +/** Records like the environment path, then transcribes in the desktop main process; audio stays on this Mac. */ +export function createAppleSpeechVoiceInputPlatform(input: { + readonly hasAudioConsent: () => boolean; + readonly getTranscriptionOptions: () => SpeechTranscriptionOptions; + readonly getMicrophoneId: () => string | Promise; + readonly onLevel: (level: number) => void; + readonly onDurationLimit: () => void; + readonly onError: (message: string) => void; +}) { + return createBrowserPcmCapture(input, async ({ signal }) => { + const appleSpeech = window.desktopBridge?.appleSpeech; + if (!appleSpeech) + throw new VoiceTranscriptionError({ + code: "unavailable", + message: "Apple Speech needs the T3 Code desktop app on macOS 26 or later.", + }); + const prepared = await appleSpeech.prepare( + appleSpeechLocale(input.getTranscriptionOptions().speechLanguage), + ); + throwIfVoiceTranscriptionAborted(signal); + if (!prepared.ok) + throw new VoiceTranscriptionError({ + code: prepared.code === "failed" ? "preparation-failed" : prepared.code, + message: prepared.message, + }); + const locale = prepared.value; + return { + locale, + transcribe: async (uri, { signal: transcriptionSignal }) => { + const response = await fetch(uri, { signal: transcriptionSignal }); + const pcm = new Uint8Array(await response.arrayBuffer()); + throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (!input.hasAudioConsent()) + throw new VoiceTranscriptionError({ + code: "cancelled", + message: "Voice transcription was cancelled.", + }); + const result = await appleSpeech.transcribe({ locale, pcm }); + throwIfVoiceTranscriptionAborted(transcriptionSignal); + if (!result.ok) + throw new VoiceTranscriptionError({ + code: "transcription-failed", + message: result.message, + }); + return result.value; + }, + }; + }); +} diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx index d1f3c36d00f2c..131cc7a6a15cc 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx +++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx @@ -58,6 +58,7 @@ const mocks = vi.hoisted(() => ({ transcripts: [] as Array, committed: vi.fn(), draft: "", + appleSpeech: false, })); vi.mock("../state/session", () => ({ @@ -164,44 +165,51 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({ return { text: `Cleaned ${transcript}` }; }), })); +vi.mock("./appleSpeech", () => ({ useAppleSpeechAvailable: () => mocks.appleSpeech })); vi.mock("./browserVoiceInput", () => ({ - createBrowserVoiceInputPlatform: ( - input: Parameters[0], - ) => { - const environmentId = input.prepared.environmentId; - mocks.platforms.push(environmentId); - return { - recorder: { - uri: `blob:${environmentId}`, - prepareToRecordAsync: async () => { - mocks.microphoneRequests.push(environmentId); - }, - record() {}, - stop: async () => {}, + createAppleSpeechVoiceInputPlatform: ( + input: Omit[0], "prepared">, + ) => mockPlatform("apple-speech", input), + createBrowserVoiceInputPlatform: (input: Parameters[0]) => + mockPlatform(input.prepared.environmentId, input), +})); + +function mockPlatform( + environmentId: string, + input: Omit[0], "prepared">, +) { + mocks.platforms.push(environmentId); + return { + recorder: { + uri: `blob:${environmentId}`, + prepareToRecordAsync: async () => { + mocks.microphoneRequests.push(environmentId); }, - transcriber: { - prepare: async () => { - mocks.recordingOptions.push(input.getTranscriptionOptions()); - return { - locale: "en", - transcribe: async () => { - mocks.transcribeCalls.push( - input.cloudProvider - ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` - : environmentId, - ); - const next = mocks.transcripts.shift() ?? "dictated"; - if (next instanceof Error) throw next; - return next; - }, - }; - }, + record() {}, + stop: async () => {}, + }, + transcriber: { + prepare: async () => { + mocks.recordingOptions.push(input.getTranscriptionOptions()); + return { + locale: "en", + transcribe: async () => { + mocks.transcribeCalls.push( + input.cloudProvider + ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}` + : environmentId, + ); + const next = mocks.transcripts.shift() ?? "dictated"; + if (next instanceof Error) throw next; + return next; + }, + }; }, - cancelRecording() {}, - deleteRecording() {}, - }; - }, -})); + }, + cancelRecording() {}, + deleteRecording() {}, + }; +} const THREAD_ENV = "thread-env"; @@ -236,6 +244,7 @@ function setup(options: { readonly destination?: string | null } = {}) { mocks.transcripts = []; mocks.committed.mockClear(); mocks.draft = ""; + mocks.appleSpeech = false; } function setDestination(environmentId: string | null) { @@ -364,6 +373,29 @@ it.each([ expect(mocks.committed).not.toHaveBeenCalled(); }); +it.each([true, false])( + "transcribes on this Mac with Apple Speech, never an environment, when available is %s", + async (available) => { + setup(); + mocks.appleSpeech = available; + chooseAudioDestination({ kind: "device" }); + await using _screen = await render(); + expect(mocks.voice.transcriptionEnvironmentId).toBeNull(); + expect(mocks.voice.available).toBe(available); + await act(() => mocks.voice.start()); + if (!available) { + expectNoAudioSent(); + return; + } + await act(() => mocks.voice.stop()); + expect(new Set([...mocks.platforms, ...mocks.transcribeCalls])).toEqual( + new Set(["apple-speech"]), + ); + expect(mocks.statusRequests).toEqual([]); + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything()); + }, +); + it("retries a failed transcription on the same chosen environment", async () => { setup({ destination: "env-a" }); mocks.transcripts = [new Error("inference failed"), "second try"]; diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts index 26a6835c31044..fc0b930eccfae 100644 --- a/apps/web/src/speech/useEnvironmentSpeechInput.ts +++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts @@ -37,8 +37,12 @@ import { } from "../hooks/useSettings"; import { useEnvironment } from "../state/environments"; import { useEnvironmentScope, usePreparedConnection } from "../state/session"; +import { useAppleSpeechAvailable } from "./appleSpeech"; import { chooseRecordingMicrophone } from "./browserMicrophones"; -import { createBrowserVoiceInputPlatform } from "./browserVoiceInput"; +import { + createAppleSpeechVoiceInputPlatform, + createBrowserVoiceInputPlatform, +} from "./browserVoiceInput"; import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent"; @@ -112,12 +116,18 @@ type HookInput = { * Microphone audio leaves this device only for the environment the person chose in * Voice settings. Nothing is inferred from the primary or the draft's environment, and * the recording is abandoned the moment that choice changes or its connection drops. + * Apple Speech ("device") keeps audio on this Mac and needs no environment. */ export function useVoiceTranscriptionDestination() { const hydrated = useClientSettingsHydrated(); const choice = useAudioChoice(); const grant = choice === "off" ? null : choice; - const environmentId = hydrated && grant ? grant.destination.environmentId : null; + const appleSpeechAvailable = useAppleSpeechAvailable(); + const device = hydrated && grant?.destination.kind === "device"; + const environmentId = + hydrated && grant && grant.destination.kind !== "device" + ? grant.destination.environmentId + : null; const environment = useEnvironment(environmentId); const connection = Option.getOrNull(usePreparedConnection(environmentId)); const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); @@ -131,11 +141,13 @@ export function useVoiceTranscriptionDestination() { return { ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}), environmentId, + ...(device ? { device: true, onDevice: appleSpeechAvailable } : {}), ...(provider ? { cloudProvider: provider } : {}), ...(model ? { cloudModel: model } : {}), grantId: grant?.id ?? null, - label: - provider && environment + label: device + ? "Apple Speech" + : provider && environment ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}` : (environment?.label ?? null), prepared: supported && canDictate ? connection : null, @@ -146,6 +158,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { const navigate = useNavigate(); const destination = useVoiceTranscriptionDestination(); const prepared = destination.prepared; + const onDevice = destination.onDevice === true; const grantId = destination.grantId; const cloudProvider = destination.cloudProvider; const cloudModel = destination.cloudModel; @@ -227,7 +240,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent useEffect(() => { - if (!prepared || !grantId || typeof navigator === "undefined") return; + if ((!prepared && !onDevice) || !grantId || typeof navigator === "undefined") return; const hasAudioConsent = bindAudioGrant(grantId); let disposed = false; let controller: VoiceInputController; @@ -257,10 +270,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { revision: draftRevisionRef.current.revision, }; }; - const platform = createBrowserVoiceInputPlatform({ - prepared, - cloudProvider, - cloudModel, + const platformInput = { hasAudioConsent, getTranscriptionOptions: (): SpeechTranscriptionOptions => { const { @@ -300,13 +310,22 @@ export function useEnvironmentSpeechInput(input: HookInput) { getMicrophoneId: chooseRecordingMicrophone, onLevel: setLevel, onDurationLimit: () => void controller.stop(), - onText: (text) => { - if (!disposed) setPreview(text); - }, - onError: (message) => { + onError: (message: string) => { if (!disposed) void controller.interruptRecording(message); }, - }); + }; + const platform = + prepared && !onDevice + ? createBrowserVoiceInputPlatform({ + ...platformInput, + prepared, + cloudProvider, + cloudModel, + onText: (text) => { + if (!disposed) setPreview(text); + }, + }) + : createAppleSpeechVoiceInputPlatform(platformInput); const postProcess: NonNullable = ( transcript, options, @@ -374,7 +393,7 @@ export function useEnvironmentSpeechInput(input: HookInput) { controller.dispose(); if (controllerRef.current === controller) controllerRef.current = null; }; - }, [prepared, grantId, cloudProvider, cloudModel]); + }, [prepared, onDevice, grantId, cloudProvider, cloudModel]); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available useEffect(() => { @@ -486,7 +505,13 @@ export function useEnvironmentSpeechInput(input: HookInput) { const request = ++startRequestRef.current; const controller = controllerRef.current; const owner = latestRef.current.input.ownerKey; - if (!controller || !prepared) return; + if (!controller) return; + if (onDevice) { + setLevel(0); + await controller.start(); + return; + } + if (!prepared) return; if (cloudProvider) { setLevel(0); await controller.start(); @@ -594,9 +619,10 @@ export function useEnvironmentSpeechInput(input: HookInput) { const destinationReady = grantId === null ? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false - : prepared !== null && - (cloudProvider !== undefined || - (status?.prepared === prepared && status.value?.supported !== false)); + : onDevice || + (prepared !== null && + (cloudProvider !== undefined || + (status?.prepared === prepared && status.value?.supported !== false))); return { available: destinationReady && diff --git a/apps/web/src/speech/voiceConsent.ts b/apps/web/src/speech/voiceConsent.ts index 7c897fbc37ea5..2e7809b066de3 100644 --- a/apps/web/src/speech/voiceConsent.ts +++ b/apps/web/src/speech/voiceConsent.ts @@ -13,7 +13,13 @@ import { // Separate keys: a stale window saving ordinary preferences must not restore either grant. const AUDIO_KEY = "t3code:voice-audio-grant:v1"; const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1"; -const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }); +// "device" is Apple Speech in the macOS desktop app. Audio stays on this device, so it stays out of the shared contract. +const AudioDestination = Schema.Union([ + VoiceTranscriptionDestination, + Schema.Struct({ kind: Schema.Literal("device") }), +]); +export type AudioDestination = typeof AudioDestination.Type; +const Grant = Schema.Struct({ id: Schema.String, destination: AudioDestination }); const AudioGrant = Schema.NullOr(Grant); const sameGrant = Schema.toEquivalence(AudioGrant); const SavedChoice = Schema.Union([AudioGrant, Schema.Literal("off")]); @@ -62,7 +68,7 @@ function replaceChoice(choice: AudioChoice, previous = readChoice()) { window.localStorage.removeItem(savedKey(old.id)); } -export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) { +export function chooseAudioDestination(destination: AudioDestination | null) { replaceChoice(destination ? { id: randomUUID(), destination } : "off"); } diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index 00964ab4ae332..b560503641bd2 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -21,6 +21,12 @@ transcription share a recorder and upload only finished recordings. The client keeps the grant in device-bound secure storage, outside preference backups, and rechecks it after native decoding and before upload. +The macOS desktop app adds a `device` destination, Apple Speech. It exists only in +the web grant schema, not in `VoiceTranscriptionDestination`, because no environment +or other client can act on it. The desktop main process runs a +[Swift helper](../../native/apple-speech/AppleSpeech.swift) that the macOS release +runners compile; the Speech framework exists only in the macOS SDK. + The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another window saving unrelated settings cannot resurrect a revoked grant. Each new choice gets a distinct grant, even if the destination was previously selected. The diff --git a/docs/user/composer.md b/docs/user/composer.md index df880e49c6ca5..fb8b024a9601f 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -199,8 +199,9 @@ and use **Attach again** or remove the missing file before sending. ## Voice input on web and desktop Use the composer's microphone or `mod+shift+d` to dictate into your draft. Audio -is transcribed on an environment you choose in **Settings → Voice**; nothing is -sent until you choose one. See [Voice input](./voice-input.md). +is transcribed on an environment you choose in **Settings → Voice**, or on the Mac +with Apple Speech in the macOS desktop app; nothing is sent until you choose one. +See [Voice input](./voice-input.md). ## Voice input on mobile diff --git a/docs/user/faq.md b/docs/user/faq.md index 8d34de5e2ea45..48a5267a87d5c 100644 --- a/docs/user/faq.md +++ b/docs/user/faq.md @@ -77,7 +77,7 @@ Open **Settings → Worktrees → Cleanup** and turn on automatic worktree clean ## Is there voice input? -Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first. The transcript goes into your draft for review. See [Voice input](./voice-input.md). +Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first; on macOS 26 or later, the desktop app can also use Apple Speech on the Mac. The transcript goes into your draft for review. See [Voice input](./voice-input.md). ## Does it work on Windows, Mac, and Linux? diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md index 95ea15d707d00..50620408ecd41 100644 --- a/docs/user/voice-input.md +++ b/docs/user/voice-input.md @@ -24,6 +24,13 @@ Set it back to **Off** at any time; a recording in progress is discarded. If the chosen environment goes offline, voice input waits for it. T3 Code never substitutes another environment. +In the desktop app on macOS 26 or later, you can instead choose **Apple Speech** +under **On this Mac**. Audio stays on the Mac and macOS transcribes it. The first +recording in a language can take longer while macOS downloads its speech model. +**Language** set to automatic uses your Mac's language. Silence filtering and +filler-word removal are not available; the dictionary applies only when transcript +cleanup is on. + For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment you want to use. If it needs a key, enter one and choose **Save and enable**. You can also choose a [secret provider](secrets.md) instead of **Value** and enter a diff --git a/native/apple-speech/AppleSpeech.swift b/native/apple-speech/AppleSpeech.swift new file mode 100644 index 0000000000000..a315a189fa9ba --- /dev/null +++ b/native/apple-speech/AppleSpeech.swift @@ -0,0 +1,125 @@ +// Adapted from AppleTranscriptionImpl.swift in @react-native-ai/apple (MIT, Callstack), +// including this repo's patch, so the Mac and iPhone transcribe the same way. +// +// t3-apple-speech available -> {"value":""} +// t3-apple-speech prepare -> {"value":"en-US"} (resolved locale) +// t3-apple-speech transcribe -> {"value":"..."} (transcript) +// Exit codes are consumed by AppleSpeech.ts; failures print one message line to stderr. + +import AVFoundation +import Foundation +import Speech + +private enum ExitCode: Int32 { + case failure = 1 + case unsupportedLocale = 2 + case unavailable = 3 + case usage = 64 +} + +private struct Failure: Error { + let code: ExitCode + let message: String +} + +@main +struct AppleSpeech { + static func main() async { + let arguments = Array(CommandLine.arguments.dropFirst()) + do { + switch (arguments.first, arguments.count) { + case ("available", 1): + try requireAvailable() + try write("") + case ("prepare", 2): + try write(try await prepare(arguments[1])) + case ("transcribe", 3): + try write(try await transcribe(arguments[1], path: arguments[2])) + default: + throw Failure(code: .usage, message: "Unknown command.") + } + } catch let failure as Failure { + fail(failure.code, failure.message) + } catch { + fail(.failure, error.localizedDescription) + } + } + + private static func requireAvailable() throws { + guard SpeechTranscriber.isAvailable else { + throw Failure(code: .unavailable, message: "Speech transcription is not available on this Mac.") + } + } + + private static func makeTranscriber(for locale: Locale) -> SpeechTranscriber { + let preset = SpeechTranscriber.Preset.timeIndexedTranscriptionWithAlternatives + return SpeechTranscriber( + locale: locale, + transcriptionOptions: preset.transcriptionOptions, + reportingOptions: preset.reportingOptions.subtracting([.alternativeTranscriptions]), + attributeOptions: preset.attributeOptions + ) + } + + private static func prepare(_ identifier: String) async throws -> String { + try requireAvailable() + guard + let locale = await SpeechTranscriber.supportedLocale( + equivalentTo: Locale(identifier: identifier)) + else { + throw Failure(code: .unsupportedLocale, message: "Locale not supported: \(identifier)") + } + let transcriber = makeTranscriber(for: locale) + switch await AssetInventory.status(forModules: [transcriber]) { + case .installed: + break + case .supported, .downloading: + if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() + } + case .unsupported: + throw Failure( + code: .unsupportedLocale, message: "Assets not supported for locale: \(identifier)") + @unknown default: + throw Failure(code: .failure, message: "Unknown asset inventory status.") + } + return locale.identifier(.bcp47) + } + + private static func transcribe(_ identifier: String, path: String) async throws -> String { + try requireAvailable() + let audioFile = try AVAudioFile(forReading: URL(fileURLWithPath: path)) + let transcriber = makeTranscriber(for: Locale(identifier: identifier)) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let collector = Task { () throws -> [String] in + var texts: [String] = [] + for try await result in transcriber.results where result.isFinal { + let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { texts.append(text) } + } + return texts + } + do { + if let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) { + try await analyzer.finalizeAndFinish(through: lastSampleTime) + } else { + await analyzer.cancelAndFinishNow() + } + return try await collector.value.joined(separator: " ") + } catch { + collector.cancel() + await analyzer.cancelAndFinishNow() + _ = try? await collector.value + throw error + } + } + + private static func write(_ value: String) throws { + FileHandle.standardOutput.write(try JSONSerialization.data(withJSONObject: ["value": value])) + } + + private static func fail(_ code: ExitCode, _ message: String) -> Never { + FileHandle.standardError.write(Data((message + "\n").utf8)) + exit(code.rawValue) + } +} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 14e58a42a9fa1..e8f72e84c6c7a 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1042,6 +1042,23 @@ export const DesktopMediaReadSchema = Schema.Struct({ }); export type DesktopMediaRead = typeof DesktopMediaReadSchema.Type; +/** `value` is the resolved locale from `prepare` and the transcript from `transcribe`. */ +export const DesktopAppleSpeechResultSchema = Schema.Union([ + Schema.Struct({ ok: Schema.Literal(true), value: Schema.String }), + Schema.Struct({ + ok: Schema.Literal(false), + code: Schema.Literals(["unavailable", "unsupported-locale", "failed"]), + message: Schema.String, + }), +]); +export type DesktopAppleSpeechResult = typeof DesktopAppleSpeechResultSchema.Type; + +export const DesktopAppleSpeechTranscribeInputSchema = Schema.Struct({ + locale: Schema.String.check(Schema.isMaxLength(64)), + // 16 kHz mono Float32 PCM, at most five minutes. + pcm: Schema.Uint8Array, +}); + export const SystemSettingsPaneSchema = Schema.Literals(["full-disk-access"]); export type SystemSettingsPane = typeof SystemSettingsPaneSchema.Type; @@ -1176,6 +1193,12 @@ export interface DesktopBridge { onRequest: (listener: (request: DesktopAppActivationRequest) => void) => () => void; }; preview: DesktopPreviewBridge; + /** On-device dictation through macOS 26 SpeechAnalyzer; absent in older desktop builds. */ + appleSpeech?: { + isAvailable: () => Promise; + prepare: (locale: string) => Promise; + transcribe: (input: { locale: string; pcm: Uint8Array }) => Promise; + }; } /** Renderer callback invoked by Electron with a fresh user gesture before display-media capture. */ diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 92c0974b546da..271ea7154bb96 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -996,6 +996,8 @@ export const DESKTOP_FILE_EXCLUSIONS = [ "!apps/desktop/resources/browser-secret/**/*", "!apps/desktop/prod-resources/browser-secret", "!apps/desktop/prod-resources/browser-secret/**/*", + "!apps/desktop/prod-resources/apple-speech", + "!apps/desktop/prod-resources/apple-speech/**/*", "!apps/desktop/prod-resources/windows-server", "!apps/desktop/prod-resources/windows-server/**/*", "!apps/desktop/prod-resources/wsl-runtime.tar.gz", @@ -1118,6 +1120,9 @@ export const LINUX_CAPTURE_EXTRA_RESOURCES = [ export const LINUX_BROWSER_SECRET_EXTRA_RESOURCES = [ { from: "apps/desktop/prod-resources/browser-secret", to: "browser-secret" }, ] as const; +export const MAC_APPLE_SPEECH_EXTRA_RESOURCES = [ + { from: "apps/desktop/prod-resources/apple-speech", to: "apple-speech" }, +] as const; export interface MacPasskeySigningConfiguration { readonly appId: string; @@ -2460,6 +2465,31 @@ export const stageBrowserSecret = Effect.fn("stageBrowserSecret")(function* (inp ); }); +export const stageAppleSpeech = Effect.fn("stageAppleSpeech")(function* (input: { + readonly repoRoot: string; + readonly stageResourcesDir: string; + readonly platform: typeof BuildPlatform.Type; + readonly arch: typeof BuildArch.Type; + readonly verbose: boolean; +}) { + if (input.platform !== "mac") return; + const path = yield* Path.Path; + yield* runCommand( + ChildProcess.make( + "node", + [ + path.join(input.repoRoot, "apps/desktop/scripts/build-apple-speech.mjs"), + "--arch", + input.arch, + "--output", + path.join(input.stageResourcesDir, "apple-speech", "t3-apple-speech"), + ], + { cwd: input.repoRoot }, + ), + { label: "build macOS Apple Speech helper", verbose: input.verbose }, + ); +}); + const generateMacIconSet = Effect.fn("build-desktop-artifact.generateMacIconSet")(function* ( sourcePng: string, targetIcns: string, @@ -2804,6 +2834,7 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( ...DESKTOP_EXTRA_RESOURCES, ...(platform === "linux" ? LINUX_CAPTURE_EXTRA_RESOURCES : []), ...(platform === "linux" ? LINUX_BROWSER_SECRET_EXTRA_RESOURCES : []), + ...(platform === "mac" ? MAC_APPLE_SPEECH_EXTRA_RESOURCES : []), ...(platform === "win" ? WINDOWS_SERVER_EXTRA_RESOURCES : []), ...(platform === "win" && wslRuntimeBundled ? WSL_RUNTIME_EXTRA_RESOURCES : []), ], @@ -2849,6 +2880,8 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( "T3 Code lets commands run by terminals and coding agents use the camera.", NSMicrophoneUsageDescription: "T3 Code lets commands run by terminals and coding agents use the microphone.", + NSSpeechRecognitionUsageDescription: + "T3 Code transcribes your dictation on this Mac when you choose Apple Speech.", // macOS lists an app under Default web browser only when it opens web // pages as documents as well as http and https links (see protocols). CFBundleDocumentTypes: [ @@ -3796,6 +3829,13 @@ const buildDesktopArtifactUnlocked = Effect.fn("buildDesktopArtifactUnlocked")(f arch: options.arch, verbose: options.verbose, }); + yield* stageAppleSpeech({ + repoRoot, + stageResourcesDir, + platform: options.platform, + arch: options.arch, + verbose: options.verbose, + }); yield* assertPlatformBuildResources( options.platform,