feat(desktop): dictate on this Mac with Apple Speech

apps/desktop/scripts/build-apple-speech.mjs

@@ -0,0 +1,74 @@
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodePath from "node:path";
+import * as NodeURL from "node:url";
+import * as NodeUtil from "node:util";
+
+// oxlint-disable-next-line t3code/no-global-process-runtime -- The Swift compiler targets the actual host; this script has no Effect runtime.
+const hostArch = process.arch;
+// oxlint-disable-next-line t3code/no-global-process-runtime -- The Speech framework exists only in the macOS SDK.
+const hostPlatform = process.platform;
+
+const { values } = NodeUtil.parseArgs({
+  options: { output: { type: "string" }, arch: { type: "string", default: hostArch } },
+});
+// Packaging passes --output and must not ship without the helper; dev builds only lose the option.
+const required = values.output !== undefined;
+
+if (hostPlatform !== "darwin") {
+  if (required) throw new Error("The Apple Speech helper can only be built on macOS.");
+} else {
+  const targets = { arm64: ["arm64"], x64: ["x86_64"], universal: ["arm64", "x86_64"] }[
+    values.arch
+  ];
+  if (targets === undefined) throw new Error(`Unsupported macOS architecture: ${values.arch}`);
+  const root = NodeURL.fileURLToPath(new URL("../../../native/apple-speech/", import.meta.url));
+  const source = NodePath.resolve(root, "AppleSpeech.swift");
+  const output = values.output ?? NodePath.resolve(root, "build", values.arch, "t3-apple-speech");
+  let current = false;
+  try {
+    current =
+      NodeFS.statSync(output).mtimeMs >=
+      Math.max(
+        NodeFS.statSync(source).mtimeMs,
+        NodeFS.statSync(NodeURL.fileURLToPath(import.meta.url)).mtimeMs,
+      );
+  } catch {}
+  if (!current) {
+    NodeFS.mkdirSync(NodePath.dirname(output), { recursive: true });
+    const temporary = `${output}.${process.pid}.tmp`;
+    const slices = targets.map((target) => `${temporary}.${target}`);
+    try {
+      targets.forEach((target, index) =>
+        NodeChildProcess.execFileSync(
+          "xcrun",
+          [
+            "swiftc",
+            "-O",
+            "-swift-version",
+            "5",
+            "-parse-as-library",
+            "-target",
+            `${target}-apple-macos26.0`,
+            source,
+            "-o",
+            slices[index],
+          ],
+          { stdio: "inherit" },
+        ),
+      );
+      NodeChildProcess.execFileSync("xcrun", ["lipo", "-create", ...slices, "-output", temporary], {
+        stdio: "inherit",
+      });
+      NodeFS.renameSync(temporary, output);
+    } catch (cause) {
+      if (required)
+        throw new Error("Building the Apple Speech helper requires Xcode 26 or later.", { cause });
+      console.warn(
+        "Skipping the Apple Speech helper (needs Xcode 26 or later); on-device Mac dictation is unavailable in this build.",
+      );
+    } finally {
+      for (const file of [temporary, ...slices]) NodeFS.rmSync(file, { force: true });
+    }
+  }
+}

apps/desktop/scripts/dev-electron.mjs

@@ -60,6 +60,11 @@ NodeChildProcess.execFileSync(
   [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")],
   { stdio: "inherit" },
 );
+NodeChildProcess.execFileSync(
+  process.execPath,
+  [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")],
+  { stdio: "inherit" },
+);
 
 await waitForResources({
   baseDir: desktopDir,

apps/desktop/scripts/start-electron.mjs

@@ -8,6 +8,11 @@ NodeChildProcess.execFileSync(
   [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")],
   { stdio: "inherit" },
 );
+NodeChildProcess.execFileSync(
+  process.execPath,
+  [NodePath.join(desktopDir, "scripts/build-apple-speech.mjs")],
+  { stdio: "inherit" },
+);
 
 const childEnv = { ...process.env };
 delete childEnv.ELECTRON_RUN_AS_NODE;

apps/desktop/src/app/DesktopApp.test.ts

@@ -71,6 +71,7 @@ import * as ElectronTheme from "../electron/ElectronTheme.ts";
 import * as ElectronWindow from "../electron/ElectronWindow.ts";
 import * as MacPermissions from "../permissions/MacPermissions.ts";
 import * as PreviewManager from "../preview/Manager.ts";
+import * as AppleSpeech from "../speech/AppleSpeech.ts";
 
 it.effect.each([false, true])(
   "stops backends together before flushing history and cancels Linux registration, stop defect=%s",
@@ -235,6 +236,7 @@ it.effect.each([false, true])(
             Layer.mock(ElectronTheme.ElectronTheme, {}),
             Layer.mock(ElectronWindow.ElectronWindow, {}),
             Layer.mock(MacPermissions.MacPermissions, {}),
+            Layer.mock(AppleSpeech.AppleSpeech, {}),
             Layer.mock(PreviewManager.PreviewManager, {
               isBrowserPartition: () => {
                 throw new Error("unexpected browser partition lookup");

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -1,6 +1,11 @@
 import * as Effect from "effect/Effect";
 
 import { receiveProviderAuthCallback, cancelProviderAuthCallback } from "./methods/providerAuth.ts";
+import {
+  appleSpeechAvailable,
+  appleSpeechPrepare,
+  appleSpeechTranscribe,
+} from "./methods/appleSpeech.ts";
 import * as DesktopIpc from "./DesktopIpc.ts";
 import { installNotificationBadge } from "./methods/notificationBadge.ts";
 import { getClientSettings, setClientSettings } from "./methods/clientSettings.ts";
@@ -155,6 +160,9 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
   yield* ipc.handle(cancelProviderAuthCallback);
   yield* ipc.handle(openSystemSettings);
   yield* ipc.handle(checkSystemPermission);
+  yield* ipc.handle(appleSpeechAvailable);
+  yield* ipc.handle(appleSpeechPrepare);
+  yield* ipc.handle(appleSpeechTranscribe);
   yield* ipc.handle(pasteAsText);
   yield* ipc.handle(revealWindow);
   yield* ipc.handle(syncHtmlFullscreen);

apps/desktop/src/ipc/channels.ts

@@ -127,6 +127,9 @@ export const PREVIEW_FIND_RESULT_CHANNEL = "desktop:preview-find-result";
 export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";
 
 export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission";
+export const APPLE_SPEECH_AVAILABLE_CHANNEL = "desktop:apple-speech-available";
+export const APPLE_SPEECH_PREPARE_CHANNEL = "desktop:apple-speech-prepare";
+export const APPLE_SPEECH_TRANSCRIBE_CHANNEL = "desktop:apple-speech-transcribe";
 
 export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input";
 

apps/desktop/src/ipc/methods/appleSpeech.ts

@@ -0,0 +1,37 @@
+import {
+  DesktopAppleSpeechResultSchema,
+  DesktopAppleSpeechTranscribeInputSchema,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+import * as AppleSpeech from "../../speech/AppleSpeech.ts";
+import * as IpcChannels from "../channels.ts";
+import * as DesktopIpc from "../DesktopIpc.ts";
+
+export const appleSpeechAvailable = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL,
+  payload: Schema.Void,
+  result: Schema.Boolean,
+  handler: Effect.fn("desktop.ipc.appleSpeech.available")(function* () {
+    return yield* (yield* AppleSpeech.AppleSpeech).isAvailable;
+  }),
+});
+
+export const appleSpeechPrepare = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL,
+  payload: Schema.String.check(Schema.isMaxLength(64)),
+  result: DesktopAppleSpeechResultSchema,
+  handler: Effect.fn("desktop.ipc.appleSpeech.prepare")(function* (locale) {
+    return yield* (yield* AppleSpeech.AppleSpeech).prepare(locale);
+  }),
+});
+
+export const appleSpeechTranscribe = DesktopIpc.makeIpcMethod({
+  channel: IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL,
+  payload: DesktopAppleSpeechTranscribeInputSchema,
+  result: DesktopAppleSpeechResultSchema,
+  handler: Effect.fn("desktop.ipc.appleSpeech.transcribe")(function* ({ locale, pcm }) {
+    return yield* (yield* AppleSpeech.AppleSpeech).transcribe(locale, pcm);
+  }),
+});

apps/desktop/src/main.ts

@@ -72,6 +72,7 @@ import * as DesktopUpdatePolicy from "./updates/DesktopUpdatePolicy.ts";
 import * as DesktopUpdates from "./updates/DesktopUpdates.ts";
 import * as BrowserImport from "./preview/BrowserImport/BrowserImport.ts";
 import * as LinuxBrowserSecret from "./preview/BrowserImport/LinuxBrowserSecret.ts";
+import * as AppleSpeech from "./speech/AppleSpeech.ts";
 import * as BrowserSession from "./preview/BrowserSession.ts";
 import * as DesktopBrowserHost from "./preview/DesktopBrowserHost.ts";
 import * as PreviewManager from "./preview/Manager.ts";
@@ -156,6 +157,7 @@ const layerDesktopFoundation = Layer.mergeAll(
   DesktopPrimaryPaste.layer,
   DesktopObservability.layer,
   DesktopRendererHistory.layer,
+  AppleSpeech.layer,
 ).pipe(Layer.provideMerge(layerDesktopEnvironment));
 
 const layerDesktopSsh = layerDesktopSshEnvironment.pipe(

apps/desktop/src/preload.ts

@@ -255,6 +255,13 @@ contextBridge.exposeInMainWorld("desktopBridge", {
   readMedia: (url: string) => ipcRenderer.invoke(IpcChannels.READ_MEDIA_CHANNEL, url),
   checkSystemPermission: (pane: string) =>
     ipcRenderer.invoke(IpcChannels.CHECK_SYSTEM_PERMISSION_CHANNEL, pane),
+  appleSpeech: {
+    isAvailable: () => ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_AVAILABLE_CHANNEL, undefined),
+    prepare: (locale: string) =>
+      ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_PREPARE_CHANNEL, locale),
+    transcribe: (input: { locale: string; pcm: Uint8Array }) =>
+      ipcRenderer.invoke(IpcChannels.APPLE_SPEECH_TRANSCRIBE_CHANNEL, input),
+  },
   openSystemSettings: (pane: string) =>
     ipcRenderer.invoke(IpcChannels.OPEN_SYSTEM_SETTINGS_CHANNEL, pane),
   probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined),