@@ -995,6 +1018,7 @@ function SpeechModelBrowser(props: {
busy: controlsDisabled,
canCancel,
comparisonLanguages: selectedLanguages,
+ recommendation: recommendations.get(model.id),
benchmarkTier: status.benchmarkTier,
onDownload: () =>
void run(
diff --git a/apps/web/src/components/settings/speechModelBrowser.test.ts b/apps/web/src/components/settings/speechModelBrowser.test.ts
index 890d43cee048b..9a3f969b60da9 100644
--- a/apps/web/src/components/settings/speechModelBrowser.test.ts
+++ b/apps/web/src/components/settings/speechModelBrowser.test.ts
@@ -1,5 +1,6 @@
import type { EnvironmentSpeechModel } from "@t3tools/contracts";
import { expect, it } from "vite-plus/test";
+import { recommendSpeechModels } from "@t3tools/shared/speech";
import { filterSpeechModels } from "./speechModelBrowser";
@@ -31,9 +32,9 @@ const models: readonly EnvironmentSpeechModel[] = [
benchmark: {
accuracy: {
en: { grade: "A", errorPercent: 3.8 },
- fr: { grade: "B", errorPercent: 8 },
+ fr: { grade: "B", errorPercent: 7 },
},
- secondsPer30s: { cpu: 2, accelerated: 2 },
+ secondsPer30s: { accelerated: 2, cpu: 1 },
},
},
{
@@ -47,7 +48,7 @@ const models: readonly EnvironmentSpeechModel[] = [
en: { grade: "A", errorPercent: 3.2 },
fr: { grade: "B", errorPercent: 9 },
},
- secondsPer30s: { cpu: 1, accelerated: 1 },
+ secondsPer30s: { accelerated: 1, cpu: 3 },
},
},
{
@@ -58,7 +59,7 @@ const models: readonly EnvironmentSpeechModel[] = [
active: false,
state: "verifying",
languages: ["en"],
- benchmark: { accuracy: {}, secondsPer30s: { cpu: null, accelerated: null } },
+ benchmark: { accuracy: {}, secondsPer30s: { accelerated: null, cpu: null } },
},
];
@@ -100,3 +101,58 @@ it("combines text, all selected languages, and completed downloads without retai
).toEqual(["fast"]);
expect(filterSpeechModels(models, { ...options, search: "small" })).toEqual([]);
});
+
+it("ranks language recommendations before the rest without changing explicit sorts or search badges", () => {
+ const catalog = models.map((model) => ({
+ ...model,
+ active: false,
+ state: "downloadable" as const,
+ }));
+ const options = {
+ search: "",
+ languages: ["en"],
+ downloadedOnly: false,
+ sort: "recommended",
+ accuracyLanguage: "en",
+ benchmarkTier: "accelerated",
+ } as const;
+ expect(filterSpeechModels(catalog, options).map(({ id }) => id)).toEqual([
+ "fast",
+ "active",
+ "small",
+ "pending",
+ ]);
+ expect(
+ filterSpeechModels(catalog, { ...options, languages: ["en", "fr"] }).map(({ id }) => id),
+ ).toEqual(["small", "fast", "active"]);
+ expect(filterSpeechModels(catalog, { ...options, languages: [] }).map(({ id }) => id)).toEqual(
+ catalog.map(({ id }) => id),
+ );
+ expect(filterSpeechModels(catalog, { ...options, sort: "size" }).map(({ id }) => id)).toEqual([
+ "small",
+ "fast",
+ "active",
+ "pending",
+ ]);
+ expect(
+ filterSpeechModels(catalog, { ...options, sort: "speed", benchmarkTier: "cpu" }).map(
+ ({ id }) => id,
+ ),
+ ).toEqual(["small", "fast", "active", "pending"]);
+ expect(
+ filterSpeechModels(catalog, { ...options, sort: "accuracy", languages: ["en", "fr"] }).map(
+ ({ id }) => id,
+ ),
+ ).toEqual(["fast", "small", "active"]);
+ expect(
+ filterSpeechModels(catalog, { ...options, sort: "speed", languages: ["en", "fr"] }).map(
+ ({ id }) => id,
+ ),
+ ).toEqual(["fast", "small", "active"]);
+ expect(filterSpeechModels(catalog, { ...options, search: "small" }).map(({ id }) => id)).toEqual([
+ "small",
+ ]);
+ expect(
+ recommendSpeechModels(catalog, options.languages, options.benchmarkTier).has("small"),
+ ).toBe(false);
+});
diff --git a/apps/web/src/components/settings/speechModelBrowser.ts b/apps/web/src/components/settings/speechModelBrowser.ts
index e08fcde0ae949..f0acb6a45e1c2 100644
--- a/apps/web/src/components/settings/speechModelBrowser.ts
+++ b/apps/web/src/components/settings/speechModelBrowser.ts
@@ -1,4 +1,5 @@
import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
+import { recommendSpeechModels } from "@t3tools/shared/speech";
export type SpeechModelSort = "recommended" | "size" | "accuracy" | "speed";
@@ -6,7 +7,7 @@ const modelSortOrder = (model: EnvironmentSpeechModel) => {
if (model.active) return 0;
if (model.state === "installed") return 1;
if (model.state === "downloading" || model.state === "verifying") return 2;
- return model.recommended ? 3 : 4;
+ return 3;
};
export function filterSpeechModels(
@@ -17,10 +18,12 @@ export function filterSpeechModels(
readonly downloadedOnly: boolean;
readonly sort: SpeechModelSort;
readonly accuracyLanguage: string;
- readonly benchmarkTier: SpeechBenchmarkTier;
+ readonly benchmarkTier: SpeechBenchmarkTier | undefined;
},
+ recommendations = recommendSpeechModels(models, options.languages, options.benchmarkTier),
) {
const query = options.search.trim().toLocaleLowerCase();
+ const ranks = new Map([...recommendations.keys()].map((id, index) => [id, index]));
const score = (model: EnvironmentSpeechModel) => {
switch (options.sort) {
case "size":
@@ -28,9 +31,11 @@ export function filterSpeechModels(
case "accuracy":
return model.benchmark?.accuracy[options.accuracyLanguage]?.errorPercent ?? Infinity;
case "speed":
- return model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity;
+ return options.benchmarkTier === undefined
+ ? Infinity
+ : (model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity);
case "recommended":
- return modelSortOrder(model);
+ return modelSortOrder(model) * 4 + (ranks.get(model.id) ?? 3);
}
};
return models
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
index b5418bcb409da..63614c4d09773 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
@@ -40,6 +40,7 @@ const mocks = vi.hoisted(() => ({
primaryEnvironmentId: "primary-env",
statusRequests: [] as string[],
statusUnavailable: false,
+ statusTier: "accelerated" as "cpu" | "accelerated",
modelState: "ready" as "ready" | "missing-model",
statusModelId: "test-model",
models: [] as EnvironmentSpeechModel[],
@@ -128,9 +129,9 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
supportsStreaming: false,
supportsTranslation: false,
acceleration: "auto",
+ benchmarkTier: mocks.statusTier,
modelUnloadTimeout: "min_15",
gpuDevices: [],
- benchmarkTier: "cpu",
};
}),
downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
@@ -145,10 +146,14 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
mocks.statusModelId = modelId;
return { models: mocks.models };
}),
- getEnvironmentSpeechModels: vi.fn(() =>
+ getEnvironmentSpeechModels: vi.fn((_prepared: PreparedConnection, includeBenchmarks = false) =>
Effect.sync(() => {
if (mocks.catalogOffline) throw new Error("Catalog offline");
- return { models: mocks.models };
+ return {
+ models: mocks.models.map(({ benchmark, ...model }) =>
+ includeBenchmarks ? { ...model, ...(benchmark ? { benchmark } : {}) } : model,
+ ),
+ };
}),
),
postProcessEnvironmentTranscript: (
@@ -219,6 +224,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
mocks.statusRequests = [];
mocks.statusUnavailable = false;
mocks.modelState = "ready";
+ mocks.statusTier = "accelerated";
mocks.statusModelId = "test-model";
mocks.models = [];
mocks.catalogOffline = false;
@@ -412,6 +418,7 @@ function catalogModel(
id: string,
languages: readonly string[],
recommended: boolean,
+ errorPercent = 4,
): EnvironmentSpeechModel {
return {
id,
@@ -422,6 +429,12 @@ function catalogModel(
accuracy: 0,
speed: 0,
recommended,
+ benchmark: {
+ accuracy: Object.fromEntries(
+ languages.map((language) => [language, { grade: "A", errorPercent }]),
+ ),
+ secondsPer30s: { accelerated: 2, cpu: 3 },
+ },
supportsStreaming: false,
supportsLanguageDetection: languages.length > 1,
supportsTranslation: false,
@@ -431,26 +444,30 @@ function catalogModel(
}
it.each([
- ["the browser prefers German", ["de-DE"], undefined, "auto"],
- ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
+ ["the browser prefers German", ["de-DE"], undefined, "auto", "accelerated"],
+ ["the remote environment uses CPU", ["de-DE"], undefined, "auto", "cpu"],
+ ["the desktop exposes no locale methods", ["de-DE"], {}, "auto", "accelerated"],
[
"an older preload exposes only the OS locale",
["en-US"],
{ getSystemLocale: () => "de-DE" },
"auto",
+ "accelerated",
],
[
"the desktop OS prefers German",
["en-US"],
{ getPreferredSystemLanguages: () => ["de-DE"] },
"auto",
+ "accelerated",
],
- ["German is the chosen Voice language", ["en-US"], undefined, "de"],
+ ["German is the chosen Voice language", ["en-US"], undefined, "de", "accelerated"],
])(
"offers and downloads a German-capable model when %s",
- async (_, languages, bridge, speechLanguage) => {
+ async (_, languages, bridge, speechLanguage, tier) => {
setup({ destination: "env-a" });
mocks.settings = { speechLanguage };
+ mocks.statusTier = tier as "cpu" | "accelerated";
using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
using _bridge = stubGlobal("desktopBridge", bridge);
const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
@@ -459,16 +476,24 @@ it.each([
mocks.statusModelId = englishDefault;
mocks.models = [
{ ...catalogModel(englishDefault, ["en"], true), active: true },
- catalogModel(german, ["en", "de", "fr"], true),
catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
+ {
+ ...catalogModel(german, ["en", "de", "fr"], false, 2),
+ benchmark: {
+ accuracy: { de: { grade: "A", errorPercent: 2 } },
+ secondsPer30s: { cpu: 20, accelerated: 2 },
+ },
+ },
];
mocks.download.mockResolvedValue(undefined);
await using _screen = await render(
);
await act(() => mocks.voice.start());
expect(mocks.voice.setup.open).toBe(true);
await act(() => mocks.voice.setup.download());
- expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
- expect(mocks.voice.setup.model?.id).toBe(german);
+ const expected = tier === "cpu" ? "handy-computer/canary-180m-flash-gguf" : german;
+ expect(mocks.download).toHaveBeenCalledExactlyOnceWith(expected);
+ expect(mocks.voice.setup.model?.id).toBe(expected);
+ expect(mocks.voice.setup.model?.recommended).toBe(true);
},
);
@@ -528,6 +553,7 @@ it.each([true, false])(
mocks.models.push(catalogModel("german", ["de"], true));
// Another client can finish setup while the catalog is offline.
mocks.modelState = "ready";
+ mocks.statusTier = "accelerated";
await userEvent.click(retry);
await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
await expect.element(download).toBeEnabled();
@@ -562,6 +588,7 @@ it.each([
mocks.download.mockImplementation(async () => {
mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
mocks.modelState = "ready";
+ mocks.statusTier = "accelerated";
});
await using _screen = await render(
);
await act(() => mocks.voice.start());
diff --git a/apps/web/src/speech/useEnvironmentSpeechInput.ts b/apps/web/src/speech/useEnvironmentSpeechInput.ts
index 94dd7cfb8b4cf..c3390024cf6fd 100644
--- a/apps/web/src/speech/useEnvironmentSpeechInput.ts
+++ b/apps/web/src/speech/useEnvironmentSpeechInput.ts
@@ -21,6 +21,7 @@ import type {
SpeechPostProcessingOptions,
SpeechStreamText,
SpeechTranscriptionOptions,
+ SpeechBenchmarkTier,
} from "@t3tools/contracts";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
@@ -379,7 +380,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
.then((result) => {
if (disposed) return;
const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
- if (model) setSetup((current) => ({ ...current, model }));
+ if (model)
+ setSetup((current) => ({
+ ...current,
+ model: { ...model, recommended: current.model?.recommended ?? false },
+ }));
})
.catch(() => undefined)
.finally(() => {
@@ -424,20 +429,26 @@ export function useEnvironmentSpeechInput(input: HookInput) {
});
const loadSetupCatalog = useStableCallback(
- async (connection: PreparedConnection, selectedModelId: string, request: number) => {
+ async (
+ connection: PreparedConnection,
+ selectedModelId: string,
+ benchmarkTier: SpeechBenchmarkTier,
+ request: number,
+ ) => {
setSetup({ ...INITIAL_SETUP, open: true });
- const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
- () => null,
- );
+ const catalog = await runEnvironmentSpeech(
+ getEnvironmentSpeechModels(connection, true),
+ ).catch(() => null);
if (request !== startRequestRef.current) return;
const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
selectedModelId,
intent: latestRef.current.clientSettings.speechLanguage,
preferred: preferredSpeechLanguages(),
+ benchmarkTier,
});
setSetup((current) => ({
...current,
- model: choice?.model ?? null,
+ model: choice ? { ...choice.model, recommended: choice.recommended } : null,
language: choice?.language ?? null,
catalogFailed: catalog === null,
error: choice
@@ -489,7 +500,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
setStatus({ prepared, value: latestStatus });
setQueuedStart(null);
if (latestStatus.supported && latestStatus.state === "missing-model") {
- return loadSetupCatalog(prepared, latestStatus.modelId, request);
+ return loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
}
if (!latestStatus.supported) {
toastManager.add({
@@ -584,7 +595,13 @@ export function useEnvironmentSpeechInput(input: HookInput) {
setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
retryCatalog:
setup.open && setup.catalogFailed && prepared && currentStatus?.supported
- ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
+ ? () =>
+ loadSetupCatalog(
+ prepared,
+ currentStatus.modelId,
+ currentStatus.benchmarkTier,
+ ++startRequestRef.current,
+ )
: null,
download: downloadSetupModel,
cancelDownload: cancelSetupDownload,
diff --git a/docs/user/voice-input.md b/docs/user/voice-input.md
index 592e110c896cc..76a6112b4bedb 100644
--- a/docs/user/voice-input.md
+++ b/docs/user/voice-input.md
@@ -66,8 +66,10 @@ format, such as Core ML, ONNX, or MLX builds of the same model, can’t be used.
Select the languages you dictate in to compare models that support all of them.
Accuracy shows how often the model got words or characters wrong in each
-language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
-**All languages** to browse the full catalog. Filtering does not switch your model.
+language; speed is the time to transcribe 30 seconds on a reference CPU or GPU,
+matching the selected environment's transcription backend. Recommendations use
+the same reference tier for every selected language. Choose **All languages** to
+browse without a language preference. Filtering does not switch your model.
**Advanced** controls GPU or CPU use and when an idle model is unloaded. With hardware set to
**Auto**, a recording that fails on the GPU is retried once on that same
diff --git a/packages/shared/src/speech.test.ts b/packages/shared/src/speech.test.ts
index 6d1829fd67a91..3d02b3d08ce48 100644
--- a/packages/shared/src/speech.test.ts
+++ b/packages/shared/src/speech.test.ts
@@ -3,6 +3,7 @@ import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "
import {
chooseSetupSpeechModel,
+ recommendSpeechModels,
decodeSpeechPcmRequest,
encodeSpeechPcmRequest,
resolveSpeechVocabulary,
@@ -79,7 +80,18 @@ describe("chooseSetupSpeechModel", () => {
languages: string[],
supportsLanguageDetection: boolean,
recommended: boolean,
- ) => ({ id, languages, supportsLanguageDetection, recommended });
+ ) => ({
+ id,
+ languages,
+ supportsLanguageDetection,
+ recommended,
+ benchmark: {
+ accuracy: Object.fromEntries(
+ languages.map((language) => [language, { errorPercent: recommended ? 4 : 10 }]),
+ ),
+ secondsPer30s: { accelerated: 2, cpu: 3 },
+ },
+ });
const catalog = [
model("unrecommended", ["en", "de", "es"], true, false),
model("english", ["en"], false, true),
@@ -146,7 +158,173 @@ describe("chooseSetupSpeechModel", () => {
undefined,
],
] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
- const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
+ const choice = chooseSetupSpeechModel(catalog, {
+ selectedModelId,
+ intent,
+ preferred,
+ benchmarkTier: "accelerated",
+ });
expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
});
+
+ it("keeps suitable selections and preference order, and falls back safely without measurements", () => {
+ const best = {
+ ...catalog[0]!,
+ benchmark: {
+ accuracy: { de: { errorPercent: 1 } },
+ secondsPer30s: { accelerated: 1, cpu: 2 },
+ },
+ };
+ expect(
+ chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
+ benchmarkTier: "accelerated",
+ selectedModelId: "gone",
+ intent: "auto",
+ preferred: ["de", "en"],
+ }),
+ ).toMatchObject({ model: { id: "best" }, language: "de", recommended: true });
+ expect(
+ chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
+ benchmarkTier: "accelerated",
+ selectedModelId: "detecting",
+ intent: "auto",
+ preferred: ["de", "en"],
+ }),
+ ).toMatchObject({ model: { id: "detecting" }, language: "de", recommended: false });
+ const unmeasured = catalog.map(({ benchmark: _benchmark, ...model }) => model);
+ expect(
+ chooseSetupSpeechModel(unmeasured, {
+ benchmarkTier: "accelerated",
+ selectedModelId: "gone",
+ intent: "auto",
+ preferred: ["de", "en"],
+ }),
+ ).toMatchObject({ model: { id: "unrecommended" }, language: "de", recommended: false });
+ expect(
+ chooseSetupSpeechModel([], {
+ benchmarkTier: "accelerated",
+ selectedModelId: "gone",
+ intent: "auto",
+ preferred: [],
+ }),
+ ).toBeUndefined();
+ });
+});
+
+describe("recommendSpeechModels", () => {
+ const model = (
+ id: string,
+ errors: Record
,
+ wait: number | null,
+ cpu: number | null,
+ ) => ({
+ id,
+ languages: Object.keys(errors),
+ benchmark: {
+ accuracy: Object.fromEntries(
+ Object.entries(errors).map(([language, errorPercent]) => [language, { errorPercent }]),
+ ),
+ secondsPer30s: { accelerated: wait, cpu },
+ },
+ });
+ const models = [
+ model("balanced", { en: 4, de: 16 }, 2, 7),
+ model("fast", { en: 7, de: 7 }, 1, 3),
+ model("accurate", { en: 2, de: 2 }, 8, 20),
+ model("incomplete", { en: 1 }, 1, 1),
+ ];
+
+ it("balances all requested languages and assigns each model its first role", () => {
+ expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "accelerated")]).toEqual([
+ ["balanced", "recommended"],
+ ["fast", "faster"],
+ ["accurate", "accurate"],
+ ]);
+ expect([...recommendSpeechModels(models, ["en", "de", "de"], "accelerated")]).toEqual([
+ ["fast", "recommended"],
+ ["accurate", "accurate"],
+ ]);
+ expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "cpu")]).toEqual([
+ ["fast", "recommended"],
+ ["balanced", "accurate"],
+ ]);
+ expect([...recommendSpeechModels(models, ["en", "de"], "cpu")]).toEqual([
+ ["fast", "recommended"],
+ ]);
+ });
+
+ it("offers alternatives only when they improve on the balanced pick on the same tier", () => {
+ expect([
+ ...recommendSpeechModels(
+ [model("cpu-fast-gpu-slow", { en: 3 }, 4, 1), model("balanced", { en: 3.2 }, 1, 8)],
+ ["en"],
+ "accelerated",
+ ),
+ ]).toEqual([
+ ["balanced", "recommended"],
+ ["cpu-fast-gpu-slow", "accurate"],
+ ]);
+ expect([
+ ...recommendSpeechModels(
+ [model("equal-accuracy", { en: 4 }, 8, 20), model("balanced", { en: 4 }, 2, 20)],
+ ["en"],
+ "accelerated",
+ ),
+ ]).toEqual([["balanced", "recommended"]]);
+ expect([
+ ...recommendSpeechModels(
+ [model("balanced", { en: 4 }, 12, 20), model("outside-budget", { en: 2 }, 30, 40)],
+ ["en"],
+ "accelerated",
+ ),
+ ]).toEqual([["balanced", "recommended"]]);
+ });
+
+ it("does not invent measurements, preferences, or a model covering missing languages", () => {
+ expect(recommendSpeechModels(models, [], "accelerated").size).toBe(0);
+ expect(recommendSpeechModels(models, ["ja"], "accelerated").size).toBe(0);
+ expect(
+ recommendSpeechModels(
+ [
+ model("missing-speed", { en: 1, de: 1 }, null, 1),
+ { ...model("missing-accuracy", { en: 1 }, 1, 1), languages: ["en", "de"] },
+ ],
+ ["en", "de"],
+ "accelerated",
+ ).size,
+ ).toBe(0);
+ expect([
+ ...recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "accelerated"),
+ ]).toEqual([["no-cpu", "recommended"]]);
+ expect(recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "cpu").size).toBe(
+ 0,
+ );
+ expect([
+ ...recommendSpeechModels([model("cpu-only", { en: 1 }, null, 2)], ["en"], "cpu"),
+ ]).toEqual([["cpu-only", "recommended"]]);
+ });
+
+ it("prefers useful accuracy before speed and relaxes the wait budget before quality", () => {
+ expect([
+ ...recommendSpeechModels(
+ [model("poor", { en: 25 }, 1, 1), model("slow", { en: 4 }, 12, 20)],
+ ["en"],
+ "accelerated",
+ ),
+ ]).toEqual([["slow", "recommended"]]);
+ expect([
+ ...recommendSpeechModels(
+ [model("worse", { en: 30, de: 40 }, 1, 1), model("least-poor", { en: 29, de: 30 }, 20, 40)],
+ ["en", "de"],
+ "accelerated",
+ ),
+ ]).toEqual([["least-poor", "recommended"]]);
+ expect([
+ ...recommendSpeechModels(
+ [model("zero", { en: 0, de: 0 }, 1, 1)],
+ ["en", "de"],
+ "accelerated",
+ ),
+ ]).toEqual([["zero", "recommended"]]);
+ });
});
diff --git a/packages/shared/src/speech.ts b/packages/shared/src/speech.ts
index c5ed9ab14ea0a..94c2eb316faae 100644
--- a/packages/shared/src/speech.ts
+++ b/packages/shared/src/speech.ts
@@ -3,13 +3,102 @@ import {
type CloudSpeechProvider,
SPEECH_MAX_OPTIONS_BYTES,
type SpeechCustomWords,
+ type SpeechBenchmarkTier,
SpeechTranscriptionOptions,
} from "@t3tools/contracts";
import * as Schema from "effect/Schema";
+import recommendationPolicy from "./speechRecommendationPolicy.json" with { type: "json" };
const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
+export type SpeechRecommendationRole = "recommended" | "faster" | "accurate";
+type RecommendationModel = {
+ readonly id: string;
+ readonly languages: readonly string[];
+ readonly benchmark?: {
+ readonly accuracy: Readonly>;
+ readonly secondsPer30s: {
+ readonly cpu?: number | null;
+ readonly accelerated: number | null;
+ };
+ };
+};
+
+/** Reference accuracy/speed picks; no preference or incomplete measurements get no badge. */
+export function recommendSpeechModels(
+ models: readonly RecommendationModel[],
+ languages: readonly string[],
+ tier: SpeechBenchmarkTier | undefined,
+) {
+ const wanted = [...new Set(languages)];
+ const roles = new Map();
+ if (wanted.length === 0 || tier === undefined) return roles;
+ const policy = recommendationPolicy;
+ const scored = models.flatMap((model) => {
+ if (!wanted.every((language) => model.languages.includes(language))) return [];
+ const errors = wanted.map((language) => model.benchmark?.accuracy[language]?.errorPercent);
+ const wait = model.benchmark?.secondsPer30s[tier];
+ if (
+ wait == null ||
+ !Number.isFinite(wait) ||
+ wait < 0 ||
+ errors.some((error) => error === undefined || !Number.isFinite(error) || error < 0)
+ )
+ return [];
+ const measured = errors as number[];
+ const error = Math.exp(
+ measured.reduce((sum, value) => sum + Math.log(value), 0) / measured.length,
+ );
+ return [
+ {
+ id: model.id,
+ error,
+ worst: Math.max(...measured),
+ wait,
+ cost: error + policy.waitWeight * Math.max(0, wait - policy.freeWaitSeconds),
+ },
+ ];
+ });
+ if (scored.length === 0) return roles;
+ const useful = scored.filter((model) => model.worst < policy.maxLanguageErrorPercent);
+ const pool = useful.length ? useful : scored;
+ const lowest = (items: typeof scored, score: (item: (typeof scored)[number]) => number) =>
+ items.reduce<(typeof scored)[number] | undefined>(
+ (best, item) => (best === undefined || score(item) < score(best) ? item : best),
+ undefined,
+ );
+ const withinBudget = pool.filter((model) => model.wait <= policy.overallMaxWaitSeconds);
+ const best = (
+ useful.length
+ ? lowest(withinBudget.length ? withinBudget : pool, (model) => model.cost)
+ : lowest(pool, (model) => model.worst + model.error / 1000)
+ )!;
+ const quicker = pool.filter((model) => model.wait < best.wait);
+ const fast = useful.length
+ ? (lowest(
+ quicker.filter((model) => model.wait <= policy.fastMaxWaitSeconds),
+ (model) => model.error,
+ ) ?? lowest(quicker, (model) => model.wait + model.error / 1000))
+ : undefined;
+ const accurate = useful.length
+ ? lowest(
+ pool.filter(
+ (model) => model.error < best.error && model.wait <= policy.accurateMaxWaitSeconds,
+ ),
+ (model) => model.error,
+ )
+ : undefined;
+ for (const [model, role] of [
+ [best, "recommended"],
+ [fast, "faster"],
+ [accurate, "accurate"],
+ ] as const) {
+ if (model && !roles.has(model.id)) roles.set(model.id, role);
+ }
+ return roles;
+}
+
export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) {
return provider === "openrouter"
? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`
@@ -37,16 +126,13 @@ export function effectiveSpeechLanguage(
}
/**
- * The model first-run setup offers: the selected one, else a recommended one, else any that the
+ * The model first-run setup offers: the selected one, else the benchmark pick, else any that the
* server would really transcribe the person's language with. Without a language hint,
* offer only the selected model, without a recommendation.
*/
export function chooseSetupSpeechModel<
- Model extends {
- readonly id: string;
- readonly languages: readonly string[];
+ Model extends RecommendationModel & {
readonly supportsLanguageDetection: boolean;
- readonly recommended: boolean;
},
>(
models: readonly Model[],
@@ -54,22 +140,26 @@ export function chooseSetupSpeechModel<
readonly selectedModelId: string;
readonly intent: string;
readonly preferred: readonly string[];
+ readonly benchmarkTier: SpeechBenchmarkTier | undefined;
},
) {
if (input.intent === "auto" && input.preferred.length === 0) {
const model = models.find(({ id }) => id === input.selectedModelId);
- return model && { model, language: null };
+ return model && { model, language: null, recommended: false };
}
for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
const capable = models.filter((model) => {
const effective = effectiveSpeechLanguage(model, input.intent);
return effective === language || (effective === "auto" && model.languages.includes(language));
});
+ const recommendedId = recommendSpeechModels(capable, [language], input.benchmarkTier)
+ .keys()
+ .next().value;
const model =
capable.find(({ id }) => id === input.selectedModelId) ??
- capable.find(({ recommended }) => recommended) ??
+ capable.find(({ id }) => id === recommendedId) ??
capable[0];
- if (model) return { model, language };
+ if (model) return { model, language, recommended: model.id === recommendedId };
}
return undefined;
}
diff --git a/packages/shared/src/speechRecommendationPolicy.json b/packages/shared/src/speechRecommendationPolicy.json
new file mode 100644
index 0000000000000..108de86b43830
--- /dev/null
+++ b/packages/shared/src/speechRecommendationPolicy.json
@@ -0,0 +1,11 @@
+{
+ "dictationSeconds": 30,
+ "freeWaitSeconds": 3,
+ "waitWeight": 0.5,
+ "overallMaxWaitSeconds": 5,
+ "maxLanguageErrorPercent": 20,
+ "noteMinErrorPercent": 15,
+ "experimentalMaxErrorPercent": 30,
+ "accurateMaxWaitSeconds": 10,
+ "fastMaxWaitSeconds": 4
+}