From ccc24396cc3a54f00192e3cfeb444ab349c6b99e Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 01:34:05 +0100 Subject: [PATCH 1/9] test(transcription): name a provider no PodNotes plans as the unknown one The tests for a provider and a model a newer PodNotes stored used OpenRouter and Grok STT 2.0, which this version is about to run. They now use AssemblyAI and universal-3-6-pro. --- src/persistence/podNotesData.test.ts | 8 ++++---- src/services/TranscriptionService.test.ts | 10 +++++----- src/settingsMigrations.test.ts | 6 +++--- src/settingsTransfer.test.ts | 8 ++++---- src/ui/settings/PodNotesSettingsTab.test.ts | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/persistence/podNotesData.test.ts b/src/persistence/podNotesData.test.ts index 28d1a296..c8d5289c 100644 --- a/src/persistence/podNotesData.test.ts +++ b/src/persistence/podNotesData.test.ts @@ -772,16 +772,16 @@ describe("PodNotes data schema", () => { ...structuredClone(released2240), transcript: { ...released2240.transcript, - provider: "openrouter", - model: "x-ai/grok-stt-2.0", + provider: "assemblyai", + model: "universal-3-6-pro", }, }; const decoded = decodePodNotesData(structuredClone(file)); expect(decoded.settings.transcript).toMatchObject({ - provider: "openrouter", - model: "x-ai/grok-stt-2.0", + provider: "assemblyai", + model: "universal-3-6-pro", }); expect(decoded.warnings).toEqual([]); expect(decoded.changed).toBe(false); diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index ac735217..cde09489 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -163,20 +163,20 @@ describe("TranscriptionService", () => { "%s refuses a provider this version does not have, and sends nothing", async (_command, run) => { const plugin = createMockPlugin({ - provider: "openrouter", - model: "x-ai/grok-stt-2.0", + provider: "assemblyai", + model: "universal-3-6-pro", }); await run(new TranscriptionService(plugin)); expect(noticeMessages).toEqual([ - "This version of PodNotes cannot transcribe with openrouter. Update PodNotes, or choose another provider in the transcript settings.", + "This version of PodNotes cannot transcribe with assemblyai. Update PodNotes, or choose another provider in the transcript settings.", ]); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); expect(providerMock).not.toHaveBeenCalled(); expect(plugin.settings.transcript).toMatchObject({ - provider: "openrouter", - model: "x-ai/grok-stt-2.0", + provider: "assemblyai", + model: "universal-3-6-pro", }); }, ); diff --git a/src/settingsMigrations.test.ts b/src/settingsMigrations.test.ts index fbb4fc7f..402c51de 100644 --- a/src/settingsMigrations.test.ts +++ b/src/settingsMigrations.test.ts @@ -213,12 +213,12 @@ describe("migrateTranscriptSettings", () => { it("keeps a provider and a model this version does not have", () => { const migrated = migrateTranscriptSettings( - { provider: "openrouter", model: "x-ai/grok-stt-2.0" }, + { provider: "assemblyai", model: "universal-3-6-pro" }, true, ); - expect(migrated.provider).toBe("openrouter"); - expect(migrated.model).toBe("x-ai/grok-stt-2.0"); + expect(migrated.provider).toBe("assemblyai"); + expect(migrated.model).toBe("universal-3-6-pro"); }); it("coalesces non-boolean enabled / non-string fields to defaults", () => { diff --git a/src/settingsTransfer.test.ts b/src/settingsTransfer.test.ts index be34bc75..e8c4367a 100644 --- a/src/settingsTransfer.test.ts +++ b/src/settingsTransfer.test.ts @@ -910,15 +910,15 @@ describe("the transcription provider in a settings file", () => { it("imports the provider and model a file names, also ones this version does not have", () => { const settings = makeSettings(); - settings.transcript.provider = "openrouter"; - settings.transcript.model = "x-ai/grok-stt-2.0"; + settings.transcript.provider = "assemblyai"; + settings.transcript.model = "universal-3-6-pro"; const file = JSON.stringify(serializeSettings(settings, {}, "2.26.0", NOW, new Map())); const { transcript } = imported(makeSettings(), file); expect([transcript.provider, transcript.model]).toEqual([ - "openrouter", - "x-ai/grok-stt-2.0", + "assemblyai", + "universal-3-6-pro", ]); }); diff --git a/src/ui/settings/PodNotesSettingsTab.test.ts b/src/ui/settings/PodNotesSettingsTab.test.ts index 26c3ca3f..a1c994ee 100644 --- a/src/ui/settings/PodNotesSettingsTab.test.ts +++ b/src/ui/settings/PodNotesSettingsTab.test.ts @@ -410,10 +410,10 @@ describe("Transcripts", () => { test("a provider this version does not have stays selected under its stored name, with no key or model row", async () => { const { ui, provider, keyButtons, writes } = openTranscripts( - transcript({ provider: "openrouter", model: "x-ai/grok-stt-2.0" }), + transcript({ provider: "assemblyai", model: "universal-3-6-pro" }), ); - expect(provider()).toHaveDisplayValue("openrouter"); + expect(provider()).toHaveDisplayValue("assemblyai"); expect(keyButtons()).toEqual([]); expect(ui.queryByRole("combobox", { name: "Model" })).toBeNull(); expect(await writes()).toBe(0); From 4b18cee13cfd051fe46159cd30cd96be32d74102 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 01:58:26 +0100 Subject: [PATCH 2/9] feat(transcription): transcribe with OpenRouter OpenRouter is a credential row, one provider file and a registry line. Its key reference is the new root field openRouterSecretId, stored in SecretStorage as podnotes-openrouter-api-key, and a settings export carries the key as secrets.openRouter. It offers three models, each with the request measured to give word times and speakers: - Grok STT 2.0, recommended: base64 JSON with verbose_json and word and segment times, routed to xAI with provider.only, which takes diarize from provider.options.xai. A request routed elsewhere is answered with no speakers. - ElevenLabs Scribe v2: multipart with diarize beside the model, or JSON with input_audio.url for an episode a saved public feed lists. - MAI-Transcribe 2: multipart with diarize beside the model. With speaker labels off, no request carries diarize. Sounds Scribe v2 lists as audio_event items are left out. OpenRouter takes 25 MB a request and gives the model 60 s, so an upload goes in parts of at most 18 MB and 1,800 s. Parts of 24 MB were answered in 20 to 31 s. A 402 says the account is out of credit, a 401 that the key was rejected, and a 404 that the model is not offered. A 404 whose message names the account's data policy or zero data retention says the model is not offered under the account's OpenRouter privacy settings. Each ends the job at the first part, and none quotes OpenRouter. --- obsidian-e2e.config.mjs | 1 + src/constants.ts | 1 + src/main.persistence.test.ts | 1 + src/persistence/podNotesData.test.ts | 7 +- src/services/CredentialRepository.test.ts | 3 + src/services/TranscriptionService.test.ts | 243 +++++++++++++++++- src/services/transcription/http.ts | 3 + .../transcription/providers/openrouter.ts | 153 +++++++++++ src/services/transcription/registry.ts | 2 + src/services/transcription/transcribe.ts | 1 + src/settingsTransfer.test.ts | 23 +- src/types/Credentials.ts | 1 + src/ui/settings/PodNotesSettingsTab.test.ts | 21 ++ tests/mocks/obsidian.ts | 3 + tests/mocks/transcriptionService.ts | 4 + 15 files changed, 459 insertions(+), 8 deletions(-) create mode 100644 src/services/transcription/providers/openrouter.ts diff --git a/obsidian-e2e.config.mjs b/obsidian-e2e.config.mjs index c28d7008..6a4ad568 100644 --- a/obsidian-e2e.config.mjs +++ b/obsidian-e2e.config.mjs @@ -115,6 +115,7 @@ export default { openAISecretId: "", deepgramSecretId: "", elevenLabsSecretId: "", + openRouterSecretId: "", transcript: { path: "transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\nPodcast: {{podcast}}\nDate: {{date}}\n\n{{transcript}}", diff --git a/src/constants.ts b/src/constants.ts index 6f3ed6ed..ba5eeca4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -159,6 +159,7 @@ export const DEFAULT_SETTINGS: IPodNotesSettings = { openAISecretId: "", deepgramSecretId: "", elevenLabsSecretId: "", + openRouterSecretId: "", transcript: { path: "transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\nPodcast: {{podcast}}\nDate: {{date}}\n\n{{transcript}}", diff --git a/src/main.persistence.test.ts b/src/main.persistence.test.ts index 1dfe647c..8764c4c8 100644 --- a/src/main.persistence.test.ts +++ b/src/main.persistence.test.ts @@ -410,6 +410,7 @@ function savedByThisVersion(released: Record): Record { headsetNextTrack: "skipForward", libraryId: "", elevenLabsSecretId: "", + openRouterSecretId: "", transcript: { ...rest.transcript, provider: "elevenlabs", model: "" }, }); }); @@ -648,8 +649,8 @@ describe("PodNotes data schema", () => { JSON.stringify(encodePodNotesData(decoded.settings, decoded.unknownFields)), ); }; - /** The root field this version adds to a 2.24.0 library when it saves. */ - const ADDED = { elevenLabsSecretId: "" }; + /** The root fields this version adds to a 2.24.0 library when it saves. */ + const ADDED = { elevenLabsSecretId: "", openRouterSecretId: "" }; it.each([ [ @@ -731,7 +732,7 @@ describe("PodNotes data schema", () => { ], ["no key", structuredClone(released2240)], ])( - "a save of a library with %s adds the provider, the model and an empty ElevenLabs key reference, and changes nothing else", + "a save of a library with %s adds the provider, the model and empty ElevenLabs and OpenRouter key references, and changes nothing else", (_state, data) => { const decoded = decodePodNotesData(structuredClone(data)); const { provider, model } = decoded.settings.transcript; diff --git a/src/services/CredentialRepository.test.ts b/src/services/CredentialRepository.test.ts index a3a832e7..7a9bfd2d 100644 --- a/src/services/CredentialRepository.test.ts +++ b/src/services/CredentialRepository.test.ts @@ -32,15 +32,18 @@ describe("CredentialRepository", () => { openAI: " sk-openai ", deepgram: "dg-key", elevenLabs: "el-key", + openRouter: "or-key", }), ).toEqual({ openAISecretId: "podnotes-openai-api-key", deepgramSecretId: "podnotes-deepgram-api-key", elevenLabsSecretId: "podnotes-elevenlabs-api-key", + openRouterSecretId: "podnotes-openrouter-api-key", }); expect(values.get("podnotes-openai-api-key")).toBe("sk-openai"); expect(values.get("podnotes-deepgram-api-key")).toBe("dg-key"); expect(values.get("podnotes-elevenlabs-api-key")).toBe("el-key"); + expect(values.get("podnotes-openrouter-api-key")).toBe("or-key"); expect(api.getSecret).toHaveBeenCalledWith("podnotes-openai-api-key"); }); diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index cde09489..43b2b04b 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -486,7 +486,7 @@ describe("TranscriptionService", () => { await vi.advanceTimersByTimeAsync(60 * 60_000); expect(String(setMessage.mock.calls.at(-1)?.[0])).toMatch( - /\n\nSaved without speaker labels, because OpenAI does not label speakers\. These providers do: ElevenLabs, Deepgram\.$/, + /\n\nSaved without speaker labels, because OpenAI does not label speakers\. These providers do: ElevenLabs, Deepgram, OpenRouter\.$/, ); expect(hide).not.toHaveBeenCalled(); expect(savedNote(plugin)).toBe("# Test Episode\n\nHello there. Hi!"); @@ -1220,8 +1220,249 @@ describe("TranscriptionService", () => { }, ); }); + + describe("an OpenRouter transcript", () => { + const TRANSCRIPTIONS = "https://openrouter.ai/api/v1/audio/transcriptions"; + const CATALOG = "https://openrouter.ai/api/v1/models?output_modalities=transcription"; + const audio = new Uint8Array(2048).map((_, index) => (index * 13) % 241); + const useOpenRouter = ( + overrides: { model?: string; speakers?: boolean; feeds?: FeedRepository } = {}, + ) => { + getEpisodeAudioBufferMock.mockResolvedValue({ buffer: audio.buffer, extension: "mp3" }); + return createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + speakers: true, + ...overrides, + }); + }; + /** A word as OpenRouter's verbose_json lists it, with a speaker when one is given. */ + const word = (text: string, start: number, end: number, speaker?: number) => ({ + word: text, + start, + end, + confidence: 0.99, + ...(speaker !== undefined && { speaker, speaker_label: `speaker_${speaker}` }), + }); + const answers = { + text: "Hello there. [laughter] Hi!", + duration: 11, + words: [ + word("Hello", 1.2, 1.5, 0), + word("there.", 1.5, 1.9, 0), + { ...word("[laughter]", 2, 3, 0), type: "audio_event" }, + word("Hi!", 10.16, 10.5, 1), + ], + }; + /** OpenRouter's public catalog, listing each model with its end date or null. */ + const catalog = (models: Record) => ({ + data: Object.entries(models).map(([id, expiration_date]) => ({ + id, + name: id, + expiration_date, + endpoints: ["/api/v1/audio/transcriptions"], + })), + }); + const EVERY_MODEL = catalog({ + "x-ai/grok-stt-2.0": null, + "x-ai/grok-stt-1.0": "2026-11-15", + "elevenlabs/scribe-v2": null, + "microsoft/mai-transcribe-2": null, + }); + /** Answers the catalog with `listed`, and each transcription request with `answer`. */ + const openRouter = ( + answer: (request: RequestUrlParam) => Response = () => Response.json(answers), + listed: unknown = EVERY_MODEL, + ) => + providerMock.mockImplementation(async (request) => + request.url === CATALOG ? Response.json(listed) : answer(request), + ); + /** The transcription requests the jobs sent, oldest first. */ + const sent = () => + providerMock.mock.calls + .map(([request]) => request) + .filter(({ url }) => url === TRANSCRIPTIONS); + const sentJson = (request: RequestUrlParam) => + JSON.parse(new TextDecoder().decode(sentBytes(request))); + const fields = async (request: RequestUrlParam) => + Object.fromEntries( + [...(await sentForm(request))].filter(([, value]) => typeof value === "string"), + ); + const link = (time: string) => + `obsidian://podnotes?episodeName=Test%20Episode&url=https%3A%2F%2Fexample.com%2Ffeed.xml&time=${time}`; + const labeled = `# Test Episode\n\n[00:00:01](${link("1.2")}) **1:** Hello there.\n\n[00:00:10](${link("10.16")}) **2:** Hi!`; + const unlabeled = `# Test Episode\n\n[00:00:01](${link("1.2")}) Hello there.\n\n[00:00:10](${link("10.16")}) Hi!`; + + test("Grok STT 2.0, the recommended model, gets even a public episode as base64 JSON, routed to xAI and asked there for speaker labels", async () => { + openRouter(); + const plugin = useOpenRouter({ feeds: savedFeedRepository(publicFeed, [mockEpisode]) }); + + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); + + expect(getEpisodeAudioBufferMock).toHaveBeenCalledOnce(); + expect(sent()).toHaveLength(1); + const [request] = sent(); + expect(request.method).toBe("POST"); + expect(request.headers).toEqual({ Authorization: "Bearer openrouter-key" }); + expect(request.contentType).toBe("application/json"); + expect(sentJson(request)).toEqual({ + model: "x-ai/grok-stt-2.0", + input_audio: { data: Buffer.from(audio).toString("base64"), format: "mp3" }, + response_format: "verbose_json", + timestamp_granularities: ["word", "segment"], + provider: { only: ["xai"], options: { xai: { diarize: true } } }, + }); + expect(savedNote(plugin)).toBe(labeled); + }); + + test.each(["elevenlabs/scribe-v2", "microsoft/mai-transcribe-2"])( + "%s gets an upload as multipart, asked for speaker labels beside the model", + async (model) => { + openRouter(); + const plugin = useOpenRouter({ model }); + + await transcribe(plugin); + + expect(sent()).toHaveLength(1); + const [request] = sent(); + expect(request.headers).toEqual({ Authorization: "Bearer openrouter-key" }); + expect(await fields(request)).toEqual({ + model, + response_format: "verbose_json", + diarize: "true", + }); + expect(new Uint8Array(await (await sentFile(request)).arrayBuffer())).toEqual( + audio, + ); + expect(savedNote(plugin)).toBe(labeled); + }, + ); + + test.each(["x-ai/grok-stt-2.0", "elevenlabs/scribe-v2", "microsoft/mai-transcribe-2"])( + "with speaker labels off, %s is asked for none, and the note names no speaker", + async (model) => { + openRouter(); + const plugin = useOpenRouter({ model, speakers: false }); + + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription completed and saved.", + ); + + expect(sent()).toHaveLength(1); + expect(sentText(sent()[0])).not.toContain("diarize"); + expect(savedNote(plugin)).toBe(unlabeled); + }, + ); + + test("Scribe v2 takes a public episode by its address in one request, and its audio is never loaded", async () => { + openRouter(); + const plugin = useOpenRouter({ + model: "elevenlabs/scribe-v2", + feeds: savedFeedRepository(publicFeed, [mockEpisode]), + }); + + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); + + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + expect(sent()).toHaveLength(1); + const [request] = sent(); + expect(request.headers).toEqual({ Authorization: "Bearer openrouter-key" }); + expect(request.contentType).toBe("application/json"); + expect(sentJson(request)).toEqual({ + model: "elevenlabs/scribe-v2", + input_audio: { url: "https://example.com/episode.mp3", format: "mp3" }, + response_format: "verbose_json", + diarize: true, + }); + expect(savedNote(plugin)).toBe(labeled); + }); + + test("a 40 MB episode goes in parts of at most 18 MB and 1,800 s, one at a time, timed from the episode's start", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + const frames = Math.ceil(40_000_000 / stream.frameBytes); + getEpisodeAudioBufferMock.mockResolvedValue({ + buffer: mp3Bytes(stream, frames).buffer, + extension: "mp3", + }); + const partBytes: number[] = []; + openRouter((request) => { + partBytes.push( + Buffer.from(sentJson(request).input_audio.data, "base64").byteLength, + ); + return Response.json({ words: [word(`Part ${partBytes.length}.`, 1, 1.5)] }); + }); + const plugin = createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + }); + + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); + + expect(partBytes.length).toBeGreaterThan(2); + expect(partBytes.reduce((sum, bytes) => sum + bytes)).toBe(frames * stream.frameBytes); + for (const bytes of partBytes) { + expect(bytes).toBeLessThanOrEqual(18_000_000); + expect(mp3Seconds(stream, bytes)).toBeLessThanOrEqual(1800); + } + // Each part's word is 1 s into it. + const expected = partBytes.map( + (_, index) => mp3Seconds(stream, sumOf(partBytes.slice(0, index))) + 1, + ); + const note = savedNote(plugin); + const times = [...note.matchAll(/time=([\d.]+)\)/g)].map(([, time]) => Number(time)); + expect(times).toHaveLength(expected.length); + times.forEach((time, index) => expect(time).toBeCloseTo(expected[index], 2)); + expect(note).toContain(`Part ${partBytes.length}.`); + }); + + test.each<[name: string, status: number, message: string, shown: string]>([ + [ + "402", + 402, + "Insufficient credits. Add more using https://openrouter.ai/settings/credits SENTINEL", + "Your OpenRouter account is out of credit. Add credit there, then try again.", + ], + [ + "404 for a model no route of which meets the account's privacy settings", + 404, + "No endpoints found matching your data policy (Zero data retention). Configure: https://openrouter.ai/settings/privacy SENTINEL", + "OpenRouter does not offer the x-ai/grok-stt-2.0 model under your OpenRouter privacy settings. Change them there, or choose another model in the transcript settings.", + ], + [ + "404", + 404, + "No endpoints found for x-ai/grok-stt-2.0. SENTINEL", + "OpenRouter does not offer the x-ai/grok-stt-2.0 model to this API key.", + ], + [ + "401", + 401, + "No cookie auth credentials found SENTINEL", + "OpenRouter rejected the API key. Check it in the transcript settings.", + ], + ])( + "a %s is said in PodNotes' words, with none of OpenRouter's, and no other part is sent", + async (_name, status, message, shown) => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + openRouter(() => Response.json({ error: { message, code: status } }, { status })); + const plugin = createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + }); + + await expect(finalNotice(plugin)).resolves.toBe(`Transcription failed: ${shown}`); + + expect(sent()).toHaveLength(1); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); + }, + ); + }); }); +function sumOf(numbers: readonly number[]): number { + return numbers.reduce((sum, number) => sum + number, 0); +} + function deepgramAnswer(words: Record[]) { return { results: { channels: [{ alternatives: [{ words }] }] } }; } diff --git a/src/services/transcription/http.ts b/src/services/transcription/http.ts index 42ec7c2f..77ee686b 100644 --- a/src/services/transcription/http.ts +++ b/src/services/transcription/http.ts @@ -6,6 +6,7 @@ export type FailureKind = | "auth" | "credits" | "model-unavailable" + | "privacy" | "rejected" | "unfetched" | "rate-limited" @@ -44,6 +45,8 @@ const SENTENCES: Record `${provider} does not offer the ${model} model to this API key.`, + privacy: ({ provider, model }) => + `${provider} does not offer the ${model} model under your ${provider} privacy settings. Change them there, or choose another model in the transcript settings.`, rejected: ({ provider }) => `${provider} could not transcribe this audio.`, unfetched: ({ provider }) => `${provider} could not fetch the episode.`, "rate-limited": ({ provider }) => diff --git a/src/services/transcription/providers/openrouter.ts b/src/services/transcription/providers/openrouter.ts new file mode 100644 index 00000000..25280f86 --- /dev/null +++ b/src/services/transcription/providers/openrouter.ts @@ -0,0 +1,153 @@ +import { arrayBufferToBase64 } from "obsidian"; +import { isPlainObject } from "src/persistence/codecUtils"; +import type { NonEmpty } from "src/services/publishedTranscript/parse"; +import type { + ModelProfile, + PartInput, + PartTranscript, + Provider, + ProviderRequest, + Word, +} from "./types"; + +const TRANSCRIPTIONS_URL = "https://openrouter.ai/api/v1/audio/transcriptions"; + +/** + * OpenRouter takes 25 MB a request and gives the model 60 s to answer. Parts of 24 MB and + * 1,980 s were answered in 20 to 31 s, so these leave room. + */ +const PART_CAPS = { maxUploadBytes: 18_000_000, maxSeconds: 1800 }; + +type Body = (input: PartInput) => Promise; + +/** The formats `input_audio.format` names, by the type of the file PodNotes sends. MP3 otherwise. */ +const FORMATS: Readonly> = { + "audio/mp4": "m4a", + "audio/ogg": "ogg", + "audio/wav": "wav", + "audio/flac": "flac", + "audio/webm": "webm", +}; + +/** + * xAI reads speaker labels from its own options. A request OpenRouter routes elsewhere + * ignores them and is answered with no speakers, so the route is pinned to xAI. + */ +const grok: Body = async ({ model, speakers, audio }) => { + if ("url" in audio) throw new Error("Grok STT takes no audio address."); + return { + model: model.id, + input_audio: { + data: arrayBufferToBase64(await audio.file.arrayBuffer()), + format: FORMATS[audio.file.type] ?? "mp3", + }, + response_format: "verbose_json", + timestamp_granularities: ["word", "segment"], + provider: { only: ["xai"], ...(speakers && { options: { xai: { diarize: true } } }) }, + }; +}; + +/** Scribe v2 and MAI-Transcribe 2 take `diarize` beside the model. Only Scribe v2 fetches an address. */ +const topLevelDiarize: Body = async ({ model, speakers, audio }) => { + if ("url" in audio) { + return { + model: model.id, + input_audio: { url: audio.url, format: "mp3" }, + response_format: "verbose_json", + ...(speakers && { diarize: true }), + }; + } + const form = new FormData(); + form.append("model", model.id); + form.append("file", audio.file); + form.append("response_format", "verbose_json"); + if (speakers) form.append("diarize", "true"); + return form; +}; + +const MODELS: NonEmpty = [ + { + id: "x-ai/grok-stt-2.0", + label: "Grok STT 2.0", + speakers: true, + url: false, + ...PART_CAPS, + body: grok, + }, + { + id: "elevenlabs/scribe-v2", + label: "ElevenLabs Scribe v2", + speakers: true, + url: true, + ...PART_CAPS, + body: topLevelDiarize, + }, + { + id: "microsoft/mai-transcribe-2", + label: "MAI-Transcribe 2", + speakers: true, + url: false, + ...PART_CAPS, + body: topLevelDiarize, + }, +]; + +/** + * https://openrouter.ai/docs/guides/overview/multimodal/stt + * + * Each model asks for speaker labels its own way, and one asked the wrong way answers + * with none, so each model has the request it was measured with. + */ +export const openrouter: Provider = { + models: MODELS, + timeoutMs: 10 * 60_000, + + async transcribe(input, send) { + const model = MODELS.find(({ id }) => id === input.model.id); + if (!model) throw new Error(`OpenRouter has no ${input.model.id} request.`); + return parse( + await send({ + url: TRANSCRIPTIONS_URL, + headers: { Authorization: `Bearer ${input.apiKey}` }, + body: await model.body(input), + }), + ); + }, + + // A 404 also answers a model whose every route breaks the account's privacy settings, + // such as zero data retention: "No endpoints found matching your data policy". + failure: (status, body) => + status === 404 && + isPlainObject(body) && + isPlainObject(body.error) && + typeof body.error.message === "string" && + /data policy|zero data retention|\bzdr\b/i.test(body.error.message) + ? "privacy" + : undefined, +}; + +function parse(answer: unknown): PartTranscript { + const words = isPlainObject(answer) ? answer.words : undefined; + if (!Array.isArray(words)) return { words: [] }; + return { + words: words.flatMap((item: unknown): Word[] => { + // Scribe v2 also lists sounds, such as "[gentle music]", as `audio_event` items. + if (!isPlainObject(item) || (item.type ?? "word") !== "word") return []; + const { word: text, start, end, speaker } = item; + if (typeof text !== "string" || typeof start !== "number" || typeof end !== "number") { + return []; + } + return [ + { + text, + start, + end, + speaker: + typeof speaker === "number" || typeof speaker === "string" + ? `${speaker}` + : undefined, + }, + ]; + }), + }; +} diff --git a/src/services/transcription/registry.ts b/src/services/transcription/registry.ts index b9bdd686..cfde79ec 100644 --- a/src/services/transcription/registry.ts +++ b/src/services/transcription/registry.ts @@ -4,6 +4,7 @@ import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; import { deepgram } from "./providers/deepgram"; import { elevenlabs } from "./providers/elevenlabs"; import { openai } from "./providers/openai"; +import { openrouter } from "./providers/openrouter"; import type { ModelProfile, Provider } from "./providers/types"; /** @@ -13,6 +14,7 @@ import type { ModelProfile, Provider } from "./providers/types"; export const PROVIDERS = { elevenlabs, deepgram, + openrouter, openai, } satisfies Record; diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts index dba19dc7..80612f45 100644 --- a/src/services/transcription/transcribe.ts +++ b/src/services/transcription/transcribe.ts @@ -30,6 +30,7 @@ const ENDS_THE_JOB: ReadonlySet = new Set([ "auth", "credits", "model-unavailable", + "privacy", "rejected", ]); diff --git a/src/settingsTransfer.test.ts b/src/settingsTransfer.test.ts index e8c4367a..95705fed 100644 --- a/src/settingsTransfer.test.ts +++ b/src/settingsTransfer.test.ts @@ -215,13 +215,20 @@ describe("serializeSettings", () => { openAISecretId: "podnotes-openai-api-key", deepgramSecretId: "podnotes-deepgram-api-key", elevenLabsSecretId: "podnotes-elevenlabs-api-key", + openRouterSecretId: "podnotes-openrouter-api-key", }); - const secrets = { openAI: "sk-secret", deepgram: "dg-secret", elevenLabs: "el-secret" }; + const secrets = { + openAI: "sk-secret", + deepgram: "dg-secret", + elevenLabs: "el-secret", + openRouter: "or-secret", + }; const without = serializeSettings(settings, {}, "2.16.0", NOW, new Map()); expect(without.settings).not.toHaveProperty("openAISecretId"); expect(without.settings).not.toHaveProperty("deepgramSecretId"); expect(without.settings).not.toHaveProperty("elevenLabsSecretId"); + expect(without.settings).not.toHaveProperty("openRouterSecretId"); const withKeys = serializeSettings(settings, { secrets }, "2.16.0", NOW, new Map()); expect(withKeys.version).toBe(2); @@ -604,14 +611,19 @@ describe("parseImport", () => { type: SETTINGS_EXPORT_TYPE, version: 2, settings: { defaultVolume: 0.5, openAISecretId: "not-transferable" }, - secrets: { openAI: " sk ", deepgram: "dg", elevenLabs: "el" }, + secrets: { openAI: " sk ", deepgram: "dg", elevenLabs: "el", openRouter: "or" }, }), ); expect(result.ok).toBe(true); if (result.ok) { expect(result.settings).not.toHaveProperty("openAISecretId"); - expect(result.secrets).toEqual({ openAI: "sk", deepgram: "dg", elevenLabs: "el" }); + expect(result.secrets).toEqual({ + openAI: "sk", + deepgram: "dg", + elevenLabs: "el", + openRouter: "or", + }); } }); @@ -986,10 +998,13 @@ describe("describeSecrets (#168)", () => { expect(describeSecrets({})).toEqual([]); expect(describeSecrets({ openAI: "sk" })).toEqual(["OpenAI API key"]); expect(describeSecrets({ deepgram: "dg" })).toEqual(["Deepgram API key"]); - expect(describeSecrets({ openAI: "sk", deepgram: "dg", elevenLabs: "el" })).toEqual([ + expect( + describeSecrets({ openAI: "sk", deepgram: "dg", elevenLabs: "el", openRouter: "or" }), + ).toEqual([ "OpenAI API key", "Deepgram API key", "ElevenLabs API key", + "OpenRouter API key", ]); }); diff --git a/src/types/Credentials.ts b/src/types/Credentials.ts index 9cc79d2c..cd75efd1 100644 --- a/src/types/Credentials.ts +++ b/src/types/Credentials.ts @@ -7,6 +7,7 @@ export const CREDENTIALS = { openai: { settingsKey: "openAISecretId", exportKey: "openAI", label: "OpenAI" }, deepgram: { settingsKey: "deepgramSecretId", exportKey: "deepgram", label: "Deepgram" }, elevenlabs: { settingsKey: "elevenLabsSecretId", exportKey: "elevenLabs", label: "ElevenLabs" }, + openrouter: { settingsKey: "openRouterSecretId", exportKey: "openRouter", label: "OpenRouter" }, } as const; export type CredentialKind = keyof typeof CREDENTIALS; diff --git a/src/ui/settings/PodNotesSettingsTab.test.ts b/src/ui/settings/PodNotesSettingsTab.test.ts index a1c994ee..bbb9a987 100644 --- a/src/ui/settings/PodNotesSettingsTab.test.ts +++ b/src/ui/settings/PodNotesSettingsTab.test.ts @@ -186,6 +186,7 @@ describe("sections", () => { "OpenAI API key", "Deepgram API key", "ElevenLabs API key", + "OpenRouter API key", "Model", "Speaker labels", "Speaker label format", @@ -326,6 +327,7 @@ describe("Transcripts", () => { expect(options(provider())).toEqual([ ["elevenlabs", "ElevenLabs"], ["deepgram", "Deepgram"], + ["openrouter", "OpenRouter"], ["openai", "OpenAI"], ]); expect(provider()).toHaveDisplayValue("OpenAI"); @@ -432,6 +434,25 @@ describe("Transcripts", () => { expect(model()).toHaveDisplayValue("nova-9"); expect(await writes()).toBe(0); }); + + describe("OpenRouter", () => { + test("choosing it shows its API key row and its three models, Grok STT 2.0 recommended", async () => { + const { provider, model, keyButtons, disk } = openTranscripts(); + + choose(provider(), "openrouter"); + + expect(keyButtons()).toEqual(["Link... OpenRouter API key"]); + await settled(); + expect(options(model())).toEqual([ + ["", "Recommended (Grok STT 2.0)"], + ["x-ai/grok-stt-2.0", "Grok STT 2.0"], + ["elevenlabs/scribe-v2", "ElevenLabs Scribe v2"], + ["microsoft/mai-transcribe-2", "MAI-Transcribe 2"], + ]); + expect(model()).toHaveDisplayValue("Recommended (Grok STT 2.0)"); + expect((await disk()).transcript).toMatchObject({ provider: "openrouter", model: "" }); + }); + }); }); describe("text fields", () => { diff --git a/tests/mocks/obsidian.ts b/tests/mocks/obsidian.ts index 875cc3f9..d36b1b72 100644 --- a/tests/mocks/obsidian.ts +++ b/tests/mocks/obsidian.ts @@ -892,6 +892,9 @@ export const setIcon = (el: HTMLElement | null, icon: string) => { el.setAttribute("data-icon", icon); }; +export const arrayBufferToBase64 = (buffer: ArrayBuffer): string => + Buffer.from(buffer).toString("base64"); + export const requestUrl = async () => ({ status: 200, headers: {}, diff --git a/tests/mocks/transcriptionService.ts b/tests/mocks/transcriptionService.ts index 05326f1d..648c5a35 100644 --- a/tests/mocks/transcriptionService.ts +++ b/tests/mocks/transcriptionService.ts @@ -72,6 +72,7 @@ export function createMockPlugin( openAIKey?: string; deepgramKey?: string; elevenLabsKey?: string; + openRouterKey?: string; /** The stored provider ID. OpenAI when left out. */ provider?: string; /** The stored model ID. The recommended model when left out. */ @@ -91,6 +92,7 @@ export function createMockPlugin( openAIKey = "test-api-key", deepgramKey = "", elevenLabsKey = "", + openRouterKey = "", provider = "openai", model = "", speakers = false, @@ -108,6 +110,7 @@ export function createMockPlugin( openai: openAIKey, deepgram: deepgramKey, elevenlabs: elevenLabsKey, + openrouter: openRouterKey, }; return { @@ -115,6 +118,7 @@ export function createMockPlugin( openAISecretId: openAIKey ? "openai-secret" : "", deepgramSecretId: "", elevenLabsSecretId: "", + openRouterSecretId: "", timestamp: { template: "- {{linktime}} ", offset: 5 }, transcript: { path: "Transcripts/{{podcast}}/{{title}}.md", From cb3b63da2a0db427d696cf2112e93661521f9e6a Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 01:59:39 +0100 Subject: [PATCH 3/9] feat(transcription): say when speaker labels are missing or restart at each part A note saved with speaker labels asked for and none returned now says so: "Saved without speaker labels, because Grok STT 2.0 returned none." OpenRouter answers 200 with no speakers when a request reaches a route that ignores the speaker option, so this is checked for every provider. Labels from two parts already name different speakers. An episode sent in parts now also says that its labels restart in each part, so one person can have several labels, until labels are merged across parts. --- src/services/TranscriptionService.test.ts | 37 +++++++++++++++++++++++ src/services/transcription/transcribe.ts | 15 +++++++++ 2 files changed, 52 insertions(+) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 43b2b04b..eaa0719a 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1415,6 +1415,43 @@ describe("TranscriptionService", () => { expect(note).toContain(`Part ${partBytes.length}.`); }); + test("labels from two parts never name the same speaker, and the notice says labels restart in each", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + openRouter(() => + Response.json({ words: [word("Hello.", 1, 1.5, 0), word("Hi.", 10, 10.5, 1)] }), + ); + const plugin = createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + speakers: true, + }); + + await expect(finalNotice(plugin)).resolves.toBe( + "OpenRouter transcribed this episode in 2 parts, and speaker labels restart in each, so one person can have several labels.", + ); + + expect(sent()).toHaveLength(2); + const labels = [...savedNote(plugin).matchAll(/\*\*(\d+):\*\* (\w+)/g)].map( + ([, speaker, text]) => `${speaker} ${text}`, + ); + expect(labels).toEqual(["1 Hello", "2 Hi", "3 Hello", "4 Hi"]); + }); + + test("speaker labels asked for and none returned, as from a route that ignores the option, saves the note and says so", async () => { + openRouter(() => + Response.json({ words: [word("Hello", 1.2, 1.5), word("there.", 1.5, 1.9)] }), + ); + const plugin = useOpenRouter(); + + await expect(finalNotice(plugin)).resolves.toBe( + "Saved without speaker labels, because Grok STT 2.0 returned none.", + ); + + expect(savedNote(plugin)).toBe( + `# Test Episode\n\n[00:00:01](${link("1.2")}) Hello there.`, + ); + }); + test.each<[name: string, status: number, message: string, shown: string]>([ [ "402", diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts index 80612f45..5bc949d1 100644 --- a/src/services/transcription/transcribe.ts +++ b/src/services/transcription/transcribe.ts @@ -111,5 +111,20 @@ export async function transcribe(job: TranscriptionJob): Promise { `Saved without speaker labels, because ${choice.label} does not label speakers. These providers do: ${providersWithSpeakers().join(", ")}.`, ); } + // A request routed where the speaker option is ignored is answered without speakers. + const labeled = parts.some( + ({ transcript }) => + transcript && + "words" in transcript && + transcript.words.some((word) => word.speaker !== undefined), + ); + if (speakers && !labeled) { + warnings.push(`Saved without speaker labels, because ${model.label} returned none.`); + } + if (speakers && labeled && parts.length > 1) { + warnings.push( + `${choice.label} transcribed this episode in ${parts.length} parts, and speaker labels restart in each, so one person can have several labels.`, + ); + } return { content, warnings }; } From d9886c91c29e2c150d7bb21006b259dff3fb8368 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 02:00:28 +0100 Subject: [PATCH 4/9] feat(transcription): offer only the OpenRouter models its catalog still lists PodNotes reads OpenRouter's public catalog of transcription models, without the key, when the Model row draws and when an OpenRouter job starts. It keeps the read in memory for 24 hours. A read that fails is not kept, so the next one asks again, and the tested list is offered as it is. The catalog only takes models away. A model it does not list is not offered, and one whose expiration_date has come is skipped. A model with a later end date reads "Grok STT 2.0, ends " in the Model row. A catalog that would leave no model is not followed. A chosen model the catalog no longer lists stays selected in the Model row as "