From aca94c4ffe9e6adbcfaa420bdbc654d2b709e3bd Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 10:50:49 +0100 Subject: [PATCH 01/26] test(transcription): send OpenAI diarization no part over 1400 s Refs #532 --- src/services/TranscriptionService.test.ts | 37 +++++++++++++++++++++++ tests/mocks/mp3.ts | 20 ++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/mocks/mp3.ts diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index cd1431f4..3955dfe0 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -11,6 +11,7 @@ import { mockEpisode, noticeMessages, } from "../../tests/mocks/transcriptionService"; +import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../tests/mocks/mp3"; import { openAIMock, sentForm } from "../../tests/mocks/openaiRequestUrl"; import { TranscriptionService } from "./TranscriptionService"; import type PodNotes from "src/main"; @@ -627,5 +628,41 @@ describe("TranscriptionService", () => { expect(openAIMock).toHaveBeenCalledTimes(3); }, ); + + test("OpenAI diarization sends no part longer than the model's 1400 s limit (#532)", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + getEpisodeAudioBufferMock.mockResolvedValue({ + buffer: mp3Bytes(stream, 44_667).buffer, + extension: "mp3", + basename: "episode", + }); + const sentSeconds: number[] = []; + openAIMock.mockImplementation(async (request: RequestUrlParam) => { + const seconds = mp3Seconds(stream, (await sentFile(request)).size); + sentSeconds.push(seconds); + return seconds > 1400 + ? Response.json( + { + error: { + message: `audio duration ${seconds} seconds is longer than 1400 seconds which is the maximum for this model`, + }, + }, + { status: 400 }, + ) + : Response.json({ + segments: [{ speaker: "A", text: `${seconds} s.`, start: 0, end: 1 }], + }); + }); + const plugin = createMockPlugin(); + plugin.settings.transcript.diarization = { + enabled: true, + provider: "openai", + speakerTemplate: "**{{speaker}}:** {{text}}", + }; + + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); + expect(Math.max(...sentSeconds)).toBeLessThanOrEqual(1400); + expect(sentSeconds.reduce((total, seconds) => total + seconds)).toBeCloseTo(3216.024); + }); }); }); diff --git a/tests/mocks/mp3.ts b/tests/mocks/mp3.ts new file mode 100644 index 00000000..2ae689b6 --- /dev/null +++ b/tests/mocks/mp3.ts @@ -0,0 +1,20 @@ +/** Constant-bitrate Layer III streams with silent frames, so a test knows each frame's bytes and duration. */ +export const MP3_STREAMS = { + /** MPEG-1, 96 kbps, 48 kHz, mono: the issue #532 episode's format. */ + mpeg1At96Kbps: { header: [0xff, 0xfb, 0x74, 0xc4], frameBytes: 288, frameSeconds: 0.024 }, + /** MPEG-2.5, 8 kbps, 8 kHz, mono: an hour fits in 3.6 MB. */ + mpeg25At8Kbps: { header: [0xff, 0xe3, 0x18, 0xc0], frameBytes: 72, frameSeconds: 0.072 }, +} as const; + +export type Mp3Stream = (typeof MP3_STREAMS)[keyof typeof MP3_STREAMS]; + +export function mp3Bytes(stream: Mp3Stream, frames: number): Uint8Array { + const bytes = new Uint8Array(frames * stream.frameBytes); + for (let frame = 0; frame < frames; frame++) bytes.set(stream.header, frame * stream.frameBytes); + return bytes; +} + +/** The seconds of audio in `bytes` of `stream`, counted from its fixed frame size. */ +export function mp3Seconds(stream: Mp3Stream, bytes: number): number { + return (bytes / stream.frameBytes) * stream.frameSeconds; +} From 217b040ae7ea803a715a4848f3beb2060e162f62 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 11:01:14 +0100 Subject: [PATCH 02/26] fix(transcription): cap OpenAI diarization chunks at 1400 s gpt-4o-transcribe-diarize rejects requests with more than 1400 s of audio, but MP3 chunks were split by size alone. Any MP3 under about 120 kbps put more than 1400 s in a 20 MB chunk, so every chunk failed. For diarization, split MP3 at frame boundaries into the fewest equal parts that stay under both 20 MB and 1400 s. Whisper keeps its size-only chunks. Fixes #532 --- src/services/TranscriptionService.ts | 11 +- src/services/audioChunker.test.ts | 82 ++++++++++++ src/services/audioChunker.ts | 140 ++++++++++++++++++--- src/services/diarization/openaiProvider.ts | 7 +- src/services/diarization/types.ts | 11 +- tests/mocks/mp3.ts | 5 +- 6 files changed, 224 insertions(+), 32 deletions(-) diff --git a/src/services/TranscriptionService.ts b/src/services/TranscriptionService.ts index 74beae91..9c936816 100644 --- a/src/services/TranscriptionService.ts +++ b/src/services/TranscriptionService.ts @@ -17,6 +17,7 @@ import { type DiarizedSegment, diarizeWithDeepgram, diarizeWithOpenAI, + OPENAI_DIARIZE_MAX_SECONDS, renderDiarizedTranscript, requiredTranscriptionKeyPresent, } from "./diarization"; @@ -382,11 +383,13 @@ export class TranscriptionService { return segments; } - // OpenAI diarization shares Whisper's ~20 MB chunk limit (a conservative - // margin under OpenAI's 25 MB request cap), so reuse the same chunking. - // Speaker labels can differ across chunks on a long episode. + // OpenAI diarization takes Whisper's ~20 MB chunks, also capped at the + // model's 1400 s. Speaker labels can differ across chunks on a long episode. updateNotice("Creating audio chunks..."); - const chunkFiles = await createChunkFiles(audio); + const chunkFiles = await createChunkFiles({ + ...audio, + maxSeconds: OPENAI_DIARIZE_MAX_SECONDS, + }); this.assertActive(); const segments = await diarizeWithOpenAI({ apiKey: this.getApiKey(), diff --git a/src/services/audioChunker.test.ts b/src/services/audioChunker.test.ts index b0c527d2..4908acfe 100644 --- a/src/services/audioChunker.test.ts +++ b/src/services/audioChunker.test.ts @@ -8,6 +8,7 @@ import { shouldConvertToWav, writeWavHeader, } from "./audioChunker"; +import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../tests/mocks/mp3"; describe("getMimeType", () => { test("returns correct mime types for audio formats", () => { @@ -170,6 +171,87 @@ describe("createChunkFiles small-file fast path (#168 / PR #204 review)", () => }); }); +describe("createChunkFiles with maxSeconds (#532)", () => { + const chunk = async (bytes: Uint8Array, maxSeconds: number) => + Promise.all( + ( + await createChunkFiles({ + buffer: bytes.buffer as ArrayBuffer, + basename: "episode", + extension: "mp3", + mimeType: "audio/mp3", + maxSeconds, + }) + ).map(async (file) => ({ + name: file.name, + bytes: new Uint8Array(await file.arrayBuffer()), + })), + ); + const joinsInto = (parts: { bytes: Uint8Array }[], episode: Uint8Array) => + Buffer.concat(parts.map((part) => part.bytes)).equals(episode); + + test("splits an MP3 under the size limit into equal parts no longer than maxSeconds", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + const episode = mp3Bytes(stream, 134_000); + + const parts = await chunk(episode, 1400); + + expect(parts.map((part) => part.name)).toEqual([ + "episode.part0.mp3", + "episode.part1.mp3", + "episode.part2.mp3", + ]); + expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ + 44_667, 44_667, 44_666, + ]); + expect(mp3Seconds(stream, parts[0].bytes.length)).toBeLessThanOrEqual(1400); + for (const part of parts) + expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("sends an MP3 that fits both limits as the original file", async () => { + const episode = mp3Bytes(MP3_STREAMS.mpeg1At96Kbps, 58_000); + + const parts = await chunk(episode, 1400); + + expect(parts.map((part) => part.name)).toEqual(["episode.mp3"]); + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("keeps parts under the size limit when it is the tighter one", async () => { + const stream = MP3_STREAMS.mpeg1At320Kbps; + const episode = mp3Bytes(stream, 50_000); + + const parts = await chunk(episode, 1400); + + expect(parts).toHaveLength(3); + for (const part of parts) { + expect(part.bytes.length).toBeLessThanOrEqual(CHUNK_SIZE_BYTES); + expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); + } + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("keeps a leading ID3 tag in the first part without counting sync-like bytes in it as audio", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + const tag = new Uint8Array(8192); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x3f, 0x76]); + for (let offset = 100; offset <= 5000; offset += 100) tag.set(stream.header, offset); + const frames = mp3Bytes(stream, 36_000); + const episode = Buffer.concat([tag, frames]); + + const parts = await chunk(episode, 1400); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + tag.length + 18_000 * stream.frameBytes, + 18_000 * stream.frameBytes, + ]); + expect(Array.from(parts[0].bytes.subarray(0, 3))).toEqual([0x49, 0x44, 0x33]); + expect(joinsInto(parts, episode)).toBe(true); + }); +}); + describe("createChunkFiles large-file routing (other-logic-bug)", () => { // A minimal AudioBuffer/AudioContext so the decode+WAV path can run under // jsdom, which ships no Web Audio implementation. decodeAudioData returns a diff --git a/src/services/audioChunker.ts b/src/services/audioChunker.ts index c58403c5..22f14d7e 100644 --- a/src/services/audioChunker.ts +++ b/src/services/audioChunker.ts @@ -52,17 +52,28 @@ export function shouldConvertToWav(extension: string, mimeType: string): boolean return !isMp3; } +/** + * `maxSeconds` also caps each chunk's audio length, for MP3 only. Other formats + * over 20 MB become WAV chunks of under four minutes, and smaller ones go up whole. + */ export async function createChunkFiles({ buffer, basename, extension, mimeType, + maxSeconds, }: { buffer: ArrayBuffer; basename: string; extension: string; mimeType: string; + maxSeconds?: number; }): Promise { + if (maxSeconds !== undefined && !shouldConvertToWav(extension, mimeType)) { + const ends = mp3ChunkEnds(new Uint8Array(buffer), CHUNK_SIZE_BYTES, maxSeconds); + return sliceIntoFiles(buffer, ends, basename, extension, mimeType); + } + // A file that already fits in a single request needs neither conversion nor // splitting: send the original (compressed) bytes. OpenAI accepts m4a/mp3/etc. // directly under the upload limit, so this skips the m4a->WAV path below for @@ -99,29 +110,120 @@ export function createBinaryChunkFiles( extension: string, mimeType: string, ): File[] { - if (buffer.byteLength <= CHUNK_SIZE_BYTES) { - return [ - new File([buffer], `${basename}.${extension}`, { - type: mimeType, - }), - ]; + const ends: number[] = []; + for (let end = CHUNK_SIZE_BYTES; end < buffer.byteLength; end += CHUNK_SIZE_BYTES) { + ends.push(end); } + ends.push(buffer.byteLength); + return sliceIntoFiles(buffer, ends, basename, extension, mimeType); +} - const files: File[] = []; - for ( - let offset = 0, index = 0; - offset < buffer.byteLength; - offset += CHUNK_SIZE_BYTES, index++ - ) { - const chunk = buffer.slice(offset, offset + CHUNK_SIZE_BYTES); - files.push( - new File([chunk], `${basename}.part${index}.${extension}`, { - type: mimeType, - }), - ); +/** One file per chunk, where `ends` holds each chunk's end offset in order. */ +function sliceIntoFiles( + buffer: ArrayBuffer, + ends: number[], + basename: string, + extension: string, + mimeType: string, +): File[] { + if (ends.length === 1) { + return [new File([buffer], `${basename}.${extension}`, { type: mimeType })]; } + return ends.map( + (end, index) => + new File( + [buffer.slice(index === 0 ? 0 : ends[index - 1], end)], + `${basename}.part${index}.${extension}`, + { type: mimeType }, + ), + ); +} - return files; +interface Mp3Frame { + offset: number; + length: number; + seconds: number; +} + +const MPEG1_LAYER3_KBPS = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320]; +const MPEG2_LAYER3_KBPS = [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160]; +/** Indexed by the header's version bits: 0 is MPEG-2.5, 2 is MPEG-2, 3 is MPEG-1. */ +const SAMPLE_RATES = [ + [11025, 12000, 8000], + undefined, + [22050, 24000, 16000], + [44100, 48000, 32000], +]; + +/** + * Where each chunk of an MP3 ends. Chunks end at frame starts, there are as few as + * both limits allow, and they hold an equal number of frames, so no short last + * chunk resets the speaker labels for a few seconds of speech. + */ +function mp3ChunkEnds(bytes: Uint8Array, maxBytes: number, maxSeconds: number): number[] { + const frames = mp3Frames(bytes); + const seconds = frames.reduce((total, frame) => total + frame.seconds, 0); + const chunks = Math.max(Math.ceil(bytes.length / maxBytes), Math.ceil(seconds / maxSeconds)); + const framesPerChunk = Math.ceil(frames.length / chunks); + const ends: number[] = []; + let start = 0; + let chunkFrames = 0; + let chunkSeconds = 0; + for (const frame of frames) { + if ( + chunkFrames === framesPerChunk || + frame.offset + frame.length - start > maxBytes || + chunkSeconds + frame.seconds > maxSeconds + ) { + ends.push(frame.offset); + start = frame.offset; + chunkFrames = 0; + chunkSeconds = 0; + } + chunkFrames++; + chunkSeconds += frame.seconds; + } + ends.push(bytes.length); + return ends; +} + +/** + * The Layer III frames in `bytes`. A frame counts only when another frame or the + * end of the buffer follows it, so sync-like bytes in a tag or cover art don't. + */ +function mp3Frames(bytes: Uint8Array): Mp3Frame[] { + const frames: Mp3Frame[] = []; + let offset = 0; + while (offset + 4 <= bytes.length) { + const frame = mp3FrameAt(bytes, offset); + const next = frame ? offset + frame.length : 0; + if (frame && (next === bytes.length || mp3FrameAt(bytes, next))) { + frames.push(frame); + offset = next; + } else { + offset++; + } + } + return frames; +} + +function mp3FrameAt(bytes: Uint8Array, offset: number): Mp3Frame | undefined { + if (offset + 4 > bytes.length || bytes[offset] !== 0xff) return undefined; + const versionAndLayer = bytes[offset + 1]; + const rates = bytes[offset + 2]; + const version = (versionAndLayer >> 3) & 3; + const isLayer3 = ((versionAndLayer >> 1) & 3) === 1; + if ((versionAndLayer & 0xe0) !== 0xe0 || !isLayer3) return undefined; + const kbps = (version === 3 ? MPEG1_LAYER3_KBPS : MPEG2_LAYER3_KBPS)[rates >> 4]; + const sampleRate = SAMPLE_RATES[version]?.[(rates >> 2) & 3]; + if (!kbps || !sampleRate) return undefined; + const samples = version === 3 ? 1152 : 576; + const padding = (rates >> 1) & 1; + return { + offset, + length: Math.floor(((samples / 8) * kbps * 1000) / sampleRate) + padding, + seconds: samples / sampleRate, + }; } async function convertToWavChunks(buffer: ArrayBuffer, basename: string): Promise { diff --git a/src/services/diarization/openaiProvider.ts b/src/services/diarization/openaiProvider.ts index 3d080edb..c8a09f73 100644 --- a/src/services/diarization/openaiProvider.ts +++ b/src/services/diarization/openaiProvider.ts @@ -6,10 +6,9 @@ import { parseOpenAIDiarizedSegments } from "./segments"; /** * Diarize with OpenAI's `gpt-4o-transcribe-diarize` model (issue #168). * - * Reuses the same chunked `File[]` the Whisper path builds because the chunker - * caps each request at ~20 MB (a conservative margin under OpenAI's 25 MB request - * cap). `chunking_strategy: "auto"` is required for audio longer than 30s and lets - * the model segment within a chunk. + * Each chunk stays under ~20 MB (a conservative margin under OpenAI's 25 MB + * request cap) and the model's 1400 s. `chunking_strategy: "auto"` is required + * for audio longer than 30s and lets the model segment within a chunk. * Speaker labels are assigned per request, so on a multi-chunk (long) episode * the labels can differ across chunk boundaries — the caller documents this; a * single-chunk episode (the common case) is fully consistent. diff --git a/src/services/diarization/types.ts b/src/services/diarization/types.ts index ef1bfa70..e2094ef2 100644 --- a/src/services/diarization/types.ts +++ b/src/services/diarization/types.ts @@ -7,10 +7,10 @@ * * - `openai`: the `gpt-4o-transcribe-diarize` model on the same * `/v1/audio/transcriptions` endpoint the plugin already uses, reusing the - * user's existing OpenAI key. Convenient, but the ~20 MB chunk limit (a - * conservative margin under OpenAI's 25 MB request cap) means a long episode is - * split into chunks whose speaker labels are not guaranteed to line up across - * chunk boundaries (the labels are assigned per request). + * user's existing OpenAI key. Convenient, but each request takes at most + * ~20 MB and 1400 s of audio, so a long episode is split into chunks whose + * speaker labels are not guaranteed to line up across chunk boundaries (the + * labels are assigned per request). * - `deepgram`: a single whole-file request to Deepgram's pre-recorded API, * which keeps speaker identity consistent across the entire episode but needs * its own API key. @@ -22,6 +22,9 @@ export const DIARIZATION_PROVIDERS: readonly DiarizationProviderId[] = ["openai" /** The fixed OpenAI model that performs diarization (no other OpenAI model does). */ export const OPENAI_DIARIZE_MODEL = "gpt-4o-transcribe-diarize"; +/** The most audio, in seconds, that OpenAI accepts in one diarization request. */ +export const OPENAI_DIARIZE_MAX_SECONDS = 1400; + /** * One contiguous turn of speech attributed to a single speaker. `speaker` is the * provider's label, normalized to a short human-facing token (OpenAI emits `A`, diff --git a/tests/mocks/mp3.ts b/tests/mocks/mp3.ts index 2ae689b6..180ee1d2 100644 --- a/tests/mocks/mp3.ts +++ b/tests/mocks/mp3.ts @@ -2,6 +2,8 @@ export const MP3_STREAMS = { /** MPEG-1, 96 kbps, 48 kHz, mono: the issue #532 episode's format. */ mpeg1At96Kbps: { header: [0xff, 0xfb, 0x74, 0xc4], frameBytes: 288, frameSeconds: 0.024 }, + /** MPEG-1, 320 kbps, 48 kHz, mono: 20 MiB holds under nine minutes. */ + mpeg1At320Kbps: { header: [0xff, 0xfb, 0xe4, 0xc4], frameBytes: 960, frameSeconds: 0.024 }, /** MPEG-2.5, 8 kbps, 8 kHz, mono: an hour fits in 3.6 MB. */ mpeg25At8Kbps: { header: [0xff, 0xe3, 0x18, 0xc0], frameBytes: 72, frameSeconds: 0.072 }, } as const; @@ -10,7 +12,8 @@ export type Mp3Stream = (typeof MP3_STREAMS)[keyof typeof MP3_STREAMS]; export function mp3Bytes(stream: Mp3Stream, frames: number): Uint8Array { const bytes = new Uint8Array(frames * stream.frameBytes); - for (let frame = 0; frame < frames; frame++) bytes.set(stream.header, frame * stream.frameBytes); + for (let frame = 0; frame < frames; frame++) + bytes.set(stream.header, frame * stream.frameBytes); return bytes; } From 61c8c5208fbf0d17c26943bbab5ac3b318c5a66a Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 11:02:00 +0100 Subject: [PATCH 03/26] test(transcription): send a rejected OpenAI request once Refs #532 --- src/services/TranscriptionService.test.ts | 32 ++++++++- src/services/openaiTranscription.test.ts | 4 +- tests/mocks/openaiFailures.ts | 80 ++++++++++++----------- 3 files changed, 76 insertions(+), 40 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 3955dfe0..47984138 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test, vi } from "vitest"; import { Notice, type RequestUrlParam } from "obsidian"; import { deferred } from "../../tests/mocks/feeds"; -import { OPENAI_FAILURES, rateLimited } from "../../tests/mocks/openaiFailures"; +import { OPENAI_FAILURES, OPENAI_REJECTIONS, rateLimited } from "../../tests/mocks/openaiFailures"; import { createMockPlugin, diarizeWithDeepgramMock, @@ -523,6 +523,18 @@ describe("TranscriptionService", () => { }, ); + test.each(OPENAI_REJECTIONS)( + "Whisper %s shows the API error after one send", + async (_name, respond, message) => { + openAIMock.mockImplementation(respond); + + await expect(finalNotice(createMockPlugin())).resolves.toBe( + `Transcription failed: all 1 audio chunk(s) failed or returned no text. Last error: ${message}`, + ); + expect(openAIMock).toHaveBeenCalledOnce(); + }, + ); + test("Whisper answering without text fails the run after three sends", async () => { openAIMock.mockImplementation(async () => Response.json({})); @@ -629,6 +641,24 @@ describe("TranscriptionService", () => { }, ); + test.each(OPENAI_REJECTIONS)( + "OpenAI diarization %s shows the API error after one send", + async (_name, respond, message) => { + openAIMock.mockImplementation(respond); + const plugin = createMockPlugin(); + plugin.settings.transcript.diarization = { + enabled: true, + provider: "openai", + speakerTemplate: "**{{speaker}}:** {{text}}", + }; + + await expect(finalNotice(plugin)).resolves.toBe( + `Transcription failed: OpenAI diarization failed for every chunk: ${message}`, + ); + expect(openAIMock).toHaveBeenCalledOnce(); + }, + ); + test("OpenAI diarization sends no part longer than the model's 1400 s limit (#532)", async () => { const stream = MP3_STREAMS.mpeg25At8Kbps; getEpisodeAudioBufferMock.mockResolvedValue({ diff --git a/src/services/openaiTranscription.test.ts b/src/services/openaiTranscription.test.ts index fb168ed7..f95c6f92 100644 --- a/src/services/openaiTranscription.test.ts +++ b/src/services/openaiTranscription.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { OPENAI_FAILURES, rateLimited } from "../../tests/mocks/openaiFailures"; +import { OPENAI_FAILURES, OPENAI_REJECTIONS, rateLimited } from "../../tests/mocks/openaiFailures"; import { corsBlockedFetch, openAIMock, sentForm } from "../../tests/mocks/openaiRequestUrl"; import { createTranscription, retryDelayMs } from "./openaiTranscription"; @@ -93,7 +93,7 @@ describe("createTranscription", () => { await expect(send()).resolves.toEqual(payload); }); - it.each([...OPENAI_FAILURES, ...EDGE_FAILURES])( + it.each([...OPENAI_FAILURES, ...OPENAI_REJECTIONS, ...EDGE_FAILURES])( "%s rejects with the openai SDK's message after one send", async (_name, respond, message) => { openAIMock.mockImplementation(respond); diff --git a/tests/mocks/openaiFailures.ts b/tests/mocks/openaiFailures.ts index 773b962e..196ba3a2 100644 --- a/tests/mocks/openaiFailures.ts +++ b/tests/mocks/openaiFailures.ts @@ -1,44 +1,50 @@ const openAIError = (status: number, message: string) => async () => Response.json({ error: { message, type: "error", param: null, code: null } }, { status }); -export const OPENAI_FAILURES: [name: string, respond: () => Promise, message: string][] = +type Failure = [name: string, respond: () => Promise, message: string]; + +/** Failures that sending the same request again can get past. */ +export const OPENAI_FAILURES: Failure[] = [ + [ + "429", + openAIError( + 429, + "You exceeded your current quota, please check your plan and billing details.", + ), + "429 You exceeded your current quota, please check your plan and billing details.", + ], + [ + "500", + openAIError( + 500, + "The server had an error while processing your request. Sorry about that!", + ), + "500 The server had an error while processing your request. Sorry about that!", + ], + ["502 text body", async () => new Response("Bad Gateway", { status: 502 }), "502 Bad Gateway"], + ["503 empty body", async () => new Response("", { status: 503 }), "503 status code (no body)"], + ["network", () => Promise.reject(new TypeError("Failed to fetch")), "Connection error."], +]; + +/** Failures that the same request always gets again. */ +export const OPENAI_REJECTIONS: Failure[] = [ + [ + "400", + openAIError( + 400, + "audio duration 1520.592 seconds is longer than 1400 seconds which is the maximum for this model", + ), + "400 audio duration 1520.592 seconds is longer than 1400 seconds which is the maximum for this model", + ], [ - [ - "401", - openAIError( - 401, - "Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", - ), - "401 Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", - ], - [ - "429", - openAIError( - 429, - "You exceeded your current quota, please check your plan and billing details.", - ), - "429 You exceeded your current quota, please check your plan and billing details.", - ], - [ - "500", - openAIError( - 500, - "The server had an error while processing your request. Sorry about that!", - ), - "500 The server had an error while processing your request. Sorry about that!", - ], - [ - "502 text body", - async () => new Response("Bad Gateway", { status: 502 }), - "502 Bad Gateway", - ], - [ - "503 empty body", - async () => new Response("", { status: 503 }), - "503 status code (no body)", - ], - ["network", () => Promise.reject(new TypeError("Failed to fetch")), "Connection error."], - ]; + "401", + openAIError( + 401, + "Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", + ), + "401 Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", + ], +]; export const rateLimited = (headers: Record) => Response.json( From 44947b5ec3f64bd2563bebc9bc83b3e5f6fd19d1 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 11:06:38 +0100 Subject: [PATCH 04/26] fix(transcription): send a rejected OpenAI request once A chunk that OpenAI rejected with a status such as 400 or 401 was sent three times, although each send fails the same way. A long diarized episode went up three times before the error showed. Retry only the statuses openai-node retries: 408, 409, 429 and 5xx, plus network errors. Refs #532 --- src/services/TranscriptionService.ts | 7 ++++--- src/services/diarization/openaiProvider.ts | 6 +++--- src/services/openaiTranscription.test.ts | 2 +- src/services/openaiTranscription.ts | 9 +++++++++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/services/TranscriptionService.ts b/src/services/TranscriptionService.ts index 9c936816..7bee9c39 100644 --- a/src/services/TranscriptionService.ts +++ b/src/services/TranscriptionService.ts @@ -10,7 +10,7 @@ import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; import { openNote } from "src/ui/openNote"; import { TimerNotice } from "src/ui/TimerNotice"; import { createChunkFiles, getMimeType } from "./audioChunker"; -import { createTranscription, retryDelayMs } from "./openaiTranscription"; +import { canRetry, createTranscription, retryDelayMs } from "./openaiTranscription"; import { type DiarizationAudio, type DiarizationProviderId, @@ -450,9 +450,9 @@ export class TranscriptionService { } catch (error) { this.assertActive(); retries++; - if (retries >= MAX_RETRIES) { + if (retries >= MAX_RETRIES || !canRetry(error)) { console.error( - `Failed to transcribe chunk ${index} after ${MAX_RETRIES} attempts:`, + `Failed to transcribe chunk ${index} after ${retries} attempt(s):`, error, ); transcriptions[index] = chunkErrorPlaceholder(index); @@ -460,6 +460,7 @@ export class TranscriptionService { lastError = error; completedChunks++; updateProgress(); + break; } else { await this.waitForRetry(retryDelayMs(error, 1000 * retries)); } diff --git a/src/services/diarization/openaiProvider.ts b/src/services/diarization/openaiProvider.ts index c8a09f73..90a1a5ae 100644 --- a/src/services/diarization/openaiProvider.ts +++ b/src/services/diarization/openaiProvider.ts @@ -1,5 +1,5 @@ import { toError } from "../../utility/toError"; -import { createTranscription, retryDelayMs } from "../openaiTranscription"; +import { canRetry, createTranscription, retryDelayMs } from "../openaiTranscription"; import { type DiarizedSegment, OPENAI_DIARIZE_MODEL } from "./types"; import { parseOpenAIDiarizedSegments } from "./segments"; @@ -62,9 +62,9 @@ export async function diarizeWithOpenAI(opts: { } catch (error) { throwIfAborted(signal); attempt++; - if (attempt >= maxRetries) { + if (attempt >= maxRetries || !canRetry(error)) { console.error( - `OpenAI diarization failed for chunk ${index + 1} after ${maxRetries} attempts:`, + `OpenAI diarization failed for chunk ${index + 1} after ${attempt} attempt(s):`, error, ); failedChunks++; diff --git a/src/services/openaiTranscription.test.ts b/src/services/openaiTranscription.test.ts index f95c6f92..901fff35 100644 --- a/src/services/openaiTranscription.test.ts +++ b/src/services/openaiTranscription.test.ts @@ -98,7 +98,7 @@ describe("createTranscription", () => { async (_name, respond, message) => { openAIMock.mockImplementation(respond); - await expect(send()).rejects.toThrow(new Error(message)); + await expect(send()).rejects.toHaveProperty("message", message); expect(openAIMock).toHaveBeenCalledOnce(); }, ); diff --git a/src/services/openaiTranscription.ts b/src/services/openaiTranscription.ts index ced3b90a..42d48a8d 100644 --- a/src/services/openaiTranscription.ts +++ b/src/services/openaiTranscription.ts @@ -43,6 +43,7 @@ export async function createTranscription( if (response.status < 200 || response.status >= 300) { throw new OpenAIStatusError( `${response.status} ${errorDetail(text) || "status code (no body)"}`, + response.status, serverRetryDelayMs(new Headers(response.headers)), ); } @@ -53,9 +54,17 @@ export function retryDelayMs(error: unknown, fallbackMs: number): number { return (error instanceof OpenAIStatusError ? error.retryAfterMs : undefined) ?? fallbackMs; } +/** Whether sending the request again can succeed. Statuses follow openai-node's retry rule. */ +export function canRetry(error: unknown): boolean { + if (!(error instanceof OpenAIStatusError)) return true; + const { status } = error; + return status === 408 || status === 409 || status === 429 || status >= 500; +} + class OpenAIStatusError extends Error { constructor( message: string, + readonly status: number, readonly retryAfterMs: number | undefined, ) { super(message); From a9a3b07da6a6dfed6b45045ba190b523dc9d50df Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 21:09:34 +0100 Subject: [PATCH 05/26] fix(transcription): keep OpenAI diarization requests inside the timeout gpt-4o-transcribe-diarize takes up to 0.58 s per second of audio, measured over 81 requests, so the 1,072 s parts a 54 min episode got would take about 10 min each, at the request timeout. Parts are now at most 600 s, which the model answers in under 6 min. Refs #532 --- src/services/TranscriptionService.test.ts | 5 ++++- src/services/TranscriptionService.ts | 8 ++++---- src/services/diarization/openaiProvider.ts | 2 +- src/services/diarization/types.ts | 9 ++++++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 47984138..edc4a494 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -659,7 +659,9 @@ describe("TranscriptionService", () => { }, ); - test("OpenAI diarization sends no part longer than the model's 1400 s limit (#532)", async () => { + test("OpenAI diarization parts stay under the model's 1400 s limit and answer inside the 10 min request timeout (#532)", async () => { + // Measured on gpt-4o-transcribe-diarize: the slowest of 81 requests took 0.58 s per second of audio. + const slowestSecondsPerAudioSecond = 0.58; const stream = MP3_STREAMS.mpeg25At8Kbps; getEpisodeAudioBufferMock.mockResolvedValue({ buffer: mp3Bytes(stream, 44_667).buffer, @@ -692,6 +694,7 @@ describe("TranscriptionService", () => { await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); expect(Math.max(...sentSeconds)).toBeLessThanOrEqual(1400); + expect(Math.max(...sentSeconds) * slowestSecondsPerAudioSecond).toBeLessThan(10 * 60); expect(sentSeconds.reduce((total, seconds) => total + seconds)).toBeCloseTo(3216.024); }); }); diff --git a/src/services/TranscriptionService.ts b/src/services/TranscriptionService.ts index 7bee9c39..7157a9d8 100644 --- a/src/services/TranscriptionService.ts +++ b/src/services/TranscriptionService.ts @@ -17,7 +17,7 @@ import { type DiarizedSegment, diarizeWithDeepgram, diarizeWithOpenAI, - OPENAI_DIARIZE_MAX_SECONDS, + OPENAI_DIARIZE_PART_SECONDS, renderDiarizedTranscript, requiredTranscriptionKeyPresent, } from "./diarization"; @@ -383,12 +383,12 @@ export class TranscriptionService { return segments; } - // OpenAI diarization takes Whisper's ~20 MB chunks, also capped at the - // model's 1400 s. Speaker labels can differ across chunks on a long episode. + // OpenAI diarization takes Whisper's ~20 MB chunks, capped at 600 s so each request + // answers inside the timeout. Speaker labels can differ across chunks. updateNotice("Creating audio chunks..."); const chunkFiles = await createChunkFiles({ ...audio, - maxSeconds: OPENAI_DIARIZE_MAX_SECONDS, + maxSeconds: OPENAI_DIARIZE_PART_SECONDS, }); this.assertActive(); const segments = await diarizeWithOpenAI({ diff --git a/src/services/diarization/openaiProvider.ts b/src/services/diarization/openaiProvider.ts index 90a1a5ae..cbc705c2 100644 --- a/src/services/diarization/openaiProvider.ts +++ b/src/services/diarization/openaiProvider.ts @@ -7,7 +7,7 @@ import { parseOpenAIDiarizedSegments } from "./segments"; * Diarize with OpenAI's `gpt-4o-transcribe-diarize` model (issue #168). * * Each chunk stays under ~20 MB (a conservative margin under OpenAI's 25 MB - * request cap) and the model's 1400 s. `chunking_strategy: "auto"` is required + * request cap) and 600 s, which the model answers inside the 10 min timeout. `chunking_strategy: "auto"` is required * for audio longer than 30s and lets the model segment within a chunk. * Speaker labels are assigned per request, so on a multi-chunk (long) episode * the labels can differ across chunk boundaries — the caller documents this; a diff --git a/src/services/diarization/types.ts b/src/services/diarization/types.ts index e2094ef2..78ec6add 100644 --- a/src/services/diarization/types.ts +++ b/src/services/diarization/types.ts @@ -8,7 +8,7 @@ * - `openai`: the `gpt-4o-transcribe-diarize` model on the same * `/v1/audio/transcriptions` endpoint the plugin already uses, reusing the * user's existing OpenAI key. Convenient, but each request takes at most - * ~20 MB and 1400 s of audio, so a long episode is split into chunks whose + * ~20 MB and 600 s of audio, so a long episode is split into chunks whose * speaker labels are not guaranteed to line up across chunk boundaries (the * labels are assigned per request). * - `deepgram`: a single whole-file request to Deepgram's pre-recorded API, @@ -22,8 +22,11 @@ export const DIARIZATION_PROVIDERS: readonly DiarizationProviderId[] = ["openai" /** The fixed OpenAI model that performs diarization (no other OpenAI model does). */ export const OPENAI_DIARIZE_MODEL = "gpt-4o-transcribe-diarize"; -/** The most audio, in seconds, that OpenAI accepts in one diarization request. */ -export const OPENAI_DIARIZE_MAX_SECONDS = 1400; +/** + * The most audio, in seconds, in one diarization request. OpenAI accepts up to 1400 s, but the model + * takes up to 0.58 s per second of audio, so 600 s answers in under 6 min, inside the 10 min request timeout. + */ +export const OPENAI_DIARIZE_PART_SECONDS = 600; /** * One contiguous turn of speech attributed to a single speaker. `speaker` is the From f128d4c105e849e725f6e6874292a9b8bc39320c Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 22:23:54 +0100 Subject: [PATCH 06/26] refactor(transcription): keep the speaker label format in the renderer The published transcript renderer is the reader of the format that stays when the diarization folder goes. The default template now lives only in DEFAULT_SETTINGS, so src/constants.ts imports no service code. --- src/constants.ts | 3 +-- src/services/diarization/segments.test.ts | 24 +++-------------- src/services/diarization/segments.ts | 26 +++---------------- .../publishedTranscript/render.test.ts | 20 ++++++++++++++ src/services/publishedTranscript/render.ts | 14 ++++++++-- 5 files changed, 39 insertions(+), 48 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index f2925198..c9114fd4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,5 @@ import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; import type { Playlist } from "./types/Playlist"; -import { DEFAULT_SPEAKER_TEMPLATE } from "./services/diarization/segments"; import { ownRecord } from "./utility/ownRecord"; export const VIEW_TYPE = "podcast_player_view"; @@ -167,7 +166,7 @@ export const DEFAULT_SETTINGS: IPodNotesSettings = { diarization: { enabled: false, provider: "openai", - speakerTemplate: DEFAULT_SPEAKER_TEMPLATE, + speakerTemplate: "**{{speaker}}:** ", }, }, }; diff --git a/src/services/diarization/segments.test.ts b/src/services/diarization/segments.test.ts index d8419900..d52f0fe3 100644 --- a/src/services/diarization/segments.test.ts +++ b/src/services/diarization/segments.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from "vitest"; +import { DEFAULT_SETTINGS } from "src/constants"; import { - DEFAULT_SPEAKER_TEMPLATE, - formatSpeakerLabel, mergeAdjacentSpeakers, parseDeepgramSegments, parseOpenAIDiarizedSegments, @@ -155,26 +154,9 @@ describe("mergeAdjacentSpeakers (#168)", () => { }); }); -describe("formatSpeakerLabel (#168)", () => { - it("substitutes the {{speaker}} token (case/space-insensitive)", () => { - expect(formatSpeakerLabel("**{{speaker}}:** ", "A")).toBe("**A:** "); - expect(formatSpeakerLabel("{{ Speaker }} - ", "2")).toBe("2 - "); - }); - - it("prefixes the label when the template has no token", () => { - expect(formatSpeakerLabel("> ", "A")).toBe("A: > "); - }); - - it("inserts a label containing $-replacement patterns verbatim", () => { - // `$&`, `$\``, `$'`, `$$` are special in a String.replace replacement string; - // the label must be inserted literally rather than expanded to matched text. - const speaker = "$`$'$&$$"; - expect(formatSpeakerLabel("**{{speaker}}:** ", speaker)).toBe(`**${speaker}:** `); - expect(formatSpeakerLabel("{{speaker}} - ", speaker)).toBe(`${speaker} - `); - }); -}); - describe("renderDiarizedTranscript (#168)", () => { + const DEFAULT_SPEAKER_TEMPLATE = DEFAULT_SETTINGS.transcript.diarization.speakerTemplate; + it("renders speaker turns separated by blank lines", () => { const body = renderDiarizedTranscript( [ diff --git a/src/services/diarization/segments.ts b/src/services/diarization/segments.ts index 50918228..cfd6fccd 100644 --- a/src/services/diarization/segments.ts +++ b/src/services/diarization/segments.ts @@ -1,3 +1,5 @@ +import { DEFAULT_SETTINGS } from "src/constants"; +import { formatSpeakerLabel } from "src/services/publishedTranscript/render"; import type { DiarizedSegment } from "./types"; /** @@ -8,12 +10,6 @@ import type { DiarizedSegment } from "./types"; * handling (untrusted JSON -> normalized segments -> markdown) is unit-testable. */ -/** The placeholder a speaker-label template uses to position the speaker name. */ -const SPEAKER_TOKEN = /\{\{\s*speaker\s*\}\}/gi; - -/** Default per-turn prefix; `{{speaker}}` is replaced with the provider's label. */ -export const DEFAULT_SPEAKER_TEMPLATE = "**{{speaker}}:** "; - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -139,22 +135,6 @@ export function mergeAdjacentSpeakers(segments: DiarizedSegment[]): DiarizedSegm return merged; } -/** - * Replace the `{{speaker}}` token in a speaker-label template. When the template - * has no token the label is prefixed instead, so even a tokenless template - * (e.g. "> ") still produces a usable, speaker-prefixed line. - */ -export function formatSpeakerLabel(template: string, speaker: string): string { - if (SPEAKER_TOKEN.test(template)) { - SPEAKER_TOKEN.lastIndex = 0; - // Pass a replacer function so the label is inserted verbatim: String.replace - // interprets `$$`, `$&`, `$\``, `$'`, `$n` in a replacement *string*, which - // would mangle a label that happened to contain one of those sequences. - return template.replace(SPEAKER_TOKEN, () => speaker); - } - return `${speaker}: ${template}`; -} - /** * Render normalized segments into the markdown body that fills `{{transcript}}`. * Each turn is ``, turns separated by a blank line so they @@ -165,7 +145,7 @@ export function renderDiarizedTranscript( segments: DiarizedSegment[], speakerTemplate: string, ): string { - const template = speakerTemplate || DEFAULT_SPEAKER_TEMPLATE; + const template = speakerTemplate || DEFAULT_SETTINGS.transcript.diarization.speakerTemplate; return segments .map((segment) => `${formatSpeakerLabel(template, segment.speaker)}${segment.text}`) .join("\n\n"); diff --git a/src/services/publishedTranscript/render.test.ts b/src/services/publishedTranscript/render.test.ts index 18dbfde7..031d15a5 100644 --- a/src/services/publishedTranscript/render.test.ts +++ b/src/services/publishedTranscript/render.test.ts @@ -4,6 +4,7 @@ import { transcriptFixture } from "../../../tests/mocks/transcriptFixtures"; import { type NonEmpty, parseTranscript, type TranscriptCue } from "./parse"; import { escapeTranscriptText, + formatSpeakerLabel, groupCues, MAX_PARAGRAPH_LENGTH, renderTranscriptBody, @@ -302,6 +303,25 @@ describe("paragraph length", () => { }); }); +describe("formatSpeakerLabel", () => { + test("substitutes the {{speaker}} token (case/space-insensitive)", () => { + expect(formatSpeakerLabel("**{{speaker}}:** ", "A")).toBe("**A:** "); + expect(formatSpeakerLabel("{{ Speaker }} - ", "2")).toBe("2 - "); + }); + + test("prefixes the label when the template has no token", () => { + expect(formatSpeakerLabel("> ", "A")).toBe("A: > "); + }); + + test("inserts a label containing $-replacement patterns verbatim", () => { + // `$&`, `$\``, `$'`, `$$` are special in a String.replace replacement string; + // the label must be inserted literally rather than expanded to matched text. + const speaker = "$`$'$&$$"; + expect(formatSpeakerLabel("**{{speaker}}:** ", speaker)).toBe(`**${speaker}:** `); + expect(formatSpeakerLabel("{{speaker}} - ", speaker)).toBe(`${speaker} - `); + }); +}); + describe("timestampStyle", () => { test.each([ ["- {{linktime}} ", { format: "HH:mm:ss", linkify: true }], diff --git a/src/services/publishedTranscript/render.ts b/src/services/publishedTranscript/render.ts index 89ae3fab..b4931050 100644 --- a/src/services/publishedTranscript/render.ts +++ b/src/services/publishedTranscript/render.ts @@ -1,8 +1,17 @@ +import { DEFAULT_SETTINGS } from "src/constants"; import type { Episode } from "src/types/Episode"; -import { DEFAULT_SPEAKER_TEMPLATE, formatSpeakerLabel } from "src/services/diarization/segments"; import { formatEpisodeTimestamp } from "src/utility/formatEpisodeTimestamp"; import { collapseWhitespace, type ParsedTranscript, type TranscriptCue } from "./parse"; +const SPEAKER_TOKEN = /\{\{\s*speaker\s*\}\}/gi; + +/** A template without `{{speaker}}` gets the name in front, so the line still says who spoke. */ +export function formatSpeakerLabel(template: string, speaker: string): string { + if (!template.match(SPEAKER_TOKEN)) return `${speaker}: ${template}`; + // A replacer function, because a replacement string would expand `$&` and `$1` in a name. + return template.replace(SPEAKER_TOKEN, () => speaker); +} + export interface TimestampStyle { readonly format: string; readonly linkify: boolean; @@ -119,7 +128,8 @@ export function renderTranscriptBody( .join("\n\n"); } const { episode, timestamp } = options; - const speakerTemplate = options.speakerTemplate || DEFAULT_SPEAKER_TEMPLATE; + const speakerTemplate = + options.speakerTemplate || DEFAULT_SETTINGS.transcript.diarization.speakerTemplate; const labels = new Map(); const label = (speaker: string) => { let written = labels.get(speaker); From 6887b5a222bd0abd619b40db26075e44ef74f58b Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 22:24:14 +0100 Subject: [PATCH 07/26] feat(transcription): add one engine for transcription providers src/services/transcription/ turns an episode into a transcript with any provider. A provider is one file: its models, its request, and how to read its answers. OpenAI (gpt-transcribe) and Deepgram (nova-3) are the first two. The engine owns what providers share: - audioSource.ts decides whether a provider may be given the episode's address. Only an address that a saved public feed lists, that carries no credential, and that names a public host passes. All other audio is uploaded. - split.ts is audioChunker.ts with the caps as arguments and a start time on each part. - http.ts sends a request at most three times, and only after a 429, a 5xx, or a lost connection. Its failure sentences never include the provider's own text, which can quote a private address. - cues.ts turns words or text into the shapes of a published transcript. Nothing calls the engine yet. Lint rejects an import of the folder's files from outside it, except its index. --- .oxlintrc.json | 17 ++ AGENTS.md | 4 + src/services/publishedTranscript/parse.ts | 5 +- src/services/transcription/audioSource.ts | 45 +++ src/services/transcription/cues.test.ts | 48 +++ src/services/transcription/cues.ts | 91 ++++++ src/services/transcription/http.ts | 188 ++++++++++++ src/services/transcription/index.ts | 2 + .../transcription/providers/deepgram.ts | 57 ++++ .../transcription/providers/openai.ts | 49 +++ src/services/transcription/providers/types.ts | 61 ++++ src/services/transcription/registry.ts | 37 +++ src/services/transcription/split.test.ts | 198 ++++++++++++ src/services/transcription/split.ts | 282 ++++++++++++++++++ src/services/transcription/transcribe.ts | 104 +++++++ 15 files changed, 1186 insertions(+), 2 deletions(-) create mode 100644 src/services/transcription/audioSource.ts create mode 100644 src/services/transcription/cues.test.ts create mode 100644 src/services/transcription/cues.ts create mode 100644 src/services/transcription/http.ts create mode 100644 src/services/transcription/index.ts create mode 100644 src/services/transcription/providers/deepgram.ts create mode 100644 src/services/transcription/providers/openai.ts create mode 100644 src/services/transcription/providers/types.ts create mode 100644 src/services/transcription/registry.ts create mode 100644 src/services/transcription/split.test.ts create mode 100644 src/services/transcription/split.ts create mode 100644 src/services/transcription/transcribe.ts diff --git a/.oxlintrc.json b/.oxlintrc.json index ee705b44..56b79aca 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -32,6 +32,23 @@ ] }, "overrides": [ + { + "files": ["src/**/*.ts", "tests/**/*.ts"], + "excludeFiles": ["src/services/transcription/**"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/services/transcription/**", "./transcription/**"], + "message": "Import from src/services/transcription, the folder's public surface." + } + ] + } + ] + } + }, { "files": ["src/**/*.ts"], "excludeFiles": ["src/**/*.test.ts", "src/**/*.spec.ts"], diff --git a/AGENTS.md b/AGENTS.md index ce001c5b..16b456af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,10 @@ in `src/main.ts`; public API code is under `src/API/`; parsing code is under are under `src/utility/`; Svelte UI lives under `src/ui/`; shared types live in `src/types/`. +Audio transcription lives in `src/services/transcription/`. A provider is one +file under `providers/` and one line in `registry.ts`. Code outside the folder +imports only its `index.ts`, which `npm run lint` enforces. + Tests are colocated with source files as `*.test.ts` where practical. Shared test mocks live in `tests/mocks/`. User-facing documentation lives in `docs/` and is built with Astro Starlight. diff --git a/src/services/publishedTranscript/parse.ts b/src/services/publishedTranscript/parse.ts index 932800d3..a0a0d34e 100644 --- a/src/services/publishedTranscript/parse.ts +++ b/src/services/publishedTranscript/parse.ts @@ -178,11 +178,12 @@ function decodeEntities(text: string): string { }); } -function timed(cues: TranscriptCue[]): ParsedTranscript | undefined { +/** Undefined when there are no cues, so an empty transcript is never a result. */ +export function timed(cues: TranscriptCue[]): ParsedTranscript | undefined { return isNonEmpty(cues) ? { kind: "timed", cues } : undefined; } -function untimed(paragraphs: string[]): ParsedTranscript | undefined { +export function untimed(paragraphs: string[]): ParsedTranscript | undefined { return isNonEmpty(paragraphs) ? { kind: "untimed", paragraphs } : undefined; } diff --git a/src/services/transcription/audioSource.ts b/src/services/transcription/audioSource.ts new file mode 100644 index 00000000..f8030e58 --- /dev/null +++ b/src/services/transcription/audioSource.ts @@ -0,0 +1,45 @@ +import { get } from "svelte/store"; +import type { FeedRepository } from "src/services/FeedRepository"; +import { publicFeedUrl } from "src/services/privateFeeds"; +import type { Episode } from "src/types/Episode"; +import { isFetchableUrl } from "src/utility/assertFetchableUrl"; +import { isCredentialBearingUrl } from "src/utility/privateFeedUrl"; + +declare const publicAudio: unique symbol; + +/** + * An enclosure address that a provider may be given. Only `publicAudioUrl` makes one, so a + * private feed's address or a local path reaches no request without a cast. + */ +export type PublicAudioUrl = string & { readonly [publicAudio]: true }; + +/** + * The episode's address when a provider may fetch it, which takes all of: + * + * - a saved public feed lists exactly this address as an item's enclosure. A private feed's + * episode never passes, even with a clean-looking address, because publishers put the + * subscriber's token in the path. Neither does a local file, which no feed lists; + * - the address carries no credential and names a public http(s) host. + * + * Anything else resolves undefined and is uploaded, which is always correct. + */ +export async function publicAudioUrl( + episode: Episode, + feeds: Pick, +): Promise { + const key = episode.podcastName; + const address = episode.streamUrl; + /** Undefined when the episode has no saved public feed. */ + const listed = (): boolean | undefined => { + const state = get(feeds).get(key); + if (!state || !publicFeedUrl(state.feed)) return undefined; + return state.episodes.some((item) => item.streamUrl === address); + }; + + if (isCredentialBearingUrl(address) || !isFetchableUrl(address)) return undefined; + if (listed() === undefined) return undefined; + const loaded = await feeds.load([key], "cached"); + // The cache keeps a long feed's newest items, so an older episode needs the whole feed. + if (listed() === false && loaded) await feeds.load([key], "complete"); + return listed() ? (address as PublicAudioUrl) : undefined; +} diff --git a/src/services/transcription/cues.test.ts b/src/services/transcription/cues.test.ts new file mode 100644 index 00000000..38783356 --- /dev/null +++ b/src/services/transcription/cues.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "vitest"; +import { transcriptFromParts } from "./cues"; + +describe("transcriptFromParts", () => { + test("times each word from the start of the episode, and never shares a speaker between parts", () => { + const transcript = transcriptFromParts([ + { + start: 0, + transcript: { + words: [ + { text: "Hello.", start: 1.5, end: 2, speaker: "0" }, + { text: "Hi.", start: 3, end: 3.5, speaker: "1" }, + ], + }, + }, + { + start: 1800, + transcript: { + words: [{ text: "Welcome back.", start: 1.5, end: 2, speaker: "0" }], + }, + }, + ]); + + expect(transcript).toEqual({ + kind: "timed", + cues: [ + { start: 1.5, end: 2, speaker: "1", text: "Hello." }, + { start: 3, end: 3.5, speaker: "2", text: "Hi." }, + { start: 1801.5, end: 1802, speaker: "3", text: "Welcome back." }, + ], + }); + }); + + test("marks a failed part at the time it starts", () => { + const transcript = transcriptFromParts([ + { start: 0, transcript: { words: [{ text: "Hello.", start: 1, end: 2 }] } }, + { start: 1800 }, + ]); + + expect(transcript).toEqual({ + kind: "timed", + cues: [ + { start: 1, end: 2, speaker: undefined, text: "Hello." }, + { start: 1800, text: "[Error transcribing chunk 2]" }, + ], + }); + }); +}); diff --git a/src/services/transcription/cues.ts b/src/services/transcription/cues.ts new file mode 100644 index 00000000..9604eafe --- /dev/null +++ b/src/services/transcription/cues.ts @@ -0,0 +1,91 @@ +import { + type ParsedTranscript, + timed, + type TranscriptCue, + untimed, +} from "src/services/publishedTranscript/parse"; +import type { PartTranscript, Word } from "./providers/types"; + +/** What one part of the episode gave back. `transcript` is absent when its request failed. */ +export interface TranscribedPart { + /** Seconds into the episode where the part starts. */ + readonly start: number; + readonly transcript?: PartTranscript; +} + +type TimedPart = TranscribedPart & { readonly transcript?: { readonly words: readonly Word[] } }; + +const PARAGRAPH_LENGTH = 500; + +/** Stands in the note for a part whose request failed, so the reader sees where speech is missing. */ +function failedPartMarker(index: number): string { + return `[Error transcribing chunk ${index + 1}]`; +} + +/** + * The episode's transcript from its parts in order, or undefined when no part holds speech. + * Words become one cue each, timed from the start of the episode, and the renderer folds + * them into paragraphs as it does for a published word-level transcript. Text from a model + * that returns no times becomes untimed paragraphs. + */ +export function transcriptFromParts( + parts: readonly TranscribedPart[], +): ParsedTranscript | undefined { + if (!parts.some(({ transcript }) => transcript && textOf(transcript).trim())) return undefined; + if (isTimed(parts)) return timed(cuesOf(parts)); + const text = parts.map(({ transcript }, index) => + transcript ? textOf(transcript) : failedPartMarker(index), + ); + return untimed(paragraphsOf(text.join(" "))); +} + +function textOf(transcript: PartTranscript): string { + return "text" in transcript + ? transcript.text + : transcript.words.map((word) => word.text).join(" "); +} + +function isTimed(parts: readonly TranscribedPart[]): parts is readonly TimedPart[] { + return parts.every(({ transcript }) => !transcript || "words" in transcript); +} + +/** Speakers are numbered in the order they first speak. A provider's label names a speaker only within its own part. */ +function cuesOf(parts: readonly TimedPart[]): TranscriptCue[] { + const numbers = new Map(); + const numberOf = (label: string) => { + const number = numbers.get(label) ?? `${numbers.size + 1}`; + numbers.set(label, number); + return number; + }; + return parts.flatMap(({ start, transcript }, index): TranscriptCue[] => + transcript + ? transcript.words.map((word) => ({ + start: start + word.start, + end: start + word.end, + speaker: + word.speaker === undefined + ? undefined + : numberOf(`${index}:${word.speaker}`), + text: word.text, + })) + : [{ start, text: failedPartMarker(index) }], + ); +} + +/** Sentences, grouped until a paragraph passes 500 characters. */ +function paragraphsOf(text: string): string[] { + const paragraphs: string[] = []; + let paragraph = ""; + for (const sentence of text + .trim() + .replace(/([.!?]["')\]]*)\s+/g, "$1\n") + .split("\n")) { + paragraph = paragraph ? `${paragraph} ${sentence}` : sentence; + if (paragraph.length >= PARAGRAPH_LENGTH) { + paragraphs.push(paragraph); + paragraph = ""; + } + } + if (paragraph) paragraphs.push(paragraph); + return paragraphs; +} diff --git a/src/services/transcription/http.ts b/src/services/transcription/http.ts new file mode 100644 index 00000000..885973a4 --- /dev/null +++ b/src/services/transcription/http.ts @@ -0,0 +1,188 @@ +import { requestWithTimeout, TimeoutError } from "src/utility/networkRequest"; +import type { Provider, ProviderRequest, Send } from "./providers/types"; + +/** Why a transcription failed, as the user can act on it. */ +export type FailureKind = + | "auth" + | "credits" + | "model-unavailable" + | "rejected" + | "unfetched" + | "rate-limited" + | "server" + | "network" + | "timeout" + | "empty"; + +export class TranscriptionError extends Error { + constructor( + readonly kind: FailureKind, + message: string, + /** How long the server asked to wait before another send. */ + readonly retryAfterMs?: number, + ) { + super(message); + this.name = "TranscriptionError"; + } +} + +/** What a failure sentence names. */ +export interface FailureContext { + readonly provider: string; + readonly model: string; + readonly timeoutMs: number; +} + +/** + * The sentence the user reads for each failure. PodNotes writes all of them: a provider's + * own error text can quote the address it was sent, so it is never shown or logged. + */ +const SENTENCES: Record string> = { + auth: ({ provider }) => + `${provider} rejected the API key. Check it in the transcript settings.`, + credits: ({ provider }) => + `Your ${provider} account is out of credit. Add credit there, then try again.`, + "model-unavailable": ({ provider, model }) => + `${provider} does not offer the ${model} model to this API key.`, + rejected: ({ provider }) => `${provider} could not transcribe this audio.`, + unfetched: ({ provider }) => `${provider} could not fetch the episode.`, + "rate-limited": ({ provider }) => + `${provider} is limiting how fast your key may be used. Try again in a few minutes.`, + server: ({ provider }, status) => `${provider} failed (HTTP ${status}). Try again later.`, + network: ({ provider }) => `Could not reach ${provider}. Check your connection.`, + timeout: ({ provider, timeoutMs }) => + `${provider} did not answer within ${timeoutMs / 60_000} minutes.`, + empty: ({ provider }) => `${provider} found no speech in this episode.`, +}; + +export function failure( + kind: FailureKind, + context: FailureContext, + status?: number, + retryAfterMs?: number, +): TranscriptionError { + return new TranscriptionError(kind, SENTENCES[kind](context, status), retryAfterMs); +} + +/** The failures another send can get past. */ +const RETRIED: ReadonlySet = new Set(["rate-limited", "server", "network"]); +const MAX_SENDS = 3; +const MAX_SERVER_RETRY_DELAY_MS = 60_000; +const EVERY_HTTP_STATUS = Array.from({ length: 500 }, (_, index) => 100 + index); + +function kindOf(status: number): FailureKind { + if (status === 401 || status === 403) return "auth"; + if (status === 402) return "credits"; + if (status === 404) return "model-unavailable"; + if (status === 429) return "rate-limited"; + if (status === 408 || status === 409 || status >= 500) return "server"; + return "rejected"; +} + +/** + * A request goes out at most three times, and only again after a failure another send can + * get past. A timeout is not one: Obsidian's requestUrl cannot cancel, so the provider may + * still be working, and a second send would be billed too. requestUrl is used because + * providers do not allow Obsidian's origin, which rules out `fetch`. + */ +export function createSender( + readFailure: Provider["failure"], + context: FailureContext, + signal: AbortSignal, +): Send { + const sendOnce = async (request: ProviderRequest): Promise => { + const { body, contentType } = await encode(request); + let response; + try { + response = await requestWithTimeout(request.url, { + method: "POST", + headers: request.headers, + contentType, + body, + timeoutMs: context.timeoutMs, + maxRequestBodyBytes: body.byteLength, + acceptedStatuses: EVERY_HTTP_STATUS, + signal, + }); + } catch (error) { + if (signal.aborted) throw signal.reason; + throw failure(error instanceof TimeoutError ? "timeout" : "network", context); + } + + const { status } = response; + const answered = status >= 200 && status < 300; + let json: unknown; + try { + json = JSON.parse(new TextDecoder().decode(response.arrayBuffer)); + } catch { + if (answered) throw failure("server", context, status); + } + if (answered) return json; + throw failure( + readFailure?.(status, json) ?? kindOf(status), + context, + status, + serverRetryDelayMs(new Headers(response.headers)), + ); + }; + + return async (request) => { + for (let sends = 1; ; sends++) { + try { + return await sendOnce(request); + } catch (error) { + const again = + error instanceof TranscriptionError && + RETRIED.has(error.kind) && + sends < MAX_SENDS; + if (!again) throw error; + await wait(error.retryAfterMs ?? 1000 * sends, signal); + } + } + }; +} + +async function encode({ + url, + body, +}: ProviderRequest): Promise<{ body: ArrayBuffer; contentType: string | undefined }> { + if (body instanceof File) return { body: await body.arrayBuffer(), contentType: body.type }; + if (body instanceof FormData) { + // The platform writes the multipart body and picks its boundary. + const multipart = new Request(url, { method: "POST", body }); + return { + body: await multipart.arrayBuffer(), + contentType: multipart.headers.get("content-type") ?? undefined, + }; + } + return { + body: new TextEncoder().encode(JSON.stringify(body)).buffer, + contentType: "application/json", + }; +} + +/** The server's `Retry-After`, when it asks for a minute or less. */ +function serverRetryDelayMs(headers: Headers): number | undefined { + let delayMs = parseFloat(headers.get("retry-after-ms") ?? ""); + const retryAfter = headers.get("retry-after"); + if (Number.isNaN(delayMs) && retryAfter) { + const seconds = parseFloat(retryAfter); + delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000; + } + return delayMs >= 0 && delayMs <= MAX_SERVER_RETRY_DELAY_MS ? delayMs : undefined; +} + +function wait(delayMs: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const onAbort = () => { + window.clearTimeout(timeout); + reject(signal.reason); + }; + const timeout = window.setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); +} diff --git a/src/services/transcription/index.ts b/src/services/transcription/index.ts new file mode 100644 index 00000000..8f982006 --- /dev/null +++ b/src/services/transcription/index.ts @@ -0,0 +1,2 @@ +export { transcribe } from "./transcribe"; +export { transcriptionChoice } from "./registry"; diff --git a/src/services/transcription/providers/deepgram.ts b/src/services/transcription/providers/deepgram.ts new file mode 100644 index 00000000..a2a3b3ec --- /dev/null +++ b/src/services/transcription/providers/deepgram.ts @@ -0,0 +1,57 @@ +import { isPlainObject } from "src/persistence/codecUtils"; +import type { PartTranscript, Provider, Word } from "./types"; + +/** + * https://developers.deepgram.com/reference/speech-to-text/listen-pre-recorded + * + * The query is the released one. Deepgram now prefers `diarize_model=latest` over + * `diarize=true`. Switch once a real key has confirmed the answer keeps its shape. + */ +export const deepgram: Provider = { + label: "Deepgram", + models: [{ id: "nova-3", speakers: true, url: true, maxUploadBytes: 2_000_000_000 }], + timeoutMs: 30 * 60_000, + + async transcribe({ apiKey, model, audio }, send) { + return parse( + await send({ + url: `https://api.deepgram.com/v1/listen?model=${model.id}&diarize=true&punctuate=true&smart_format=true&utterances=true`, + headers: { Authorization: `Token ${apiKey}` }, + body: "url" in audio ? { url: audio.url } : audio.file, + }), + ); + }, + + // Deepgram could not download the address (https://developers.deepgram.com/docs/errors). + failure: (status, body) => + status === 400 && isPlainObject(body) && body.err_code === "REMOTE_CONTENT_ERROR" + ? "unfetched" + : undefined, +}; + +interface Answer { + results?: { channels?: { alternatives?: { words?: unknown }[] }[] }; +} + +function parse(answer: unknown): PartTranscript { + const words = (answer as Answer | null)?.results?.channels?.[0]?.alternatives?.[0]?.words; + if (!Array.isArray(words)) return { words: [] }; + return { + words: words.flatMap((word: unknown): Word[] => { + if (!isPlainObject(word)) return []; + const text = word.punctuated_word ?? word.word; + const { start, end, speaker } = word; + if (typeof text !== "string" || typeof start !== "number" || typeof end !== "number") { + return []; + } + return [ + { + text, + start, + end, + speaker: typeof speaker === "number" ? `${speaker}` : undefined, + }, + ]; + }), + }; +} diff --git a/src/services/transcription/providers/openai.ts b/src/services/transcription/providers/openai.ts new file mode 100644 index 00000000..75832a93 --- /dev/null +++ b/src/services/transcription/providers/openai.ts @@ -0,0 +1,49 @@ +import { isPlainObject } from "src/persistence/codecUtils"; +import type { Provider } from "./types"; + +/** + * https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create + * + * gpt-transcribe answers with text only, so an OpenAI transcript has no timestamps and no + * speaker labels. OpenAI shuts down its other transcription models on 2027-02-26. + */ +export const openai: Provider = { + label: "OpenAI", + models: [ + { + id: "gpt-transcribe", + speakers: false, + url: false, + // OpenAI's limit is 25 MB a request. + maxUploadBytes: 24_000_000, + // OpenAI states no length limit, so this is the longest request measured to work. + maxSeconds: 2070, + }, + ], + timeoutMs: 10 * 60_000, + + async transcribe({ apiKey, model, audio }, send) { + if ("url" in audio) throw new Error("OpenAI takes no audio address."); + const form = new FormData(); + form.append("model", model.id); + form.append("file", audio.file); + form.append("response_format", "json"); + const answer = await send({ + url: "https://api.openai.com/v1/audio/transcriptions", + headers: { Authorization: `Bearer ${apiKey}` }, + body: form, + }); + return { + text: isPlainObject(answer) && typeof answer.text === "string" ? answer.text : "", + }; + }, + + // A used-up quota answers 429 like a rate limit, but waiting does not cure it. + failure: (status, body) => + status === 429 && + isPlainObject(body) && + isPlainObject(body.error) && + body.error.code === "insufficient_quota" + ? "credits" + : undefined, +}; diff --git a/src/services/transcription/providers/types.ts b/src/services/transcription/providers/types.ts new file mode 100644 index 00000000..fa7bce39 --- /dev/null +++ b/src/services/transcription/providers/types.ts @@ -0,0 +1,61 @@ +import type { NonEmpty } from "src/services/publishedTranscript/parse"; +import type { PublicAudioUrl } from "../audioSource"; +import type { FailureKind } from "../http"; + +/** A model PodNotes has a tested request for, with what that request can carry. */ +export interface ModelProfile { + /** The ID the provider's API takes. */ + readonly id: string; + /** Labels who spoke. */ + readonly speakers: boolean; + /** Fetches a public address itself, so PodNotes uploads nothing. */ + readonly url: boolean; + /** The largest file one request may carry. */ + readonly maxUploadBytes: number; + /** The most audio, in seconds, one request may carry. */ + readonly maxSeconds?: number; +} + +export interface Word { + readonly text: string; + /** Seconds from the start of the audio this request carried. */ + readonly start: number; + readonly end: number; + /** The provider's label for who spoke. Labels from two requests are unrelated. */ + readonly speaker?: string; +} + +/** What one request gave back: timed words, or text from a model that returns no times. */ +export type PartTranscript = { readonly words: readonly Word[] } | { readonly text: string }; + +export type AudioInput = { readonly url: PublicAudioUrl } | { readonly file: File }; + +export interface PartInput { + readonly apiKey: string; + readonly model: ModelProfile; + /** A `url` only for a model whose profile says `url`. */ + readonly audio: AudioInput; +} + +export interface ProviderRequest { + readonly url: string; + readonly headers: Readonly>; + /** FormData goes out as multipart, a File as its bytes with its type, anything else as JSON. */ + readonly body: FormData | File | Readonly>; +} + +/** POSTs one request and resolves with the JSON of a 2xx answer. Rejects with a TranscriptionError. */ +export type Send = (request: ProviderRequest) => Promise; + +/** Everything that is one provider's own: its models, its request and how to read its answers. */ +export interface Provider { + /** The name notices and buttons show. */ + readonly label: string; + /** Best first. The first is the one PodNotes uses. */ + readonly models: NonEmpty; + /** How long PodNotes waits for one answer. */ + readonly timeoutMs: number; + transcribe(input: PartInput, send: Send): Promise; + /** What a failed answer means here, when the status alone does not say. `body` is the answer's JSON. */ + failure?(status: number, body: unknown): FailureKind | undefined; +} diff --git a/src/services/transcription/registry.ts b/src/services/transcription/registry.ts new file mode 100644 index 00000000..3dfe8665 --- /dev/null +++ b/src/services/transcription/registry.ts @@ -0,0 +1,37 @@ +import type { CredentialKind } from "src/types/Credentials"; +import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; +import { deepgram } from "./providers/deepgram"; +import { openai } from "./providers/openai"; +import type { ModelProfile, Provider } from "./providers/types"; + +/** A provider is keyed by the API key it needs, so one without a credential kind fails the build. */ +export const PROVIDERS = { openai, deepgram } satisfies Record; + +export interface TranscriptionChoice { + readonly provider: CredentialKind; + /** The provider's name, for buttons and notices. */ + readonly label: string; + readonly model: ModelProfile; + /** The settings ask for speaker labels. Whether the model has them is `model.speakers`. */ + readonly speakers: boolean; +} + +/** + * What the transcript settings choose. The only reader of the settings that pick a provider, + * so the panel button, the key check and the run agree. + */ +export function transcriptionChoice({ + transcript, +}: Pick): TranscriptionChoice { + const { enabled, provider: selected } = transcript.diarization; + const provider = enabled && selected === "deepgram" ? "deepgram" : "openai"; + const { label, models } = PROVIDERS[provider]; + return { provider, label, model: models[0], speakers: enabled }; +} + +/** The names of the providers that label speakers. */ +export function providersWithSpeakers(): string[] { + return Object.values(PROVIDERS) + .filter(({ models }) => models[0].speakers) + .map(({ label }) => label); +} diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts new file mode 100644 index 00000000..64c8675b --- /dev/null +++ b/src/services/transcription/split.test.ts @@ -0,0 +1,198 @@ +// @vitest-environment jsdom +import { afterEach, describe, expect, test, vi } from "vitest"; +import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../../tests/mocks/mp3"; +import { type PartCaps, splitAudio } from "./split"; + +const MIB = 1024 * 1024; + +const split = async (bytes: Uint8Array, caps: PartCaps, extension = "mp3") => + Promise.all( + (await splitAudio({ buffer: bytes.buffer as ArrayBuffer, extension }, caps)).map( + async ({ file, start }) => ({ + name: file.name, + type: file.type, + start, + bytes: new Uint8Array(await file.arrayBuffer()), + }), + ), + ); +const joinsInto = (parts: { bytes: Uint8Array }[], episode: Uint8Array) => + Buffer.concat(parts.map((part) => part.bytes)).equals(episode); + +describe("audio within the caps", () => { + test.each([ + ["mp3", "audio/mp3"], + ["MP3", "audio/mp3"], + ["m4a", "audio/mp4"], + ["ogg", "audio/ogg"], + ["wav", "audio/wav"], + ["flac", "audio/flac"], + ["webm", "audio/webm"], + ["opus", "audio/mpeg"], + ])("a .%s file goes whole as %s, named for its format only", async (extension, type) => { + const episode = new Uint8Array(4096).fill(7); + + const parts = await split(episode, { maxBytes: 20 * MIB }, extension); + + expect(parts).toEqual([{ name: `audio.${extension}`, type, start: 0, bytes: episode }]); + }); + + test("an MP3 that fits both caps goes whole", async () => { + const episode = mp3Bytes(MP3_STREAMS.mpeg1At96Kbps, 58_000); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect(parts.map((part) => part.name)).toEqual(["audio.mp3"]); + expect(joinsInto(parts, episode)).toBe(true); + }); +}); + +describe("an MP3 over a cap", () => { + test("under the size cap is cut into equal parts no longer than maxSeconds, each starting where the last ended", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + const episode = mp3Bytes(stream, 134_000); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect(parts.map((part) => part.name)).toEqual([ + "audio.part0.mp3", + "audio.part1.mp3", + "audio.part2.mp3", + ]); + expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ + 44_667, 44_667, 44_666, + ]); + expect(mp3Seconds(stream, parts[0].bytes.length)).toBeLessThanOrEqual(1400); + expect(parts[0].start).toBe(0); + expect(parts[1].start).toBeCloseTo(44_667 * stream.frameSeconds, 6); + expect(parts[2].start).toBeCloseTo(89_334 * stream.frameSeconds, 6); + for (const part of parts) { + expect(part.type).toBe("audio/mp3"); + expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); + } + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("over the size cap is cut at frame starts into parts under it", async () => { + const stream = MP3_STREAMS.mpeg1At320Kbps; + const episode = mp3Bytes(stream, 50_000); + + const parts = await split(episode, { maxBytes: 20 * MIB }); + + expect(parts).toHaveLength(3); + let frames = 0; + for (const part of parts) { + expect(part.bytes.length).toBeLessThanOrEqual(20 * MIB); + expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); + expect(part.start).toBeCloseTo(frames * stream.frameSeconds, 6); + frames += part.bytes.length / stream.frameBytes; + } + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("keeps a leading ID3 tag in the first part without counting sync-like bytes in it as audio", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + const tag = new Uint8Array(8192); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x3f, 0x76]); + for (let offset = 100; offset <= 5000; offset += 100) tag.set(stream.header, offset); + const frames = mp3Bytes(stream, 36_000); + const episode = Buffer.concat([tag, frames]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + tag.length + 18_000 * stream.frameBytes, + 18_000 * stream.frameBytes, + ]); + expect(Array.from(parts[0].bytes.subarray(0, 3))).toEqual([0x49, 0x44, 0x33]); + expect(parts[1].start).toBeCloseTo(18_000 * stream.frameSeconds, 6); + expect(joinsInto(parts, episode)).toBe(true); + }); +}); + +describe("audio over the size cap that cannot be cut where it is", () => { + const SAMPLE_RATE = 44_100; + + // jsdom has no Web Audio, so a decoder that returns silence of the given length stands in. + class FakeAudioBuffer { + numberOfChannels = 1; + sampleRate = SAMPLE_RATE; + private channel: Float32Array; + constructor(readonly length: number) { + this.channel = new Float32Array(length); + } + getChannelData(): Float32Array { + return this.channel; + } + } + + function stubAudioContext(decode: (buffer: ArrayBuffer) => unknown): void { + vi.stubGlobal( + "AudioContext", + class { + async decodeAudioData(buffer: ArrayBuffer): Promise { + return decode(buffer); + } + async close(): Promise {} + }, + ); + } + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + test.each(["ogg", "mp3"])( + "a .%s file without MP3 frames is decoded into WAV parts under the cap, each with its start time", + async (extension) => { + const samples = 100_000; + stubAudioContext(() => new FakeAudioBuffer(samples)); + const maxBytes = 64 * 1024; + const samplesPerPart = (maxBytes - 44) / 2; + + const parts = await split(new Uint8Array(maxBytes + 1), { maxBytes }, extension); + + expect(parts.map((part) => part.name)).toEqual([ + "audio.part0.wav", + "audio.part1.wav", + "audio.part2.wav", + "audio.part3.wav", + ]); + expect(parts.map((part) => part.start)).toEqual( + [0, 1, 2, 3].map((index) => (index * samplesPerPart) / SAMPLE_RATE), + ); + for (const part of parts) { + expect(part.type).toBe("audio/wav"); + expect(part.bytes.length).toBeLessThanOrEqual(maxBytes); + const view = new DataView(part.bytes.buffer); + expect(new TextDecoder().decode(part.bytes.subarray(0, 4))).toBe("RIFF"); + expect(new TextDecoder().decode(part.bytes.subarray(8, 12))).toBe("WAVE"); + expect(view.getUint16(22, true)).toBe(1); + expect(view.getUint32(24, true)).toBe(SAMPLE_RATE); + expect(view.getUint16(34, true)).toBe(16); + expect(view.getUint32(40, true)).toBe(part.bytes.length - 44); + } + expect(parts.reduce((total, part) => total + (part.bytes.length - 44) / 2, 0)).toBe( + samples, + ); + }, + ); + + test("throws when no decoder is available", async () => { + await expect(split(new Uint8Array(1025), { maxBytes: 1024 }, "flac")).rejects.toThrow( + /Could not split flac audio/, + ); + }); + + test("throws when the decoder cannot decode it", async () => { + stubAudioContext(() => { + throw new Error("Unsupported codec"); + }); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(split(new Uint8Array(1025), { maxBytes: 1024 }, "webm")).rejects.toThrow( + /Could not split webm audio/, + ); + consoleWarn.mockRestore(); + }); +}); diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts new file mode 100644 index 00000000..f11ff12f --- /dev/null +++ b/src/services/transcription/split.ts @@ -0,0 +1,282 @@ +const WAV_HEADER_SIZE = 44; +const PCM_BYTES_PER_SAMPLE = 2; + +export interface AudioPart { + readonly file: File; + /** Seconds into the episode where this part starts. */ + readonly start: number; +} + +/** What one request may carry. `maxSeconds` limits MP3 parts only, because other formats are not measured without decoding them. */ +export interface PartCaps { + readonly maxBytes: number; + readonly maxSeconds?: number; +} + +function getMimeType(fileExtension: string): string { + switch (fileExtension.toLowerCase()) { + case "mp3": + return "audio/mp3"; + case "m4a": + return "audio/mp4"; + case "ogg": + return "audio/ogg"; + case "wav": + return "audio/wav"; + case "flac": + return "audio/flac"; + case "webm": + return "audio/webm"; + default: + return "audio/mpeg"; + } +} + +/** + * The files to send for an episode, in order. Audio within the caps goes whole, in its own + * format. A larger MP3 is cut at frame starts, where each part decodes on its own and its + * start time is known. Every other format keeps its codec setup at the start of the file, + * so it is decoded and sent as WAV parts. + * + * Files are named for their format only: a local file's own name is part of its path. + */ +export async function splitAudio( + audio: { readonly buffer: ArrayBuffer; readonly extension: string }, + { maxBytes, maxSeconds = Infinity }: PartCaps, +): Promise { + const { buffer, extension } = audio; + const type = getMimeType(extension); + const whole = () => [{ file: new File([buffer], `audio.${extension}`, { type }), start: 0 }]; + const fits = buffer.byteLength <= maxBytes; + if (fits && maxSeconds === Infinity) return whole(); + + const frames = type === "audio/mp3" ? mp3Frames(new Uint8Array(buffer)) : []; + if (frames.length > 0) { + const starts = mp3PartStarts(frames, buffer.byteLength, maxBytes, maxSeconds); + if (starts.length === 1) return whole(); + return starts.map(({ offset, seconds }, index) => ({ + file: new File( + [buffer.slice(offset, starts[index + 1]?.offset)], + `audio.part${index}.${extension}`, + { type }, + ), + start: seconds, + })); + } + + if (fits) return whole(); + const parts = await convertToWavParts(buffer, maxBytes); + if (parts.length === 0) { + throw new Error( + `Could not split ${extension || "audio"} audio for transcription. This format must be decoded to WAV before splitting, but the audio decoder is unavailable or could not decode it.`, + ); + } + return parts; +} + +interface Mp3Frame { + offset: number; + length: number; + seconds: number; +} + +const MPEG1_LAYER3_KBPS = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320]; +const MPEG2_LAYER3_KBPS = [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160]; +/** Indexed by the header's version bits: 0 is MPEG-2.5, 2 is MPEG-2, 3 is MPEG-1. */ +const SAMPLE_RATES = [ + [11025, 12000, 8000], + undefined, + [22050, 24000, 16000], + [44100, 48000, 32000], +]; + +/** + * Where each part of an MP3 starts, in bytes and in seconds of audio. Parts start at frame + * starts, there are as few as both caps allow, and they hold an equal number of frames, so + * no part is a few seconds long. The first part starts at byte 0 and keeps a leading tag. + */ +function mp3PartStarts( + frames: Mp3Frame[], + byteLength: number, + maxBytes: number, + maxSeconds: number, +): { offset: number; seconds: number }[] { + const total = frames.reduce((seconds, frame) => seconds + frame.seconds, 0); + const parts = Math.max(Math.ceil(byteLength / maxBytes), Math.ceil(total / maxSeconds)); + const framesPerPart = Math.ceil(frames.length / parts); + const starts = [{ offset: 0, seconds: 0 }]; + let elapsed = 0; + let partFrames = 0; + let partSeconds = 0; + for (const frame of frames) { + if ( + partFrames === framesPerPart || + frame.offset + frame.length - starts[starts.length - 1].offset > maxBytes || + partSeconds + frame.seconds > maxSeconds + ) { + starts.push({ offset: frame.offset, seconds: elapsed }); + partFrames = 0; + partSeconds = 0; + } + partFrames++; + partSeconds += frame.seconds; + elapsed += frame.seconds; + } + return starts; +} + +/** + * The Layer III frames in `bytes`. A frame counts only when another frame or the + * end of the buffer follows it, so sync-like bytes in a tag or cover art don't. + */ +function mp3Frames(bytes: Uint8Array): Mp3Frame[] { + const frames: Mp3Frame[] = []; + let offset = 0; + while (offset + 4 <= bytes.length) { + const frame = mp3FrameAt(bytes, offset); + const next = frame ? offset + frame.length : 0; + if (frame && (next === bytes.length || mp3FrameAt(bytes, next))) { + frames.push(frame); + offset = next; + } else { + offset++; + } + } + return frames; +} + +function mp3FrameAt(bytes: Uint8Array, offset: number): Mp3Frame | undefined { + if (offset + 4 > bytes.length || bytes[offset] !== 0xff) return undefined; + const versionAndLayer = bytes[offset + 1]; + const rates = bytes[offset + 2]; + const version = (versionAndLayer >> 3) & 3; + const isLayer3 = ((versionAndLayer >> 1) & 3) === 1; + if ((versionAndLayer & 0xe0) !== 0xe0 || !isLayer3) return undefined; + const kbps = (version === 3 ? MPEG1_LAYER3_KBPS : MPEG2_LAYER3_KBPS)[rates >> 4]; + const sampleRate = SAMPLE_RATES[version]?.[(rates >> 2) & 3]; + if (!kbps || !sampleRate) return undefined; + const samples = version === 3 ? 1152 : 576; + const padding = (rates >> 1) & 1; + return { + offset, + length: Math.floor(((samples / 8) * kbps * 1000) / sampleRate) + padding, + seconds: samples / sampleRate, + }; +} + +async function convertToWavParts(buffer: ArrayBuffer, maxBytes: number): Promise { + const audioContext = createAudioContext(); + if (!audioContext) return []; + + try { + const audioBuffer = await audioContext.decodeAudioData(buffer.slice(0)); + return renderWavParts(audioBuffer, maxBytes); + } catch (error) { + console.warn("Failed to convert audio buffer for transcription", error); + return []; + } finally { + try { + await audioContext.close(); + } catch (error) { + console.warn("Failed to close audio context", error); + } + } +} + +function createAudioContext(): AudioContext | null { + if (typeof window === "undefined") { + return null; + } + + const contextCtor = + window.AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + if (!contextCtor) { + return null; + } + + return new contextCtor(); +} + +function renderWavParts(audioBuffer: AudioBuffer, maxBytes: number): AudioPart[] { + const numChannels = audioBuffer.numberOfChannels; + const bytesPerFrame = numChannels * PCM_BYTES_PER_SAMPLE; + const maxSamplesPerPart = Math.max(1, Math.floor((maxBytes - WAV_HEADER_SIZE) / bytesPerFrame)); + const channelData = Array.from({ length: numChannels }, (_, channelIndex) => + audioBuffer.getChannelData(channelIndex), + ); + const parts: AudioPart[] = []; + + for (let startSample = 0; startSample < audioBuffer.length; startSample += maxSamplesPerPart) { + const endSample = Math.min(audioBuffer.length, startSample + maxSamplesPerPart); + const wavBuffer = renderWavBuffer( + channelData, + audioBuffer.sampleRate, + startSample, + endSample, + ); + parts.push({ + file: new File([wavBuffer], `audio.part${parts.length}.wav`, { type: "audio/wav" }), + start: startSample / audioBuffer.sampleRate, + }); + } + + return parts; +} + +function renderWavBuffer( + channelData: Float32Array[], + sampleRate: number, + startSample: number, + endSample: number, +): ArrayBuffer { + const numChannels = channelData.length; + const sampleCount = Math.max(0, endSample - startSample); + const blockAlign = numChannels * PCM_BYTES_PER_SAMPLE; + const buffer = new ArrayBuffer(WAV_HEADER_SIZE + sampleCount * blockAlign); + const view = new DataView(buffer); + writeWavHeader(view, sampleRate, numChannels, sampleCount); + let offset = WAV_HEADER_SIZE; + + for (let i = 0; i < sampleCount; i++) { + for (let channel = 0; channel < numChannels; channel++) { + const sample = channelData[channel][startSample + i] ?? 0; + const clamped = Math.max(-1, Math.min(1, sample)); + const intSample = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + view.setInt16(offset, Math.round(intSample), true); + offset += PCM_BYTES_PER_SAMPLE; + } + } + + return buffer; +} + +function writeWavHeader( + view: DataView, + sampleRate: number, + numChannels: number, + sampleCount: number, +): void { + const blockAlign = numChannels * PCM_BYTES_PER_SAMPLE; + const byteRate = sampleRate * blockAlign; + const dataSize = sampleCount * blockAlign; + writeString(view, 0, "RIFF"); + view.setUint32(4, 36 + dataSize, true); + writeString(view, 8, "WAVE"); + writeString(view, 12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, PCM_BYTES_PER_SAMPLE * 8, true); + writeString(view, 36, "data"); + view.setUint32(40, dataSize, true); +} + +function writeString(view: DataView, offset: number, str: string): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts new file mode 100644 index 00000000..a4b4d068 --- /dev/null +++ b/src/services/transcription/transcribe.ts @@ -0,0 +1,104 @@ +import { getEpisodeAudioBuffer } from "src/downloadEpisode"; +import type { FeedRepository } from "src/services/FeedRepository"; +import type { ParsedTranscript } from "src/services/publishedTranscript/parse"; +import type { Episode } from "src/types/Episode"; +import { publicAudioUrl } from "./audioSource"; +import { type TranscribedPart, transcriptFromParts } from "./cues"; +import { createSender, failure, type FailureKind, TranscriptionError } from "./http"; +import type { AudioInput } from "./providers/types"; +import { PROVIDERS, providersWithSpeakers, type TranscriptionChoice } from "./registry"; +import { splitAudio } from "./split"; + +export interface TranscriptionJob { + readonly choice: TranscriptionChoice; + readonly apiKey: string; + readonly episode: Episode; + /** Read to tell whether the provider may be given the episode's address. */ + readonly feeds: Pick; + readonly signal: AbortSignal; + readonly onProgress: (message: string) => void; +} + +export interface Transcript { + readonly content: ParsedTranscript; + /** One sentence each, for a transcript that is not all that was asked for. */ + readonly warnings: readonly string[]; +} + +/** The failures the next part would meet too, so the job stops instead of uploading the rest. */ +const ENDS_THE_JOB: ReadonlySet = new Set([ + "auth", + "credits", + "model-unavailable", + "rejected", +]); + +/** + * The episode's transcript from the chosen provider. Rejects with a TranscriptionError, + * whose message is the sentence the user reads, or with the signal's reason once aborted. + */ +export async function transcribe(job: TranscriptionJob): Promise { + const { choice, apiKey, episode, signal, onProgress } = job; + const { model } = choice; + const provider = PROVIDERS[choice.provider]; + const context = { provider: provider.label, model: model.id, timeoutMs: provider.timeoutMs }; + const send = createSender(provider.failure, context, signal); + const transcribeOne = (audio: AudioInput) => + provider.transcribe({ apiKey, model, audio }, send); + const progress = `Transcribing with ${provider.label}...`; + + const parts: TranscribedPart[] = []; + let lastFailure: TranscriptionError | undefined; + + const url = model.url ? await publicAudioUrl(episode, job.feeds) : undefined; + if (url) { + onProgress(progress); + try { + parts.push({ start: 0, transcript: await transcribeOne({ url }) }); + } catch (error) { + // Only a provider that could not download the address gets the bytes instead. + if (!(error instanceof TranscriptionError) || error.kind !== "unfetched") throw error; + } + } + + if (parts.length === 0) { + onProgress("Fetching episode audio..."); + const files = await splitAudio(await getEpisodeAudioBuffer(episode), { + maxBytes: model.maxUploadBytes, + maxSeconds: model.maxSeconds, + }); + // One part at a time: an upload is held in memory whole, and a failure that ends the + // job ends it before the next part is sent. + for (const [index, { file, start }] of files.entries()) { + onProgress( + files.length === 1 ? progress : `${progress} part ${index + 1} of ${files.length}`, + ); + try { + parts.push({ start, transcript: await transcribeOne({ file }) }); + } catch (error) { + if (!(error instanceof TranscriptionError) || ENDS_THE_JOB.has(error.kind)) { + throw error; + } + lastFailure = error; + parts.push({ start }); + } + } + } + + const content = transcriptFromParts(parts); + if (!content) throw lastFailure ?? failure("empty", context); + + const warnings: string[] = []; + const failed = parts.filter((part) => !part.transcript).length; + if (failed > 0) { + warnings.push( + `Transcription saved, but ${failed} of ${parts.length} chunk(s) failed - look for [Error transcribing chunk N] markers and re-run after deleting the note to retry.`, + ); + } + if (choice.speakers && !model.speakers) { + warnings.push( + `Saved without speaker labels, because ${provider.label} does not label speakers. These providers do: ${providersWithSpeakers().join(", ")}.`, + ); + } + return { content, warnings }; +} From 42b55a3995688672decb3e8ad2941f0edceaa894 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 22:24:34 +0100 Subject: [PATCH 08/26] feat(transcription): transcribe through the engine with current models TranscriptionService keeps its queue, the published-transcript path, the progress notice, and note saving. It calls the engine for audio. - OpenAI transcribes with gpt-transcribe. whisper-1 and gpt-4o-transcribe-diarize, which OpenAI shuts down on 2027-02-26, are gone. A tooling test searches the built main.js for them. - With speaker labels on and OpenAI selected, the transcript is saved without labels. A notice that stays until clicked names the providers that label speakers. - Deepgram transcripts go through renderTranscriptBody, so each paragraph has a timestamp. Speakers are numbered in the order they first speak. - Deepgram gets the address of a public episode and downloads it itself. - A failure notice stays until clicked. Parts go out one at a time, and a 401, 402, 404, or 400 stops the job before the next part. Deletes src/services/diarization/, openaiTranscription.ts, the Whisper worker pool, and createBinaryChunkFiles. getEpisodeAudioBuffer no longer returns a basename, because uploads are named for their format only. --- src/constants.ts | 2 - src/downloadEpisode.test.ts | 9 - src/downloadEpisode.ts | 12 +- .../TranscriptionService.published.test.ts | 43 +- src/services/TranscriptionService.test.ts | 1030 ++++++++++------- src/services/TranscriptionService.ts | 340 +----- src/services/audioChunker.test.ts | 354 ------ src/services/audioChunker.ts | 348 ------ .../diarization/deepgramProvider.test.ts | 183 --- src/services/diarization/deepgramProvider.ts | 51 - src/services/diarization/index.ts | 24 - .../diarization/openaiProvider.test.ts | 184 --- src/services/diarization/openaiProvider.ts | 113 -- .../requiredTranscriptionKeyPresent.test.ts | 60 - src/services/diarization/segments.test.ts | 189 --- src/services/diarization/segments.ts | 152 --- src/services/diarization/types.ts | 50 - src/services/openaiTranscription.test.ts | 171 --- src/services/openaiTranscription.ts | 100 -- src/services/publishedTranscript/index.ts | 1 + src/settingsMigrations.ts | 9 +- src/types/IPodNotesSettings.ts | 10 +- src/ui/PodcastView/TranscriptPanel.svelte | 6 +- src/ui/settings/PodNotesSettingsTab.ts | 4 +- tests/e2e/journey-resilience.test.ts | 8 +- tests/e2e/journey/fixtures.ts | 2 + tests/mocks/openaiFailures.ts | 60 - tests/mocks/openaiRequestUrl.ts | 31 - tests/mocks/providerRequestUrl.ts | 47 + tests/mocks/transcriptionService.ts | 53 +- tests/tooling/transcriptionModels.test.ts | 28 + 31 files changed, 809 insertions(+), 2865 deletions(-) delete mode 100644 src/services/audioChunker.test.ts delete mode 100644 src/services/audioChunker.ts delete mode 100644 src/services/diarization/deepgramProvider.test.ts delete mode 100644 src/services/diarization/deepgramProvider.ts delete mode 100644 src/services/diarization/index.ts delete mode 100644 src/services/diarization/openaiProvider.test.ts delete mode 100644 src/services/diarization/openaiProvider.ts delete mode 100644 src/services/diarization/requiredTranscriptionKeyPresent.test.ts delete mode 100644 src/services/diarization/segments.test.ts delete mode 100644 src/services/diarization/segments.ts delete mode 100644 src/services/diarization/types.ts delete mode 100644 src/services/openaiTranscription.test.ts delete mode 100644 src/services/openaiTranscription.ts delete mode 100644 tests/mocks/openaiFailures.ts delete mode 100644 tests/mocks/openaiRequestUrl.ts create mode 100644 tests/mocks/providerRequestUrl.ts create mode 100644 tests/tooling/transcriptionModels.test.ts diff --git a/src/constants.ts b/src/constants.ts index c9114fd4..fa9a3dcf 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -161,8 +161,6 @@ export const DEFAULT_SETTINGS: IPodNotesSettings = { transcript: { path: "transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\nPodcast: {{podcast}}\nDate: {{date}}\n\n{{transcript}}", - // Diarization is off by default so existing behaviour (plain Whisper) is - // unchanged; enabling it routes audio to the chosen provider (#168). diarization: { enabled: false, provider: "openai", diff --git a/src/downloadEpisode.test.ts b/src/downloadEpisode.test.ts index fec75c20..50563815 100644 --- a/src/downloadEpisode.test.ts +++ b/src/downloadEpisode.test.ts @@ -941,10 +941,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { const slash = path.lastIndexOf("/"); (file as { path: string }).path = path; (file as { extension: string }).extension = dot > slash ? path.slice(dot + 1) : ""; - (file as { basename: string }).basename = path.slice( - slash + 1, - dot > slash ? dot : undefined, - ); return file; } @@ -1031,7 +1027,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { expect(decode(result.buffer)).toBe("CACHED-CORRECT-AUDIO"); expect(result.extension).toBe("mp3"); - expect(result.basename).toBe("cached"); expect(requestUrlMock).not.toHaveBeenCalled(); }); @@ -1357,7 +1352,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { expect(decode(result.buffer)).toBe("AUDIO-WEBM-BYTES"); expect(result.extension).toBe("webm"); - expect(result.basename).toBe("audio-webm"); expect(requestUrlMock).not.toHaveBeenCalled(); }); @@ -1380,7 +1374,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { expect(decode(result.buffer)).toBe("AUDIO-MP4-BYTES"); expect(result.extension).toBe("m4a"); - expect(result.basename).toBe("audio-mp4"); expect(requestUrlMock).not.toHaveBeenCalled(); }); @@ -1402,7 +1395,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { expect(decode(result.buffer)).toBe("LEGACY-AUDIO-MP4-BYTES"); expect(result.extension).toBe("m4a"); - expect(result.basename).toBe("legacy-audio-mp4"); expect(requestUrlMock).not.toHaveBeenCalled(); }); @@ -1439,7 +1431,6 @@ describe("getEpisodeAudioBuffer (issue #107)", () => { expect(decode(result.buffer)).toBe("LOCAL-AUDIO"); expect(result.extension).toBe("m4a"); - expect(result.basename).toBe("recording"); expect(requestUrlMock).not.toHaveBeenCalled(); }); diff --git a/src/downloadEpisode.ts b/src/downloadEpisode.ts index eb396dbb..5a2f59fa 100644 --- a/src/downloadEpisode.ts +++ b/src/downloadEpisode.ts @@ -586,7 +586,7 @@ function normalizeAudioExtension( */ export async function getEpisodeAudioBuffer( episode: Episode, -): Promise<{ buffer: ArrayBuffer; extension: string; basename: string }> { +): Promise<{ buffer: ArrayBuffer; extension: string }> { const { app } = get(plugin); const episodeMediaType = getEpisodeMediaType(episode); if (episodeMediaType !== "audio") { @@ -640,7 +640,6 @@ export async function getEpisodeAudioBuffer( return { buffer: data, extension: normalizeAudioExtension(inferredExtension, audioContainerHint) ?? "mp3", - basename: replaceIllegalFileNameCharactersInString(episode.title) || "episode", }; } catch (error: unknown) { throw new Error(`Failed to fetch ${episode.title}: ${getErrorMessage(error)}`); @@ -655,7 +654,7 @@ function getAudioContainerHint(episode: Episode): EpisodeMediaType | undefined { async function readVaultAudio( filePath: string, -): Promise<{ buffer: ArrayBuffer; extension: string; basename: string }> { +): Promise<{ buffer: ArrayBuffer; extension: string }> { const { app } = get(plugin); const file = app.vault.getAbstractFileByPath(filePath); if (!(file instanceof TFile)) { @@ -674,15 +673,8 @@ async function readVaultAudio( return { buffer, extension: fileExtension.toLowerCase() === "mp4" ? "m4a" : fileExtension, - basename: file.basename || getFileBasename(file.path || filePath), }; } -function getFileBasename(filePath: string): string { - const fileName = filePath.split("/").pop() ?? filePath; - const dot = fileName.lastIndexOf("."); - return dot > 0 ? fileName.slice(0, dot) : fileName; -} - // Binary signature detection lives in ./download/mediaSignatures // (detectAudioFileExtension), imported above. diff --git a/src/services/TranscriptionService.published.test.ts b/src/services/TranscriptionService.published.test.ts index 13c44696..893d9326 100644 --- a/src/services/TranscriptionService.published.test.ts +++ b/src/services/TranscriptionService.published.test.ts @@ -1,20 +1,18 @@ // @vitest-environment jsdom import { afterEach, beforeEach, describe, expect, type MockInstance, test, vi } from "vitest"; import { Notice } from "obsidian"; -import type PodNotes from "src/main"; import type { TranscriptLink } from "src/parser/feedDocumentParser"; import { NetworkError } from "src/utility/networkRequest"; import { privateFeedPlaceholder } from "src/utility/privateFeedUrl"; import { deferred } from "../../tests/mocks/feeds"; import { createMockPlugin, - diarizeWithDeepgramMock, getEpisodeAudioBufferMock, installTranscriptionMocks, mockEpisode, noticeMessages, } from "../../tests/mocks/transcriptionService"; -import { openAIMock } from "../../tests/mocks/openaiRequestUrl"; +import { providerMock } from "../../tests/mocks/providerRequestUrl"; import { TranscriptionService } from "./TranscriptionService"; vi.mock("obsidian", async (importOriginal) => { @@ -34,12 +32,6 @@ vi.mock("../downloadEpisode", async () => ({ .getEpisodeAudioBufferMock, })); -vi.mock("./diarization", async (importOriginal) => ({ - ...(await importOriginal()), - diarizeWithDeepgram: (await import("../../tests/mocks/transcriptionService")) - .diarizeWithDeepgramMock, -})); - installTranscriptionMocks(); beforeEach(() => { @@ -79,20 +71,7 @@ describe("published transcripts", () => { .waitFor(() => expect(timerMessages()).toContain(message)) .finally(() => service.dispose()); - const withDeepgram = (plugin: PodNotes) => { - plugin.settings.transcript.diarization = { - enabled: true, - provider: "deepgram", - speakerTemplate: "", - }; - vi.mocked(plugin.credentials.has).mockImplementation( - (_settings, kind) => kind === "deepgram", - ); - vi.mocked(plugin.credentials.get).mockImplementation((_settings, kind) => - kind === "deepgram" ? "deepgram-key" : null, - ); - return plugin; - }; + const deepgram = { openAIKey: "", deepgramKey: "deepgram-key", speakers: "deepgram" } as const; test("with no API key, the feed's VTT becomes the note with no audio and no provider", async () => { fetchTextMock.mockResolvedValue(VTT); @@ -116,13 +95,12 @@ describe("published transcripts", () => { "Preparing to transcribe...", ]); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); - expect(openAIMock).not.toHaveBeenCalled(); - expect(diarizeWithDeepgramMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); }); test("with a key and Deepgram diarization, the feed's transcript still wins", async () => { fetchTextMock.mockResolvedValue(VTT); - const plugin = withDeepgram(createMockPlugin({ transcripts: published })); + const plugin = createMockPlugin({ ...deepgram, transcripts: published }); const service = new TranscriptionService(plugin); await service.transcribeCurrentEpisode(); @@ -130,8 +108,7 @@ describe("published transcripts", () => { expect(vi.mocked(plugin.app.vault.create).mock.calls[0][1]).toBe(publishedNote); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); - expect(openAIMock).not.toHaveBeenCalled(); - expect(diarizeWithDeepgramMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); }); test("a transcript quoting its own credential-bearing address and its private feed's keeps neither in the note", async () => { @@ -166,7 +143,7 @@ describe("published transcripts", () => { expect(noticeMessages).toEqual(["No published transcript was found for this episode."]); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); - expect(openAIMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); }); test("links of unknown types keep today's key notice", async () => { @@ -197,13 +174,13 @@ describe("published transcripts", () => { expect(plugin.app.vault.create).not.toHaveBeenCalled(); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); - expect(openAIMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); }); test("the offer names Deepgram when diarization uses it", async () => { fetchTextMock.mockResolvedValue("WEBVTT\n"); const service = new TranscriptionService( - withDeepgram(createMockPlugin({ transcripts: published })), + createMockPlugin({ ...deepgram, transcripts: published }), ); vi.spyOn(console, "error").mockImplementation(() => {}); @@ -213,7 +190,7 @@ describe("published transcripts", () => { `The published transcript could not be read.${fromAudioOffer.replace("OpenAI", "Deepgram")}`, ); - expect(diarizeWithDeepgramMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); }); @@ -235,7 +212,7 @@ describe("published transcripts", () => { }); test("transcribing from audio never looks for a published transcript", async () => { - openAIMock.mockImplementation(async () => Response.json({ text: "Spoken words." })); + providerMock.mockImplementation(async () => Response.json({ text: "Spoken words." })); const plugin = createMockPlugin({ transcripts: published }); const service = new TranscriptionService(plugin); diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index edc4a494..53ddb9f4 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1,21 +1,23 @@ // @vitest-environment jsdom import { describe, expect, test, vi } from "vitest"; import { Notice, type RequestUrlParam } from "obsidian"; -import { deferred } from "../../tests/mocks/feeds"; -import { OPENAI_FAILURES, OPENAI_REJECTIONS, rateLimited } from "../../tests/mocks/openaiFailures"; +import { deferred, feedDocument, MemoryStorage } from "../../tests/mocks/feeds"; import { createMockPlugin, - diarizeWithDeepgramMock, getEpisodeAudioBufferMock, installTranscriptionMocks, mockEpisode, noticeMessages, } from "../../tests/mocks/transcriptionService"; import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../tests/mocks/mp3"; -import { openAIMock, sentForm } from "../../tests/mocks/openaiRequestUrl"; +import { providerMock, sentBytes, sentForm, sentText } from "../../tests/mocks/providerRequestUrl"; +import { FeedRepository } from "./FeedRepository"; import { TranscriptionService } from "./TranscriptionService"; import type PodNotes from "src/main"; import type { Episode } from "src/types/Episode"; +import type { LocalEpisode } from "src/types/LocalEpisode"; +import type { PodcastFeed } from "src/types/PodcastFeed"; +import { privateFeedPlaceholder } from "src/utility/privateFeedUrl"; vi.mock("obsidian", async (importOriginal) => { const { mockedObsidian } = await import("../../tests/mocks/transcriptionService"); @@ -42,23 +44,83 @@ vi.mock("../downloadEpisode", async () => ({ .getEpisodeAudioBufferMock, })); -vi.mock("./diarization", async (importOriginal) => ({ - ...(await importOriginal()), - diarizeWithDeepgram: (await import("../../tests/mocks/transcriptionService")) - .diarizeWithDeepgramMock, -})); - installTranscriptionMocks(); type AudioJob = { episode: Episode; source: { kind: "audio" } }; const audioJob: AudioJob = { episode: mockEpisode, source: { kind: "audio" } }; -const whisperReturns = (text: string) => - openAIMock.mockImplementation(async () => Response.json({ text })); - -// Vitest's jsdom Request names every multipart file "blob", so these tests tell chunks apart by size. +/** Runs one audio job to its end, as the queue does. */ +const transcribe = (plugin: PodNotes, episode = mockEpisode) => + ( + new TranscriptionService(plugin) as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + } + ).transcribeEpisode({ episode, source: { kind: "audio" } }); + +/** The last message of the job's progress notice, after every timer has run. */ +const finalNotice = async (plugin: PodNotes, episode = mockEpisode): Promise => { + vi.useFakeTimers(); + const setMessage = vi.spyOn(Notice.prototype, "setMessage"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const pending = transcribe(plugin, episode); + await vi.runAllTimersAsync(); + await pending; + const calls = setMessage.mock.calls; + const message = String(calls[calls.length - 1][0]); + return message.slice(message.indexOf("\n\n") + 2); + } finally { + setMessage.mockRestore(); + consoleError.mockRestore(); + vi.useRealTimers(); + } +}; + +const savedNote = (plugin: PodNotes) => + String(vi.mocked(plugin.app.vault.create).mock.calls[0]?.[1]); + +const openAIReturns = (text: string) => + providerMock.mockImplementation(async () => Response.json({ text })); + +// Vitest's jsdom Request names every multipart file "blob", so these tests tell parts apart by size. const sentFile = async (request: RequestUrlParam) => (await sentForm(request)).get("file") as File; +const openAIError = (status: number, error: Record) => async () => + Response.json({ error }, { status }); + +const rateLimited = (headers: Record) => + Response.json( + { error: { message: "Rate limit reached for requests.", code: "rate_limit_exceeded" } }, + { status: 429, headers }, + ); + +/** + * A feed repository with `feed` saved under its title. Loading it gives a document that + * lists `episodes`, fetched from the address `resolveUrl` gives for the feed. + */ +function savedFeedRepository( + feed: PodcastFeed, + episodes: readonly Episode[], + resolveUrl: (feed: Pick) => string | null = ({ url }) => url, +): FeedRepository { + const feeds = new FeedRepository({ + load: async () => feedDocument(feed.title, episodes), + resolveUrl, + storage: new MemoryStorage(), + now: Date.now, + }); + feeds.setSubscriptions({ [feed.title]: feed }); + return feeds; +} + +/** An hour of MP3 in 3.2 MB, which OpenAI takes in two parts of 26 min 48 s. */ +const twoPartStream = MP3_STREAMS.mpeg25At8Kbps; +const twoPartFrames = 44_668; +const twoPartAudio = () => ({ + buffer: mp3Bytes(twoPartStream, twoPartFrames).buffer, + extension: "mp3", +}); + describe("TranscriptionService", () => { describe("transcribeCurrentEpisode validation", () => { test("shows notice when no API key is configured", async () => { @@ -72,6 +134,16 @@ describe("TranscriptionService", () => { ]); }); + test("asks for the Deepgram key when speaker labels use Deepgram", async () => { + const service = new TranscriptionService(createMockPlugin({ speakers: "deepgram" })); + + await service.transcribeCurrentEpisode(); + + expect(noticeMessages).toEqual([ + "Select or create a Deepgram API key in the transcript settings on this device.", + ]); + }); + test("refuses an episode whose note is already at the transcript path", async () => { const path = "Transcripts/Test Podcast/Test Episode.md"; const service = new TranscriptionService( @@ -104,20 +176,13 @@ describe("TranscriptionService", () => { service.dispose(); (service as unknown as { drainQueue: () => void }).drainQueue(); - expect(() => (service as unknown as { getApiKey: () => string }).getApiKey()).toThrow( - "unloaded", - ); expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); expect(plugin.credentials.get).not.toHaveBeenCalled(); }); test("settles promptly when unloaded during audio acquisition", async () => { - const audioRequest = deferred<{ - buffer: ArrayBuffer; - extension: string; - basename: string; - }>(); + const audioRequest = deferred<{ buffer: ArrayBuffer; extension: string }>(); getEpisodeAudioBufferMock.mockReturnValue(audioRequest.promise); const plugin = createMockPlugin(); const service = new TranscriptionService(plugin); @@ -140,106 +205,76 @@ describe("TranscriptionService", () => { consoleError.mockRestore(); }); - test("aborts active OpenAI work without writing a note or updating notices", async () => { - const plugin = createMockPlugin(); - const service = new TranscriptionService(plugin); - let finishRequest!: (response: Response) => void; - openAIMock.mockImplementation( - () => - new Promise((resolve) => { - finishRequest = resolve; - }), - ); - const setMessage = vi.spyOn(Notice.prototype, "setMessage"); - try { - const pending = ( - service as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; - } - ).transcribeEpisode(audioJob); - await vi.waitFor(() => expect(openAIMock).toHaveBeenCalledOnce()); - - const messagesBeforeDispose = setMessage.mock.calls.length; - service.dispose(); - finishRequest(Response.json({ text: "This must not be saved." })); - await pending; - - expect(plugin.app.vault.create).not.toHaveBeenCalled(); - expect(setMessage).toHaveBeenCalledTimes(messagesBeforeDispose); - } finally { - setMessage.mockRestore(); - } - }); - - test("does not save Deepgram results that finish after unload", async () => { - const plugin = createMockPlugin(); - plugin.settings.transcript.diarization = { - enabled: true, - provider: "deepgram", - speakerTemplate: "**{{speaker}}:** {{text}}", - }; - vi.mocked(plugin.credentials.get).mockImplementation((_settings, kind) => - kind === "deepgram" ? "deepgram-key" : "test-api-key", - ); - let finishRequest!: (segments: Array<{ speaker: string; text: string }>) => void; - diarizeWithDeepgramMock.mockImplementation( - () => - new Promise((resolve) => { - finishRequest = resolve; - }), - ); - const service = new TranscriptionService(plugin); + test("sends nothing for audio that arrives after unload", async () => { + const audioRequest = deferred<{ buffer: ArrayBuffer; extension: string }>(); + getEpisodeAudioBufferMock.mockReturnValue(audioRequest.promise); + const service = new TranscriptionService(createMockPlugin()); const pending = ( service as unknown as { transcribeEpisode: (job: AudioJob) => Promise; } ).transcribeEpisode(audioJob); - await vi.waitFor(() => expect(diarizeWithDeepgramMock).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(getEpisodeAudioBufferMock).toHaveBeenCalledOnce()); service.dispose(); - finishRequest([{ speaker: "A", text: "This must not be saved." }]); await pending; + audioRequest.resolve({ buffer: new ArrayBuffer(1024), extension: "mp3" }); + await new Promise((resolve) => setTimeout(resolve, 0)); - expect(plugin.app.vault.create).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); }); - test("settles promptly when unloaded during a non-cancelable Deepgram request", async () => { - const plugin = createMockPlugin(); - plugin.settings.transcript.diarization = { - enabled: true, - provider: "deepgram", - speakerTemplate: "**{{speaker}}:** {{text}}", - }; - vi.mocked(plugin.credentials.get).mockImplementation((_settings, kind) => - kind === "deepgram" ? "deepgram-key" : "test-api-key", - ); - const request = deferred>(); - diarizeWithDeepgramMock.mockReturnValue(request.promise); - const service = new TranscriptionService(plugin); - const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); - const pending = ( - service as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; + test.each([ + ["OpenAI", {}, { text: "This must not be saved." }], + [ + "Deepgram", + { speakers: "deepgram", deepgramKey: "deepgram-key" } as const, + deepgramAnswer([{ word: "Unsaved.", start: 0, end: 1, speaker: 0 }]), + ], + ])( + "aborts active %s work without writing a note or updating notices", + async (_provider, overrides, answer) => { + const plugin = createMockPlugin(overrides); + const service = new TranscriptionService(plugin); + let finishRequest!: (response: Response) => void; + providerMock.mockImplementation( + () => + new Promise((resolve) => { + finishRequest = resolve; + }), + ); + const setMessage = vi.spyOn(Notice.prototype, "setMessage"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const pending = ( + service as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + } + ).transcribeEpisode(audioJob); + await vi.waitFor(() => expect(providerMock).toHaveBeenCalledOnce()); + + const messagesBeforeDispose = setMessage.mock.calls.length; + service.dispose(); + const settledPromptly = await settlesAfterMicrotasks(pending); + finishRequest(Response.json(answer)); + await pending; + + expect(settledPromptly).toBe(true); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); + expect(setMessage).toHaveBeenCalledTimes(messagesBeforeDispose); + expect(consoleError).not.toHaveBeenCalled(); + } finally { + setMessage.mockRestore(); + consoleError.mockRestore(); } - ).transcribeEpisode(audioJob); - await vi.waitFor(() => expect(diarizeWithDeepgramMock).toHaveBeenCalledOnce()); - - service.dispose(); - const settledPromptly = await settlesAfterMicrotasks(pending); - request.reject(new Error("late Deepgram failure")); - await pending; - - expect(settledPromptly).toBe(true); - expect(plugin.app.vault.create).not.toHaveBeenCalled(); - expect(consoleError).not.toHaveBeenCalled(); - consoleError.mockRestore(); - }); + }, + ); test("cancels a completed notice hide timer when disposed", async () => { vi.useFakeTimers(); const hide = vi.spyOn(Notice.prototype, "hide"); try { - whisperReturns("Completed transcript."); + openAIReturns("Completed transcript."); const plugin = createMockPlugin(); const service = new TranscriptionService(plugin); await ( @@ -261,298 +296,290 @@ describe("TranscriptionService", () => { }); }); - describe("empty Whisper transcript (TR-01)", () => { - // transcribeEpisode is private; drive it directly so the empty-body throw is - // observed as "no file written" (the throw is caught and surfaced as a - // "Transcription failed" notice, never as a saved transcript). - const runTranscribeEpisode = async (plugin: PodNotes) => { - const service = new TranscriptionService(plugin); - await ( - service as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; - } - ).transcribeEpisode(audioJob); - return service; + describe("the progress notice", () => { + const hiddenAfter = async (plugin: PodNotes, ms: number): Promise => { + vi.useFakeTimers(); + const hide = vi.spyOn(Notice.prototype, "hide"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + await transcribe(plugin); + await vi.advanceTimersByTimeAsync(ms); + return hide.mock.calls.length > 0; + } finally { + hide.mockRestore(); + consoleError.mockRestore(); + vi.useRealTimers(); + } }; - test("does not write a file when the transcript is empty", async () => { - whisperReturns(""); - const plugin = createMockPlugin(); - - await runTranscribeEpisode(plugin); + test("hides 5 s after a transcript is saved", async () => { + openAIReturns("Completed transcript."); - expect(plugin.app.vault.create).not.toHaveBeenCalled(); + expect(await hiddenAfter(createMockPlugin(), 5000)).toBe(true); }); - test("does not write a file when the transcript is only whitespace", async () => { - // Multiple empty chunks join to " ", not "", so the trimmed-emptiness check - // must still treat this as failure. - whisperReturns(" \n\t "); - const plugin = createMockPlugin(); - - await runTranscribeEpisode(plugin); + test("stays until clicked after a failure", async () => { + providerMock.mockImplementation(openAIError(401, { message: "Incorrect API key." })); - expect(plugin.app.vault.create).not.toHaveBeenCalled(); + expect(await hiddenAfter(createMockPlugin(), 60 * 60_000)).toBe(false); }); + }); - test("writes a file when the transcript has content", async () => { - whisperReturns("Hello world. This is a transcript."); - const plugin = createMockPlugin(); + describe("an OpenAI transcript", () => { + test("sends OpenAI the model, the episode file and the JSON format", async () => { + openAIReturns("Hello."); - await runTranscribeEpisode(plugin); + await transcribe(createMockPlugin()); - expect(plugin.app.vault.create).toHaveBeenCalledTimes(1); + expect(providerMock).toHaveBeenCalledOnce(); + const [request] = providerMock.mock.calls[0]; + expect(request.url).toBe("https://api.openai.com/v1/audio/transcriptions"); + expect(request.method).toBe("POST"); + expect(request.headers).toEqual({ Authorization: "Bearer test-api-key" }); + const form = await sentForm(request); + expect([...form.keys()]).toEqual(["model", "file", "response_format"]); + expect(form.get("model")).toBe("gpt-transcribe"); + expect(form.get("response_format")).toBe("json"); + const file = form.get("file") as File; + expect(file.type).toBe("audio/mp3"); + expect(file.size).toBe(1024); }); - }); - describe("buildTranscriptBody (TR-01)", () => { - const buildBody = ( - plugin: PodNotes, - audio: { - buffer: ArrayBuffer; - mimeType: string; - extension: string; - basename: string; - } = { - buffer: new ArrayBuffer(1024), - mimeType: "audio/mpeg", - extension: "mp3", - basename: "episode", - }, - ) => { - const service = new TranscriptionService(plugin); - return ( - service as unknown as { - buildTranscriptBody: ( - audio: { - buffer: ArrayBuffer; - mimeType: string; - extension: string; - basename: string; - }, - update: (message: string) => void, - ) => Promise<{ body: string; warning?: string }>; - } - ).buildTranscriptBody(audio, () => {}); - }; + test("is saved as paragraphs without timestamps", async () => { + const sentence = "This sentence is forty characters long. "; + openAIReturns(sentence.repeat(20)); + const plugin = createMockPlugin(); - test("throws when the trimmed Whisper body is empty", async () => { - whisperReturns(" \n "); + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); - await expect(buildBody(createMockPlugin())).rejects.toThrow( - "Transcription returned no text.", + const paragraph = sentence.repeat(13).trim(); + expect(plugin.app.vault.create).toHaveBeenCalledExactlyOnceWith( + "Transcripts/Test Podcast/Test Episode.md", + `# Test Episode\n\n${paragraph}\n\n${sentence.repeat(7).trim()}`, ); }); - test("returns the reflowed body when there is text", async () => { - whisperReturns("One. Two."); - - await expect(buildBody(createMockPlugin())).resolves.toEqual({ - body: "One.\n\nTwo.", - warning: undefined, - }); - }); - }); + test.each(["", " \n\t "])("writes no note for the text %j", async (text) => { + openAIReturns(text); + const plugin = createMockPlugin(); - describe("failed chunks are not saved as a completed transcript (other-silent-failure)", () => { - const buildBodyDirect = ( - plugin: PodNotes, - audio: { - buffer: ArrayBuffer; - mimeType: string; - extension: string; - basename: string; - }, - ) => { - const service = new TranscriptionService(plugin); - return ( - service as unknown as { - buildTranscriptBody: ( - a: typeof audio, - update: (message: string) => void, - ) => Promise<{ body: string; warning?: string }>; - } - ).buildTranscriptBody(audio, () => {}); - }; + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription failed: OpenAI found no speech in this episode.", + ); - const mp3Audio = (byteLength: number) => ({ - buffer: new ArrayBuffer(byteLength), - mimeType: "audio/mp3", - extension: "mp3", - basename: "episode", + expect(plugin.app.vault.create).not.toHaveBeenCalled(); }); - test("throws (no file) when the single chunk fails every retry", async () => { - openAIMock.mockImplementation(async () => new Response("boom", { status: 500 })); + test("with speaker labels on is saved without them, and the notice names the providers that label speakers and stays", async () => { vi.useFakeTimers(); + const hide = vi.spyOn(Notice.prototype, "hide"); + const setMessage = vi.spyOn(Notice.prototype, "setMessage"); try { - const promise = buildBodyDirect(createMockPlugin(), mp3Audio(1024)); - const assertion = expect(promise).rejects.toThrow( - "all 1 audio chunk(s) failed or returned no text. Last error: 500 boom", + openAIReturns("Hello there. Hi!"); + const plugin = createMockPlugin({ speakers: "openai" }); + + await transcribe(plugin); + await vi.advanceTimersByTimeAsync(60 * 60_000); + + expect(String(setMessage.mock.calls.at(-1)?.[0])).toMatch( + /\n\nSaved without speaker labels, because OpenAI does not label speakers\. These providers do: Deepgram\.$/, ); - await vi.runAllTimersAsync(); - await assertion; + expect(hide).not.toHaveBeenCalled(); + expect(savedNote(plugin)).toBe("# Test Episode\n\nHello there. Hi!"); + const form = await sentForm(providerMock.mock.calls[0][0]); + expect([...form.keys()]).toEqual(["model", "file", "response_format"]); + expect(form.get("model")).toBe("gpt-transcribe"); } finally { + hide.mockRestore(); + setMessage.mockRestore(); vi.useRealTimers(); } }); - test("throws when failed chunks plus empty successes leave no real text", async () => { - // >20 MB mp3 → two chunks. chunk 0 fails every retry; chunk 1 "succeeds" - // but returns empty text. The body is then only an error marker, which - // must NOT be saved as a completed transcript. - openAIMock.mockImplementation(async (request) => - (await sentFile(request)).size > 1024 - ? new Response("boom", { status: 500 }) - : Response.json({ text: " " }), + test("reads the current OpenAI key for each transcription", async () => { + openAIReturns("Hello."); + const plugin = createMockPlugin({ openAIKey: "first-key" }); + const service = new TranscriptionService(plugin) as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + }; + + await service.transcribeEpisode(audioJob); + vi.mocked(plugin.credentials.get).mockImplementation((_settings, kind) => + kind === "openai" ? "second-key" : null, ); + await service.transcribeEpisode(audioJob); - vi.useFakeTimers(); - try { - const promise = buildBodyDirect( - createMockPlugin(), - mp3Audio(20 * 1024 * 1024 + 1024), - ); - const assertion = expect(promise).rejects.toThrow( - "all 2 audio chunk(s) failed or returned no text. Last error: 500 boom", - ); - await vi.runAllTimersAsync(); - await assertion; - } finally { - vi.useRealTimers(); - } + const authorization = (call: number) => + providerMock.mock.calls[call][0].headers?.Authorization; + expect(providerMock).toHaveBeenCalledTimes(2); + expect(authorization(0)).toBe("Bearer first-key"); + expect(authorization(1)).toBe("Bearer second-key"); }); + }); - test("does not write a file when transcription fails completely", async () => { - openAIMock.mockImplementation(async () => new Response("boom", { status: 500 })); + describe("an episode over one request's limits", () => { + test("goes to OpenAI in parts no longer than 2,070 s, one at a time, and reads in order", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + const sentSeconds: number[] = []; + let inFlight = 0; + let mostInFlight = 0; + providerMock.mockImplementation(async (request) => { + mostInFlight = Math.max(mostInFlight, ++inFlight); + const seconds = mp3Seconds(twoPartStream, (await sentFile(request)).size); + sentSeconds.push(seconds); + await new Promise((resolve) => setTimeout(resolve, 0)); + inFlight--; + return Response.json({ text: `Part ${sentSeconds.length}.` }); + }); const plugin = createMockPlugin(); - const service = new TranscriptionService(plugin); - vi.useFakeTimers(); - try { - const promise = ( - service as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; - } - ).transcribeEpisode(audioJob); - // One chunk, MAX_RETRIES=3 → backoff 1000ms + 2000ms before it gives up. - await vi.advanceTimersByTimeAsync(3500); - await promise; - } finally { - vi.useRealTimers(); - } + await transcribe(plugin); - expect(plugin.app.vault.create).not.toHaveBeenCalled(); + expect(sentSeconds).toHaveLength(2); + expect(Math.max(...sentSeconds)).toBeLessThanOrEqual(2070); + expect(sentSeconds[0] + sentSeconds[1]).toBeCloseTo( + mp3Seconds(twoPartStream, twoPartFrames * twoPartStream.frameBytes), + ); + expect(mostInFlight).toBe(1); + expect(savedNote(plugin)).toBe("# Test Episode\n\nPart 1. Part 2."); }); - test("keeps an otherwise-good transcript but warns when only some chunks fail", async () => { - // A >20 MB mp3 byte-splits into two chunks; fail the second one. - openAIMock.mockImplementation(async (request) => - (await sentFile(request)).size === 1024 - ? new Response("boom", { status: 500 }) - : Response.json({ text: "Good chunk." }), + test("keeps the parts that worked, marks the one that failed, and says so", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + let sends = 0; + providerMock.mockImplementation(async () => + ++sends === 1 + ? Response.json({ text: "Good part." }) + : new Response("boom", { status: 500 }), ); + const plugin = createMockPlugin(); - vi.useFakeTimers(); - try { - const promise = buildBodyDirect( - createMockPlugin(), - mp3Audio(20 * 1024 * 1024 + 1024), - ); - await vi.runAllTimersAsync(); - const result = await promise; + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription saved, but 1 of 2 chunk(s) failed - look for [Error transcribing chunk N] markers and re-run after deleting the note to retry.", + ); - expect(result.body).toContain("Good chunk."); - expect(result.body).toContain("[Error transcribing chunk 1]"); - expect(result.warning).toContain("1 of 2 chunk(s) failed"); - } finally { - vi.useRealTimers(); - } + expect(sends).toBe(4); + expect(savedNote(plugin)).toBe( + "# Test Episode\n\nGood part. \\[Error transcribing chunk 2]", + ); }); - }); - - describe("OpenAI requests and failures", () => { - const transcribe = (plugin: PodNotes) => - ( - new TranscriptionService(plugin) as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; - } - ).transcribeEpisode(audioJob); - const finalNotice = async (plugin: PodNotes): Promise => { - vi.useFakeTimers(); - const setMessage = vi.spyOn(Notice.prototype, "setMessage"); - const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); - try { - const pending = transcribe(plugin); - await vi.runAllTimersAsync(); - await pending; - const calls = setMessage.mock.calls; - const message = String(calls[calls.length - 1][0]); - return message.slice(message.indexOf("\n\n") + 2); - } finally { - setMessage.mockRestore(); - consoleError.mockRestore(); - vi.useRealTimers(); - } - }; - - test("sends Whisper only the model and the episode file", async () => { - whisperReturns("Hello."); + test("writes no note when one part fails and the other holds no speech", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + let sends = 0; + providerMock.mockImplementation(async () => + ++sends <= 3 ? new Response("boom", { status: 500 }) : Response.json({ text: " " }), + ); + const plugin = createMockPlugin(); - await transcribe(createMockPlugin()); + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription failed: OpenAI failed (HTTP 500). Try again later.", + ); - expect(openAIMock).toHaveBeenCalledOnce(); - const form = await sentForm(openAIMock.mock.calls[0][0]); - expect([...form.keys()]).toEqual(["model", "file"]); - expect(form.get("model")).toBe("whisper-1"); - const file = form.get("file") as File; - expect(file.type).toBe("audio/mp3"); - expect(file.size).toBe(1024); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); }); - test.each(OPENAI_FAILURES)( - "Whisper %s shows the API error after three sends", - async (_name, respond, message) => { - openAIMock.mockImplementation(respond); + test.each([401, 402, 404, 400])( + "stops at a %i on the first part and sends no second part", + async (status) => { + getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); + providerMock.mockImplementation(openAIError(status, { message: "No." })); + const plugin = createMockPlugin(); - await expect(finalNotice(createMockPlugin())).resolves.toBe( - `Transcription failed: all 1 audio chunk(s) failed or returned no text. Last error: ${message}`, - ); - expect(openAIMock).toHaveBeenCalledTimes(3); + await finalNotice(plugin); + + expect(providerMock).toHaveBeenCalledOnce(); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); }, ); + }); - test.each(OPENAI_REJECTIONS)( - "Whisper %s shows the API error after one send", - async (_name, respond, message) => { - openAIMock.mockImplementation(respond); + describe("a failed request", () => { + const neverAnswers = () => new Promise(() => {}); + const quotaUsedUp = openAIError(429, { + message: "You exceeded your current quota, please check your plan and billing details.", + type: "insufficient_quota", + code: "insufficient_quota", + }); - await expect(finalNotice(createMockPlugin())).resolves.toBe( - `Transcription failed: all 1 audio chunk(s) failed or returned no text. Last error: ${message}`, - ); - expect(openAIMock).toHaveBeenCalledOnce(); - }, - ); + test.each<[name: string, sends: number, respond: () => Promise, shown: string]>([ + [ + "401", + 1, + openAIError(401, { message: "Incorrect API key provided: sk-test." }), + "OpenAI rejected the API key. Check it in the transcript settings.", + ], + [ + "402", + 1, + openAIError(402, { message: "Payment required." }), + "Your OpenAI account is out of credit. Add credit there, then try again.", + ], + [ + "404", + 1, + openAIError(404, { message: "The model `gpt-transcribe` does not exist." }), + "OpenAI does not offer the gpt-transcribe model to this API key.", + ], + [ + "400", + 1, + openAIError(400, { message: "Audio file might be corrupted or unsupported." }), + "OpenAI could not transcribe this audio.", + ], + [ + "429 for a used-up quota", + 1, + quotaUsedUp, + "Your OpenAI account is out of credit. Add credit there, then try again.", + ], + [ + "429", + 3, + async () => rateLimited({}), + "OpenAI is limiting how fast your key may be used. Try again in a few minutes.", + ], + [ + "503 with an empty body", + 3, + async () => new Response("", { status: 503 }), + "OpenAI failed (HTTP 503). Try again later.", + ], + [ + "200 that is not JSON", + 3, + async () => new Response("Sign in"), + "OpenAI failed (HTTP 200). Try again later.", + ], + [ + "lost connection", + 3, + () => Promise.reject(new TypeError("Failed to fetch")), + "Could not reach OpenAI. Check your connection.", + ], + ["timeout", 1, neverAnswers, "OpenAI did not answer within 10 minutes."], + ])("%s saves no note after %i of 3 sends", async (_name, sends, respond, shown) => { + providerMock.mockImplementation(respond); + const plugin = createMockPlugin(); - test("Whisper answering without text fails the run after three sends", async () => { - openAIMock.mockImplementation(async () => Response.json({})); + await expect(finalNotice(plugin)).resolves.toBe(`Transcription failed: ${shown}`); - await expect(finalNotice(createMockPlugin())).resolves.toBe( - "Transcription failed: all 1 audio chunk(s) failed or returned no text. Last error: OpenAI returned a transcription without text.", - ); - expect(openAIMock).toHaveBeenCalledTimes(3); + expect(providerMock).toHaveBeenCalledTimes(sends); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); }); - test("Whisper waits out the server's Retry-After and recovers", async () => { + test.each<[name: string, failure: () => Response, waitMs: number]>([ + ["429 with Retry-After", () => rateLimited({ "retry-after": "4" }), 4000], + ["429 with retry-after-ms", () => rateLimited({ "retry-after-ms": "1500" }), 1500], + ["429 asking for over a minute", () => rateLimited({ "retry-after": "61" }), 1000], + ["503", () => new Response("", { status: 503 }), 1000], + ])("%s is sent again after the wait and the note is saved", async (_name, fail, waitMs) => { vi.useFakeTimers(); try { const sentAt: number[] = []; - openAIMock.mockImplementation(async () => { + providerMock.mockImplementation(async () => { sentAt.push(Date.now()); - return Date.now() - sentAt[0] < 4000 - ? rateLimited({ "retry-after": "4" }) - : Response.json({ text: "Recovered." }); + return sentAt.length === 1 ? fail() : Response.json({ text: "Recovered." }); }); const plugin = createMockPlugin(); @@ -560,19 +587,18 @@ describe("TranscriptionService", () => { await vi.runAllTimersAsync(); await pending; - expect(sentAt.map((time) => time - sentAt[0])).toEqual([0, 4000]); - expect(plugin.app.vault.create).toHaveBeenCalledOnce(); - expect(vi.mocked(plugin.app.vault.create).mock.calls[0][1]).toContain("Recovered."); + expect(sentAt.map((time) => time - sentAt[0])).toEqual([0, waitMs]); + expect(savedNote(plugin)).toBe("# Test Episode\n\nRecovered."); } finally { vi.useRealTimers(); } }); - test("unloading while Whisper waits out a Retry-After cancels the run", async () => { + test("unloading during a Retry-After wait cancels the run", async () => { vi.useFakeTimers(); const setMessage = vi.spyOn(Notice.prototype, "setMessage"); try { - openAIMock.mockImplementation(async () => rateLimited({ "retry-after": "30" })); + providerMock.mockImplementation(async () => rateLimited({ "retry-after": "30" })); const plugin = createMockPlugin(); const service = new TranscriptionService(plugin); let settled = false; @@ -586,7 +612,7 @@ describe("TranscriptionService", () => { settled = true; }); await vi.advanceTimersByTimeAsync(29_000); - expect(openAIMock).toHaveBeenCalledOnce(); + expect(providerMock).toHaveBeenCalledOnce(); const messagesBeforeDispose = setMessage.mock.calls.length; service.dispose(); @@ -594,7 +620,8 @@ describe("TranscriptionService", () => { expect(settled).toBe(true); await pending; - expect(openAIMock).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(60_000); + expect(providerMock).toHaveBeenCalledOnce(); expect(plugin.app.vault.create).not.toHaveBeenCalled(); expect(setMessage).toHaveBeenCalledTimes(messagesBeforeDispose); } finally { @@ -603,99 +630,258 @@ describe("TranscriptionService", () => { } }); - test("reads the current OpenAI key for each transcription", async () => { - whisperReturns("Hello."); - const plugin = createMockPlugin({ openAIKey: "first-key" }); - const service = new TranscriptionService(plugin) as unknown as { - transcribeEpisode: (job: AudioJob) => Promise; - }; - - await service.transcribeEpisode(audioJob); - vi.mocked(plugin.credentials.get).mockImplementation((_settings, kind) => - kind === "openai" ? "second-key" : null, - ); - await service.transcribeEpisode(audioJob); + test("shows and logs none of the provider's own words, which can quote a private address", async () => { + vi.useFakeTimers(); + const setMessage = vi.spyOn(Notice.prototype, "setMessage"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + providerMock.mockImplementation(async () => + Response.json( + { + err_code: "Bad Request", + err_msg: "Could not read https://private.example.com/feed/SENTINEL.mp3", + }, + { status: 400 }, + ), + ); + const plugin = createMockPlugin({ + speakers: "deepgram", + deepgramKey: "deepgram-key", + }); - const authorization = (call: number) => - openAIMock.mock.calls[call][0].headers?.Authorization; - expect(openAIMock).toHaveBeenCalledTimes(2); - expect(authorization(0)).toBe("Bearer first-key"); - expect(authorization(1)).toBe("Bearer second-key"); - }); + const pending = transcribe(plugin); + await vi.runAllTimersAsync(); + await pending; - test.each(OPENAI_FAILURES)( - "OpenAI diarization %s shows the API error after three sends", - async (_name, respond, message) => { - openAIMock.mockImplementation(respond); - const plugin = createMockPlugin(); - plugin.settings.transcript.diarization = { - enabled: true, - provider: "openai", - speakerTemplate: "**{{speaker}}:** {{text}}", - }; - - await expect(finalNotice(plugin)).resolves.toBe( - `Transcription failed: OpenAI diarization failed for every chunk: ${message}`, + const shown = setMessage.mock.calls.map(([message]) => String(message)); + expect(shown.at(-1)).toMatch( + /\n\nTranscription failed: Deepgram could not transcribe this audio\.$/, ); - expect(openAIMock).toHaveBeenCalledTimes(3); - }, - ); - - test.each(OPENAI_REJECTIONS)( - "OpenAI diarization %s shows the API error after one send", - async (_name, respond, message) => { - openAIMock.mockImplementation(respond); - const plugin = createMockPlugin(); - plugin.settings.transcript.diarization = { - enabled: true, - provider: "openai", - speakerTemplate: "**{{speaker}}:** {{text}}", - }; - - await expect(finalNotice(plugin)).resolves.toBe( - `Transcription failed: OpenAI diarization failed for every chunk: ${message}`, + expect(consoleError).toHaveBeenCalled(); + const logged = consoleError.mock.calls.flatMap((args) => + args.map((arg) => (arg instanceof Error ? `${arg.stack}` : String(arg))), ); - expect(openAIMock).toHaveBeenCalledOnce(); - }, - ); + for (const text of [...shown, ...noticeMessages, ...logged]) { + expect(text).not.toContain("SENTINEL"); + } + } finally { + setMessage.mockRestore(); + consoleError.mockRestore(); + vi.useRealTimers(); + } + }); + }); - test("OpenAI diarization parts stay under the model's 1400 s limit and answer inside the 10 min request timeout (#532)", async () => { - // Measured on gpt-4o-transcribe-diarize: the slowest of 81 requests took 0.58 s per second of audio. - const slowestSecondsPerAudioSecond = 0.58; - const stream = MP3_STREAMS.mpeg25At8Kbps; + describe("a Deepgram transcript", () => { + const publicFeed: PodcastFeed = { + title: "Test Podcast", + url: "https://example.com/feed.xml", + artworkUrl: "", + }; + const audio = new Uint8Array(2048).map((_, index) => (index * 31) % 251); + const useDeepgram = (feeds?: ReturnType) => { getEpisodeAudioBufferMock.mockResolvedValue({ - buffer: mp3Bytes(stream, 44_667).buffer, + buffer: audio.buffer, extension: "mp3", - basename: "episode", }); - const sentSeconds: number[] = []; - openAIMock.mockImplementation(async (request: RequestUrlParam) => { - const seconds = mp3Seconds(stream, (await sentFile(request)).size); - sentSeconds.push(seconds); - return seconds > 1400 + return createMockPlugin({ speakers: "deepgram", deepgramKey: "deepgram-key", feeds }); + }; + const answers = deepgramAnswer([ + { word: "hello", punctuated_word: "Hello", start: 1.2, end: 1.5, speaker: 1 }, + { word: "there", punctuated_word: "there.", start: 1.5, end: 1.9, speaker: 1 }, + { word: "hi", punctuated_word: "Hi!", start: 10.16, end: 10.5, speaker: 0 }, + ]); + const link = (time: string) => + `obsidian://podnotes?episodeName=Test%20Episode&url=https%3A%2F%2Fexample.com%2Ffeed.xml&time=${time}`; + const LISTEN = + "https://api.deepgram.com/v1/listen?model=nova-3&diarize=true&punctuate=true&smart_format=true&utterances=true"; + + /** The one request a job sent, which must carry the audio and nothing of `secrets`. */ + const expectUpload = (...secrets: string[]) => { + expect(providerMock).toHaveBeenCalledOnce(); + const [request] = providerMock.mock.calls[0]; + expect(request.contentType).toBe("audio/mp3"); + expect(sentBytes(request)).toEqual(audio); + for (const secret of secrets) expect(sentText(request)).not.toContain(secret); + }; + + test("has a timestamp on each paragraph and numbers the speakers in the order they first speak", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const plugin = useDeepgram(); + + await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); + + expect(savedNote(plugin)).toBe( + `# Test Episode\n\n[00:00:01](${link("1.2")}) **1:** Hello there.\n\n` + + `[00:00:10](${link("10.16")}) **2:** Hi!`, + ); + }); + + test("a public episode goes by its address, and its audio is never loaded", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const plugin = useDeepgram(savedFeedRepository(publicFeed, [mockEpisode])); + + await transcribe(plugin); + + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + expect(providerMock).toHaveBeenCalledOnce(); + const [request] = providerMock.mock.calls[0]; + expect(request.url).toBe(LISTEN); + expect(request.method).toBe("POST"); + expect(request.headers).toEqual({ Authorization: "Token deepgram-key" }); + expect(request.contentType).toBe("application/json"); + expect(JSON.parse(new TextDecoder().decode(sentBytes(request)))).toEqual({ + url: "https://example.com/episode.mp3", + }); + expect(savedNote(plugin)).toContain("**1:** Hello there."); + }); + + test("an episode with no saved feed is uploaded as the released request", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const plugin = useDeepgram(); + + await transcribe(plugin); + + expectUpload(); + const [request] = providerMock.mock.calls[0]; + expect(request.url).toBe(LISTEN); + expect(request.headers).toEqual({ Authorization: "Token deepgram-key" }); + }); + + test("a private feed's episode is uploaded, and no request holds its address or its feed's", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const secretFeed = "https://private.example.com/feed/SUBSCRIBER-TOKEN.rss"; + const privateFeed: PodcastFeed = { + ...publicFeed, + url: privateFeedPlaceholder("Test Podcast"), + urlSecretId: "podnotes-feed-url-1", + }; + const episode: Episode = { + ...mockEpisode, + streamUrl: "https://cdn.example.com/SUBSCRIBER-TOKEN/episode.mp3", + feedUrl: privateFeed.url, + }; + const plugin = useDeepgram( + savedFeedRepository(privateFeed, [episode], () => secretFeed), + ); + + await transcribe(plugin, episode); + + expectUpload("SUBSCRIBER-TOKEN", "cdn.example.com", "private.example.com"); + }); + + test("an episode whose address carries a credential is uploaded, and no request holds the address", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const episode: Episode = { + ...mockEpisode, + streamUrl: "https://cdn.example.com/episode.mp3?token=SIGNED-TOKEN", + }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [episode])); + + await transcribe(plugin, episode); + + expectUpload("SIGNED-TOKEN", "cdn.example.com"); + }); + + test("an episode on a private network is uploaded, and no request holds its address", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const episode: Episode = { + ...mockEpisode, + streamUrl: "http://192.168.1.20/library/episode.mp3", + }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [episode])); + + await transcribe(plugin, episode); + + expectUpload("192.168.1.20", "library"); + }); + + test("an episode its saved feed does not list is uploaded, and no request holds its address", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const unlisted: Episode = { + ...mockEpisode, + streamUrl: "https://elsewhere.example.com/episode.mp3", + }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [mockEpisode])); + + await transcribe(plugin, unlisted); + + expectUpload("elsewhere.example.com"); + }); + + test("a local file is uploaded, and no request holds its path", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const local: LocalEpisode = { + title: "Interview", + streamUrl: "app://f3a9/home/me/vault/Recordings/Interview.mp3?1700000000000", + url: "[[Recordings/Interview.mp3]]", + description: "", + content: "", + podcastName: "local file", + filePath: "Recordings/Interview.mp3", + }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [mockEpisode])); + + await transcribe(plugin, local); + + expectUpload("Recordings", "Interview", "app://", "vault"); + }); + + test("an address Deepgram could not fetch is followed by one upload", async () => { + providerMock.mockImplementation(async (request) => + request.contentType === "application/json" ? Response.json( { - error: { - message: `audio duration ${seconds} seconds is longer than 1400 seconds which is the maximum for this model`, - }, + err_code: "REMOTE_CONTENT_ERROR", + err_msg: + "The remote server hosting the media returned a client error: 403 Forbidden.", }, { status: 400 }, ) - : Response.json({ - segments: [{ speaker: "A", text: `${seconds} s.`, start: 0, end: 1 }], - }); - }); - const plugin = createMockPlugin(); - plugin.settings.transcript.diarization = { - enabled: true, - provider: "openai", - speakerTemplate: "**{{speaker}}:** {{text}}", - }; + : Response.json(answers), + ); + const plugin = useDeepgram(savedFeedRepository(publicFeed, [mockEpisode])); await expect(finalNotice(plugin)).resolves.toBe("Transcription completed and saved."); - expect(Math.max(...sentSeconds)).toBeLessThanOrEqual(1400); - expect(Math.max(...sentSeconds) * slowestSecondsPerAudioSecond).toBeLessThan(10 * 60); - expect(sentSeconds.reduce((total, seconds) => total + seconds)).toBeCloseTo(3216.024); + + expect(providerMock.mock.calls.map(([request]) => request.contentType)).toEqual([ + "application/json", + "audio/mp3", + ]); + expect(sentBytes(providerMock.mock.calls[1][0])).toEqual(audio); + expect(getEpisodeAudioBufferMock).toHaveBeenCalledOnce(); + expect(savedNote(plugin)).toContain("**1:** Hello there."); + }); + + test.each([ + [401, "Deepgram rejected the API key. Check it in the transcript settings."], + [400, "Deepgram could not transcribe this audio."], + ])("a %i for a request by address is not followed by an upload", async (status, shown) => { + providerMock.mockImplementation(async () => + Response.json({ err_code: "INVALID_AUTH", err_msg: "No." }, { status }), + ); + const plugin = useDeepgram(savedFeedRepository(publicFeed, [mockEpisode])); + + await expect(finalNotice(plugin)).resolves.toBe(`Transcription failed: ${shown}`); + + expect(providerMock).toHaveBeenCalledOnce(); + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + expect(plugin.app.vault.create).not.toHaveBeenCalled(); + }); + + test("an answer without words writes no note", async () => { + providerMock.mockImplementation(async () => Response.json(deepgramAnswer([]))); + const plugin = useDeepgram(); + + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription failed: Deepgram found no speech in this episode.", + ); + + expect(plugin.app.vault.create).not.toHaveBeenCalled(); }); }); }); + +function deepgramAnswer(words: Record[]) { + return { results: { channels: [{ alternatives: [{ words }] }] } }; +} diff --git a/src/services/TranscriptionService.ts b/src/services/TranscriptionService.ts index 7157a9d8..edffcedb 100644 --- a/src/services/TranscriptionService.ts +++ b/src/services/TranscriptionService.ts @@ -1,6 +1,5 @@ import { Notice, TFile, type Vault } from "obsidian"; import type PodNotes from "../main"; -import { getEpisodeAudioBuffer } from "../downloadEpisode"; import { FilePathTemplateEngine, TranscriptTemplateEngine } from "../TemplateEngine"; import { enforceMaxPathLength, lastSegmentExtension } from "../utility/enforceMaxPathLength"; import { ensureFolderExists } from "../utility/ensureFolderExists"; @@ -9,26 +8,16 @@ import type { Episode } from "src/types/Episode"; import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; import { openNote } from "src/ui/openNote"; import { TimerNotice } from "src/ui/TimerNotice"; -import { createChunkFiles, getMimeType } from "./audioChunker"; -import { canRetry, createTranscription, retryDelayMs } from "./openaiTranscription"; -import { - type DiarizationAudio, - type DiarizationProviderId, - type DiarizedSegment, - diarizeWithDeepgram, - diarizeWithOpenAI, - OPENAI_DIARIZE_PART_SECONDS, - renderDiarizedTranscript, - requiredTranscriptionKeyPresent, -} from "./diarization"; import { fetchPublishedTranscript, findPublishedTranscript, + type ParsedTranscript, type PublishedTranscript, PublishedTranscriptError, renderTranscriptBody, timestampStyle, } from "./publishedTranscript"; +import { transcribe, transcriptionChoice } from "./transcription"; export const TRANSCRIBE_FROM_AUDIO_COMMAND = "Transcribe current episode from audio"; @@ -52,17 +41,7 @@ export function transcriptNote( return vault.getFileByPath(transcriptPath(settings, episode)); } -// A chunk that exhausts its retries leaves this placeholder in its transcript -// slot; the pattern matches it so buildTranscriptBody can tell real speech apart -// from a body made only of error markers. Keep the builder and matcher in sync. -function chunkErrorPlaceholder(index: number): string { - return `[Error transcribing chunk ${index}]`; -} -const CHUNK_ERROR_PLACEHOLDER_PATTERN = /\[Error transcribing chunk \d+\]/g; - -const MAX_RETRIES = 3; const MAX_CONCURRENT_TRANSCRIPTIONS = 2; -const MAX_CONCURRENT_CHUNK_TRANSCRIPTIONS = 3; function queueKey(episode: Episode): string { return `${episode.podcastName}:${episode.title}`; @@ -75,6 +54,15 @@ interface TranscriptionJob { | { readonly kind: "audio" }; } +interface Outcome { + /** Fills the template's `{{transcript}}` tag. */ + readonly body: string; + /** What the notice says once the note is saved. */ + readonly completion: string; + /** The note is not all that was asked for, and `completion` says why. */ + readonly warns: boolean; +} + export class TranscriptionService { private plugin: PodNotes; private disposed = false; @@ -133,27 +121,17 @@ export class TranscriptionService { } private hasRequiredKey(): boolean { - return requiredTranscriptionKeyPresent(this.plugin.settings, (kind) => - this.plugin.credentials.has(this.plugin.settings, kind), - ); - } - - private needsDeepgram(): boolean { - const diarization = this.plugin.settings.transcript.diarization; - return Boolean(diarization?.enabled && diarization.provider === "deepgram"); + const { settings, credentials } = this.plugin; + return credentials.has(settings, transcriptionChoice(settings).provider); } private noticeMissingKey(): void { - const needsDeepgram = this.needsDeepgram(); - const kind = needsDeepgram ? "deepgram" : "openai"; - const unavailableOnDevice = - this.plugin.credentials.status(this.plugin.settings, kind) === "missing"; + const { settings, credentials } = this.plugin; + const { provider, label } = transcriptionChoice(settings); new Notice( - unavailableOnDevice - ? `The selected ${needsDeepgram ? "Deepgram" : "OpenAI"} API key is not available on this device. Select or create it in the transcript settings.` - : needsDeepgram - ? "Select or create a Deepgram API key in the transcript settings on this device." - : "Select or create an OpenAI API key in the transcript settings on this device.", + credentials.status(settings, provider) === "missing" + ? `The selected ${label} API key is not available on this device. Select or create it in the transcript settings.` + : `Select or create ${/^[AEIOU]/.test(label) ? "an" : "a"} ${label} API key in the transcript settings on this device.`, ); } @@ -210,27 +188,31 @@ export class TranscriptionService { const updateNotice = (message: string) => { if (!this.disposed) notice.update(message); }; + // A failure, or a note saved with a warning, asks for an action, so it stays until clicked. + let lasting = true; try { const existing = transcriptNote(this.plugin.app.vault, this.plugin.settings, episode); if (existing) { notice.stop(); + lasting = false; updateNotice(`Transcript already exists - skipped (${existing.path}).`); return; } - const { body, completion } = + const outcome = source.kind === "published" ? await this.readPublishedTranscript(episode, source.transcript, updateNotice) : await this.transcribeAudio(episode, updateNotice); this.assertActive(); updateNotice("Saving transcription..."); - await this.saveTranscription(episode, body); + await this.saveTranscription(episode, outcome.body); this.assertActive(); notice.stop(); - updateNotice(completion); + lasting = outcome.warns; + updateNotice(outcome.completion); } catch (error) { if (this.disposed || this.signal.aborted) return; console.error("Transcription error:", error); @@ -245,7 +227,7 @@ export class TranscriptionService { if (this.disposed) { notice.dispose(); this.activeNotices.delete(notice); - } else { + } else if (!lasting) { notice.scheduleHide(5000, () => this.activeNotices.delete(notice)); } } @@ -255,247 +237,56 @@ export class TranscriptionService { episode: Episode, transcript: PublishedTranscript, updateNotice: (message: string) => void, - ): Promise<{ body: string; completion: string }> { + ): Promise { updateNotice("Fetching the published transcript..."); const parsed = await this.waitForLifecycle( fetchPublishedTranscript(this.plugin.feeds, episode, transcript, this.signal), ); - const { timestamp, transcript: settings } = this.plugin.settings; - const body = renderTranscriptBody(parsed, { - episode, - speakerTemplate: settings.diarization?.speakerTemplate ?? "", - timestamp: timestampStyle(timestamp.template), - }); - return { body, completion: "Saved the transcript the feed publishes." }; + return { + body: this.renderBody(episode, parsed), + completion: "Saved the transcript the feed publishes.", + warns: false, + }; } private async transcribeAudio( episode: Episode, updateNotice: (message: string) => void, - ): Promise<{ body: string; completion: string }> { - updateNotice("Fetching episode audio..."); - const { buffer, extension, basename } = await this.waitForLifecycle( - getEpisodeAudioBuffer(episode), - ); - this.assertActive(); - const { body, warning } = await this.buildTranscriptBody( - { buffer, mimeType: getMimeType(extension), extension, basename }, - updateNotice, + ): Promise { + const { settings, credentials, feeds } = this.plugin; + const choice = transcriptionChoice(settings); + const apiKey = credentials.get(settings, choice.provider); + if (!apiKey) throw new Error(`Missing ${choice.label} API key on this device.`); + const { content, warnings } = await this.waitForLifecycle( + transcribe({ + choice, + apiKey, + episode, + feeds, + signal: this.signal, + onProgress: updateNotice, + }), ); - return { body, completion: warning ?? "Transcription completed and saved." }; - } - - private publishedTranscriptFailure(error: PublishedTranscriptError): string { - if (!this.hasRequiredKey()) return error.message; - const provider = this.needsDeepgram() ? "Deepgram" : "OpenAI"; - return `${error.message} Run "${TRANSCRIBE_FROM_AUDIO_COMMAND}" to transcribe it with ${provider} instead, which uses your API credit.`; - } - - /** - * Produce the transcript body that fills the template's `{{transcript}}` tag. - * Plain Whisper returns one run-on block, so it is reflowed after sentence - * periods for readability; diarization returns speaker-labeled turns already - * separated into paragraphs, so it is rendered as-is. Either path yielding no - * speech is treated as a failure rather than writing an empty transcript (empty - * Whisper chunks join to whitespace, so the body is trimmed before the check). - * - * A chunk that exhausts its retries leaves an `[Error transcribing chunk N]` - * placeholder in its slot. If stripping those placeholders leaves no real text - - * every chunk failed, or the only successes were empty - we throw, so no file is - * written and the episode stays retryable instead of saving a "transcript" made - * only of error markers that the existence check would then refuse to re-run - * (mirrors the diarization provider's all-chunks-failed contract). When some - * chunks fail but real speech remains we keep the otherwise-good transcript and - * return a `warning` so the run is reported as partial, not a clean success. - */ - private async buildTranscriptBody( - audio: DiarizationAudio, - updateNotice: (message: string) => void, - ): Promise<{ body: string; warning?: string }> { - const diarization = this.plugin.settings.transcript.diarization; - - if (diarization?.enabled) { - const segments = await this.diarize(audio, diarization.provider, updateNotice); - this.assertActive(); - if (segments.length === 0) { - throw new Error("Diarization returned no speech segments."); - } - return { - body: renderDiarizedTranscript(segments, diarization.speakerTemplate), - }; - } - - updateNotice("Creating audio chunks..."); - const files = await createChunkFiles(audio); - this.assertActive(); - updateNotice("Starting transcription..."); - const { text, failedChunks, lastError } = await this.transcribeChunks(files, updateNotice); - this.assertActive(); - - // Strip the error placeholders (and trim) to see whether ANY real speech was - // transcribed. Nothing real means every chunk failed or the only successes - // were empty - either way there is no usable transcript, so throw instead of - // saving a body of pure error markers (empty chunks join to " ", not ""). - const realText = text.replace(CHUNK_ERROR_PLACEHOLDER_PATTERN, " ").trim(); - if (realText.length === 0) { - if (failedChunks === 0) throw new Error("Transcription returned no text."); - const detail = lastError instanceof Error ? lastError.message : String(lastError); - throw new Error( - `all ${files.length} audio chunk(s) failed or returned no text. Last error: ${detail}`, - ); - } - - // Reflow the full body (placeholders kept inline so the user can see which - // chunks failed) after sentence periods for readability. - const body = text.trim().replace(/\.\s+/g, ".\n\n"); - - const warning = - failedChunks > 0 - ? `Transcription saved, but ${failedChunks} of ${files.length} chunk(s) failed - look for [Error transcribing chunk N] markers and re-run after deleting the note to retry.` - : undefined; - return { body, warning }; + return { + body: this.renderBody(episode, content), + completion: warnings.join(" ") || "Transcription completed and saved.", + warns: warnings.length > 0, + }; } - /** Route the episode audio to the configured diarization provider (#168). */ - private async diarize( - audio: DiarizationAudio, - provider: DiarizationProviderId, - updateNotice: (message: string) => void, - ): Promise { - this.assertActive(); - if (provider === "deepgram") { - const apiKey = this.plugin.credentials.get(this.plugin.settings, "deepgram"); - if (!apiKey) { - throw new Error("Missing Deepgram API key on this device."); - } - // Deepgram ingests the whole file in one request, so it needs no - // chunking, which is exactly why its speaker labels stay consistent - // across the entire episode. - const segments = await this.waitForLifecycle( - diarizeWithDeepgram({ - audio, - apiKey, - onProgress: updateNotice, - signal: this.signal, - }), - ); - this.assertActive(); - return segments; - } - - // OpenAI diarization takes Whisper's ~20 MB chunks, capped at 600 s so each request - // answers inside the timeout. Speaker labels can differ across chunks. - updateNotice("Creating audio chunks..."); - const chunkFiles = await createChunkFiles({ - ...audio, - maxSeconds: OPENAI_DIARIZE_PART_SECONDS, - }); - this.assertActive(); - const segments = await diarizeWithOpenAI({ - apiKey: this.getApiKey(), - chunkFiles, - maxRetries: MAX_RETRIES, - onProgress: updateNotice, - signal: this.signal, + private renderBody(episode: Episode, transcript: ParsedTranscript): string { + const { timestamp, transcript: settings } = this.plugin.settings; + return renderTranscriptBody(transcript, { + episode, + speakerTemplate: settings.diarization.speakerTemplate, + timestamp: timestampStyle(timestamp.template), }); - this.assertActive(); - return segments; } - private async transcribeChunks( - files: File[], - updateNotice: (message: string) => void, - ): Promise<{ text: string; failedChunks: number; lastError: unknown }> { - const apiKey = this.getApiKey(); - const transcriptions: string[] = Array.from({ length: files.length }); - let completedChunks = 0; - let failedChunks = 0; - let lastError: unknown; - let nextIndex = 0; - - const updateProgress = () => { - const progress = ((completedChunks / files.length) * 100).toFixed(1); - updateNotice( - `Transcribing... ${completedChunks}/${files.length} chunks completed (${progress}%)`, - ); - }; - - updateProgress(); - - const worker = async () => { - while (true) { - this.assertActive(); - const index = nextIndex++; - if (index >= files.length) return; - const file = files[index]; - - let retries = 0; - while (retries < MAX_RETRIES) { - this.assertActive(); - try { - const result = await createTranscription( - apiKey, - { model: "whisper-1", file }, - this.signal, - ); - this.assertActive(); - const text = (result as { text?: unknown } | null)?.text; - if (typeof text !== "string") { - throw new Error("OpenAI returned a transcription without text."); - } - transcriptions[index] = text; - completedChunks++; - updateProgress(); - break; - } catch (error) { - this.assertActive(); - retries++; - if (retries >= MAX_RETRIES || !canRetry(error)) { - console.error( - `Failed to transcribe chunk ${index} after ${retries} attempt(s):`, - error, - ); - transcriptions[index] = chunkErrorPlaceholder(index); - failedChunks++; - lastError = error; - completedChunks++; - updateProgress(); - break; - } else { - await this.waitForRetry(retryDelayMs(error, 1000 * retries)); - } - } - } - } - }; - - const workerCount = Math.min(MAX_CONCURRENT_CHUNK_TRANSCRIPTIONS, files.length); - const workers = Array.from({ length: workerCount }, () => worker()); - - await Promise.all(workers); - this.assertActive(); - - return { text: transcriptions.join(" "), failedChunks, lastError }; - } - - private waitForRetry(delayMs: number): Promise { - this.assertActive(); - const { signal } = this; - - return new Promise((resolve, reject) => { - const timeout = window.setTimeout(() => { - signal.removeEventListener("abort", onAbort); - resolve(); - }, delayMs); - const onAbort = () => { - window.clearTimeout(timeout); - signal.removeEventListener("abort", onAbort); - reject(toError(this.getAbortReason(), "PodNotes transcription was aborted.")); - }; - - signal.addEventListener("abort", onAbort, { once: true }); - if (signal.aborted) onAbort(); - }); + private publishedTranscriptFailure(error: PublishedTranscriptError): string { + if (!this.hasRequiredKey()) return error.message; + const { label } = transcriptionChoice(this.plugin.settings); + return `${error.message} Run "${TRANSCRIBE_FROM_AUDIO_COMMAND}" to transcribe it with ${label} instead, which uses your API credit.`; } private waitForLifecycle(operation: Promise): Promise { @@ -549,15 +340,6 @@ export class TranscriptionService { } } - private getApiKey(): string { - this.assertActive(); - const apiKey = this.plugin.credentials.get(this.plugin.settings, "openai"); - if (!apiKey) { - throw new Error("Missing OpenAI API key on this device"); - } - return apiKey; - } - dispose(): void { if (this.disposed) return; this.disposed = true; diff --git a/src/services/audioChunker.test.ts b/src/services/audioChunker.test.ts deleted file mode 100644 index 4908acfe..00000000 --- a/src/services/audioChunker.test.ts +++ /dev/null @@ -1,354 +0,0 @@ -// @vitest-environment jsdom -import { afterEach, describe, expect, test, vi } from "vitest"; -import { - CHUNK_SIZE_BYTES, - createBinaryChunkFiles, - createChunkFiles, - getMimeType, - shouldConvertToWav, - writeWavHeader, -} from "./audioChunker"; -import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../tests/mocks/mp3"; - -describe("getMimeType", () => { - test("returns correct mime types for audio formats", () => { - expect(getMimeType("mp3")).toBe("audio/mp3"); - expect(getMimeType("MP3")).toBe("audio/mp3"); - expect(getMimeType("m4a")).toBe("audio/mp4"); - expect(getMimeType("ogg")).toBe("audio/ogg"); - expect(getMimeType("wav")).toBe("audio/wav"); - expect(getMimeType("flac")).toBe("audio/flac"); - expect(getMimeType("webm")).toBe("audio/webm"); - expect(getMimeType("unknown")).toBe("audio/mpeg"); - }); -}); - -describe("shouldConvertToWav", () => { - test("only frame-synced mp3 is byte-splittable; everything else needs WAV", () => { - // MP3 resyncs at the next frame header, so it can be sliced raw. - expect(shouldConvertToWav("mp3", "audio/mp3")).toBe(false); - expect(shouldConvertToWav("MP3", "audio/mp3")).toBe(false); - // An mp3 extension wins even if the mime is mislabeled. - expect(shouldConvertToWav("mp3", "audio/mp4")).toBe(false); - - // Container/header-dependent formats can't be byte-split standalone. - expect(shouldConvertToWav("m4a", "audio/mp4")).toBe(true); - expect(shouldConvertToWav("M4A", "audio/mp4")).toBe(true); - expect(shouldConvertToWav("ogg", "audio/ogg")).toBe(true); - expect(shouldConvertToWav("webm", "audio/webm")).toBe(true); - expect(shouldConvertToWav("flac", "audio/flac")).toBe(true); - expect(shouldConvertToWav("wav", "audio/wav")).toBe(true); - - // Unknown formats default to the safe WAV path; the catch-all audio/mpeg - // mime is deliberately not treated as splittable. - expect(shouldConvertToWav("opus", "audio/mpeg")).toBe(true); - expect(shouldConvertToWav("", "audio/mpeg")).toBe(true); - }); -}); - -describe("createBinaryChunkFiles", () => { - test("returns single file when buffer is smaller than chunk size", () => { - const smallBuffer = new ArrayBuffer(1024); - const files = createBinaryChunkFiles(smallBuffer, "test", "mp3", "audio/mpeg"); - - expect(files).toHaveLength(1); - expect(files[0].name).toBe("test.mp3"); - expect(files[0].type).toBe("audio/mpeg"); - expect(files[0].size).toBe(1024); - }); - - test("returns multiple files when buffer exceeds chunk size", () => { - const largeBuffer = new ArrayBuffer(CHUNK_SIZE_BYTES * 2 + 1024); - const files = createBinaryChunkFiles(largeBuffer, "test", "mp3", "audio/mpeg"); - - expect(files).toHaveLength(3); - expect(files[0].name).toBe("test.part0.mp3"); - expect(files[1].name).toBe("test.part1.mp3"); - expect(files[2].name).toBe("test.part2.mp3"); - }); -}); - -describe("writeWavHeader", () => { - const WAV_HEADER_SIZE = 44; - - test("writes correct RIFF header", () => { - const buffer = new ArrayBuffer(WAV_HEADER_SIZE); - const view = new DataView(buffer); - - writeWavHeader(view, 44100, 2, 44100); - - const riff = String.fromCharCode( - view.getUint8(0), - view.getUint8(1), - view.getUint8(2), - view.getUint8(3), - ); - expect(riff).toBe("RIFF"); - - const wave = String.fromCharCode( - view.getUint8(8), - view.getUint8(9), - view.getUint8(10), - view.getUint8(11), - ); - expect(wave).toBe("WAVE"); - - const fmt = String.fromCharCode( - view.getUint8(12), - view.getUint8(13), - view.getUint8(14), - view.getUint8(15), - ); - expect(fmt).toBe("fmt "); - - const data = String.fromCharCode( - view.getUint8(36), - view.getUint8(37), - view.getUint8(38), - view.getUint8(39), - ); - expect(data).toBe("data"); - }); - - test("writes correct sample rate and channels", () => { - const buffer = new ArrayBuffer(WAV_HEADER_SIZE); - const view = new DataView(buffer); - - writeWavHeader(view, 44100, 2, 1000); - - expect(view.getUint16(22, true)).toBe(2); - expect(view.getUint32(24, true)).toBe(44100); - expect(view.getUint16(34, true)).toBe(16); - }); -}); - -describe("createChunkFiles small-file fast path (#168 / PR #204 review)", () => { - test("sends a small m4a as a single original file instead of WAV-splitting it", async () => { - // A 1 KB m4a is well under the 20 MB chunk size, so it must not be - // converted to WAV (which would balloon it into many chunks and reset - // diarization speaker labels). It should be one intact .m4a file. - const files = await createChunkFiles({ - buffer: new ArrayBuffer(1024), - basename: "episode", - extension: "m4a", - mimeType: "audio/mp4", - }); - - expect(files).toHaveLength(1); - expect(files[0].name).toBe("episode.m4a"); - expect(files[0].type).toBe("audio/mp4"); - expect(files[0].name).not.toContain(".wav"); - }); - - test("still sends a small mp3 as a single original file (unchanged)", async () => { - const files = await createChunkFiles({ - buffer: new ArrayBuffer(2048), - basename: "episode", - extension: "mp3", - mimeType: "audio/mpeg", - }); - - expect(files).toHaveLength(1); - expect(files[0].name).toBe("episode.mp3"); - expect(files[0].size).toBe(2048); - }); - - test("sends a small ogg/flac as a single original file (no byte-split)", async () => { - for (const [extension, mimeType] of [ - ["ogg", "audio/ogg"], - ["flac", "audio/flac"], - ]) { - const files = await createChunkFiles({ - buffer: new ArrayBuffer(4096), - basename: "episode", - extension, - mimeType, - }); - - expect(files).toHaveLength(1); - expect(files[0].name).toBe(`episode.${extension}`); - } - }); -}); - -describe("createChunkFiles with maxSeconds (#532)", () => { - const chunk = async (bytes: Uint8Array, maxSeconds: number) => - Promise.all( - ( - await createChunkFiles({ - buffer: bytes.buffer as ArrayBuffer, - basename: "episode", - extension: "mp3", - mimeType: "audio/mp3", - maxSeconds, - }) - ).map(async (file) => ({ - name: file.name, - bytes: new Uint8Array(await file.arrayBuffer()), - })), - ); - const joinsInto = (parts: { bytes: Uint8Array }[], episode: Uint8Array) => - Buffer.concat(parts.map((part) => part.bytes)).equals(episode); - - test("splits an MP3 under the size limit into equal parts no longer than maxSeconds", async () => { - const stream = MP3_STREAMS.mpeg1At96Kbps; - const episode = mp3Bytes(stream, 134_000); - - const parts = await chunk(episode, 1400); - - expect(parts.map((part) => part.name)).toEqual([ - "episode.part0.mp3", - "episode.part1.mp3", - "episode.part2.mp3", - ]); - expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ - 44_667, 44_667, 44_666, - ]); - expect(mp3Seconds(stream, parts[0].bytes.length)).toBeLessThanOrEqual(1400); - for (const part of parts) - expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); - expect(joinsInto(parts, episode)).toBe(true); - }); - - test("sends an MP3 that fits both limits as the original file", async () => { - const episode = mp3Bytes(MP3_STREAMS.mpeg1At96Kbps, 58_000); - - const parts = await chunk(episode, 1400); - - expect(parts.map((part) => part.name)).toEqual(["episode.mp3"]); - expect(joinsInto(parts, episode)).toBe(true); - }); - - test("keeps parts under the size limit when it is the tighter one", async () => { - const stream = MP3_STREAMS.mpeg1At320Kbps; - const episode = mp3Bytes(stream, 50_000); - - const parts = await chunk(episode, 1400); - - expect(parts).toHaveLength(3); - for (const part of parts) { - expect(part.bytes.length).toBeLessThanOrEqual(CHUNK_SIZE_BYTES); - expect(Array.from(part.bytes.subarray(0, 4))).toEqual(stream.header); - } - expect(joinsInto(parts, episode)).toBe(true); - }); - - test("keeps a leading ID3 tag in the first part without counting sync-like bytes in it as audio", async () => { - const stream = MP3_STREAMS.mpeg25At8Kbps; - const tag = new Uint8Array(8192); - tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x3f, 0x76]); - for (let offset = 100; offset <= 5000; offset += 100) tag.set(stream.header, offset); - const frames = mp3Bytes(stream, 36_000); - const episode = Buffer.concat([tag, frames]); - - const parts = await chunk(episode, 1400); - - expect(parts.map((part) => part.bytes.length)).toEqual([ - tag.length + 18_000 * stream.frameBytes, - 18_000 * stream.frameBytes, - ]); - expect(Array.from(parts[0].bytes.subarray(0, 3))).toEqual([0x49, 0x44, 0x33]); - expect(joinsInto(parts, episode)).toBe(true); - }); -}); - -describe("createChunkFiles large-file routing (other-logic-bug)", () => { - // A minimal AudioBuffer/AudioContext so the decode+WAV path can run under - // jsdom, which ships no Web Audio implementation. decodeAudioData returns a - // buffer long enough to force several WAV chunks. - class FakeAudioBuffer { - numberOfChannels = 1; - sampleRate = 44100; - length: number; - private channel: Float32Array; - constructor(length: number) { - this.length = length; - this.channel = new Float32Array(length); - } - getChannelData(): Float32Array { - return this.channel; - } - } - - function stubAudioContext(decode: (buffer: ArrayBuffer) => unknown): void { - vi.stubGlobal( - "AudioContext", - class { - async decodeAudioData(buffer: ArrayBuffer): Promise { - return decode(buffer); - } - async close(): Promise {} - }, - ); - } - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - test("byte-splits a large mp3 into raw .mp3 parts (frame-synced, splittable)", async () => { - const buffer = new ArrayBuffer(CHUNK_SIZE_BYTES + 1024); - const files = await createChunkFiles({ - buffer, - basename: "episode", - extension: "mp3", - mimeType: "audio/mp3", - }); - - expect(files).toHaveLength(2); - expect(files[0].name).toBe("episode.part0.mp3"); - expect(files[1].name).toBe("episode.part1.mp3"); - for (const file of files) { - expect(file.type).toBe("audio/mp3"); - expect(file.name).not.toContain(".wav"); - } - }); - - test("decodes a large ogg to standalone WAV chunks instead of byte-splitting", async () => { - // Without the fix this returned raw .ogg byte slices that lack stream - // headers and can't be decoded standalone. Now it must produce .wav chunks. - stubAudioContext((buffer) => new FakeAudioBuffer(Math.ceil(buffer.byteLength / 2))); - - const files = await createChunkFiles({ - buffer: new ArrayBuffer(CHUNK_SIZE_BYTES + 1024), - basename: "episode", - extension: "ogg", - mimeType: "audio/ogg", - }); - - expect(files.length).toBeGreaterThan(1); - for (const file of files) { - expect(file.type).toBe("audio/wav"); - expect(file.name).toMatch(/^episode\.part\d+\.wav$/); - expect(file.name).not.toContain(".ogg"); - } - }); - - test("throws rather than byte-splitting a large flac when no decoder is available", async () => { - // jsdom has no AudioContext, so convertToWavChunks yields nothing. The old - // code fell through to a corrupt raw byte-split; now it must throw. - await expect( - createChunkFiles({ - buffer: new ArrayBuffer(CHUNK_SIZE_BYTES + 1024), - basename: "episode", - extension: "flac", - mimeType: "audio/flac", - }), - ).rejects.toThrow(/Could not split flac audio/); - }); - - test("throws when the decoder cannot decode a large webm chunk", async () => { - stubAudioContext(() => { - throw new Error("Unsupported codec"); - }); - - await expect( - createChunkFiles({ - buffer: new ArrayBuffer(CHUNK_SIZE_BYTES + 1024), - basename: "episode", - extension: "webm", - mimeType: "audio/webm", - }), - ).rejects.toThrow(/Could not split webm audio/); - }); -}); diff --git a/src/services/audioChunker.ts b/src/services/audioChunker.ts deleted file mode 100644 index 22f14d7e..00000000 --- a/src/services/audioChunker.ts +++ /dev/null @@ -1,348 +0,0 @@ -/** - * Pure audio chunking + WAV encoding for transcription uploads. Splits an episode - * audio buffer into request-sized files, decoding and re-encoding to WAV any - * format that can't be byte-split (everything except frame-synced MP3). Kept free - * of any TranscriptionService/plugin state so it can be unit-tested directly — the - * service just orchestrates it. - */ - -// OpenAI's per-request limit is 25 MB; we cap at 20 MB for a safety margin. -// Buffers at or under this are sent as a single file. -export const CHUNK_SIZE_BYTES = 20 * 1024 * 1024; -const WAV_HEADER_SIZE = 44; -const PCM_BYTES_PER_SAMPLE = 2; - -export function getMimeType(fileExtension: string): string { - switch (fileExtension.toLowerCase()) { - case "mp3": - return "audio/mp3"; - case "m4a": - return "audio/mp4"; - case "ogg": - return "audio/ogg"; - case "wav": - return "audio/wav"; - case "flac": - return "audio/flac"; - case "webm": - return "audio/webm"; - default: - return "audio/mpeg"; - } -} - -/** - * Whether a buffer too large for one request must be decoded and re-encoded to - * standalone WAV chunks instead of sliced at raw byte boundaries. - * - * Only MP3 tolerates raw byte-splitting: its frames are self-synchronizing, so a - * decoder resyncs at the next frame header even when a slice starts mid-stream. - * Every other podcast format is container/header-dependent - m4a/mp4 (ISO-BMFF), - * Ogg, WebM/Matroska (EBML) and FLAC all carry codec setup data only at the - * stream start, and WAV is raw PCM behind a leading 44-byte header - so a - * non-first arbitrary byte slice lacks the headers needed to decode standalone - * and the transcription API rejects or garbles it. Treat anything that isn't - * unambiguously MP3 as needing the WAV path (the safe default for unknown - * formats); `getMimeType` only emits `audio/mp3` for an mp3 extension, while the - * catch-all `audio/mpeg` is deliberately NOT treated as splittable. - */ -export function shouldConvertToWav(extension: string, mimeType: string): boolean { - const normalizedExtension = extension.toLowerCase(); - const isMp3 = normalizedExtension === "mp3" || mimeType.toLowerCase() === "audio/mp3"; - return !isMp3; -} - -/** - * `maxSeconds` also caps each chunk's audio length, for MP3 only. Other formats - * over 20 MB become WAV chunks of under four minutes, and smaller ones go up whole. - */ -export async function createChunkFiles({ - buffer, - basename, - extension, - mimeType, - maxSeconds, -}: { - buffer: ArrayBuffer; - basename: string; - extension: string; - mimeType: string; - maxSeconds?: number; -}): Promise { - if (maxSeconds !== undefined && !shouldConvertToWav(extension, mimeType)) { - const ends = mp3ChunkEnds(new Uint8Array(buffer), CHUNK_SIZE_BYTES, maxSeconds); - return sliceIntoFiles(buffer, ends, basename, extension, mimeType); - } - - // A file that already fits in a single request needs neither conversion nor - // splitting: send the original (compressed) bytes. OpenAI accepts m4a/mp3/etc. - // directly under the upload limit, so this skips the m4a->WAV path below for - // small m4a episodes — that path only exists to SAFELY SPLIT an m4a (which - // can't be byte-split) and would otherwise balloon a small m4a into many - // uncompressed WAV chunks, multiplying requests and, for diarization, - // resetting speaker labels at artificial chunk boundaries (#168 / PR #204 review). - if (buffer.byteLength <= CHUNK_SIZE_BYTES) { - return [new File([buffer], `${basename}.${extension}`, { type: mimeType })]; - } - - // Container/header-dependent formats must be decoded and re-encoded to WAV; - // only frame-synced MP3 falls through to raw byte-splitting. - if (shouldConvertToWav(extension, mimeType)) { - const wavChunks = await convertToWavChunks(buffer, basename); - if (wavChunks.length > 0) { - return wavChunks; - } - // WAV conversion produced nothing - the platform has no Web Audio decoder - // or couldn't decode this codec. Byte-splitting the raw container here would - // hand the API undecodable chunks (garbled or rejected output saved as a - // transcript), so fail loudly and keep the run retryable instead. - throw new Error( - `Could not split ${extension || "audio"} audio for transcription. This format must be decoded to WAV before splitting, but the audio decoder is unavailable or could not decode it.`, - ); - } - - return createBinaryChunkFiles(buffer, basename, extension, mimeType); -} - -export function createBinaryChunkFiles( - buffer: ArrayBuffer, - basename: string, - extension: string, - mimeType: string, -): File[] { - const ends: number[] = []; - for (let end = CHUNK_SIZE_BYTES; end < buffer.byteLength; end += CHUNK_SIZE_BYTES) { - ends.push(end); - } - ends.push(buffer.byteLength); - return sliceIntoFiles(buffer, ends, basename, extension, mimeType); -} - -/** One file per chunk, where `ends` holds each chunk's end offset in order. */ -function sliceIntoFiles( - buffer: ArrayBuffer, - ends: number[], - basename: string, - extension: string, - mimeType: string, -): File[] { - if (ends.length === 1) { - return [new File([buffer], `${basename}.${extension}`, { type: mimeType })]; - } - return ends.map( - (end, index) => - new File( - [buffer.slice(index === 0 ? 0 : ends[index - 1], end)], - `${basename}.part${index}.${extension}`, - { type: mimeType }, - ), - ); -} - -interface Mp3Frame { - offset: number; - length: number; - seconds: number; -} - -const MPEG1_LAYER3_KBPS = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320]; -const MPEG2_LAYER3_KBPS = [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160]; -/** Indexed by the header's version bits: 0 is MPEG-2.5, 2 is MPEG-2, 3 is MPEG-1. */ -const SAMPLE_RATES = [ - [11025, 12000, 8000], - undefined, - [22050, 24000, 16000], - [44100, 48000, 32000], -]; - -/** - * Where each chunk of an MP3 ends. Chunks end at frame starts, there are as few as - * both limits allow, and they hold an equal number of frames, so no short last - * chunk resets the speaker labels for a few seconds of speech. - */ -function mp3ChunkEnds(bytes: Uint8Array, maxBytes: number, maxSeconds: number): number[] { - const frames = mp3Frames(bytes); - const seconds = frames.reduce((total, frame) => total + frame.seconds, 0); - const chunks = Math.max(Math.ceil(bytes.length / maxBytes), Math.ceil(seconds / maxSeconds)); - const framesPerChunk = Math.ceil(frames.length / chunks); - const ends: number[] = []; - let start = 0; - let chunkFrames = 0; - let chunkSeconds = 0; - for (const frame of frames) { - if ( - chunkFrames === framesPerChunk || - frame.offset + frame.length - start > maxBytes || - chunkSeconds + frame.seconds > maxSeconds - ) { - ends.push(frame.offset); - start = frame.offset; - chunkFrames = 0; - chunkSeconds = 0; - } - chunkFrames++; - chunkSeconds += frame.seconds; - } - ends.push(bytes.length); - return ends; -} - -/** - * The Layer III frames in `bytes`. A frame counts only when another frame or the - * end of the buffer follows it, so sync-like bytes in a tag or cover art don't. - */ -function mp3Frames(bytes: Uint8Array): Mp3Frame[] { - const frames: Mp3Frame[] = []; - let offset = 0; - while (offset + 4 <= bytes.length) { - const frame = mp3FrameAt(bytes, offset); - const next = frame ? offset + frame.length : 0; - if (frame && (next === bytes.length || mp3FrameAt(bytes, next))) { - frames.push(frame); - offset = next; - } else { - offset++; - } - } - return frames; -} - -function mp3FrameAt(bytes: Uint8Array, offset: number): Mp3Frame | undefined { - if (offset + 4 > bytes.length || bytes[offset] !== 0xff) return undefined; - const versionAndLayer = bytes[offset + 1]; - const rates = bytes[offset + 2]; - const version = (versionAndLayer >> 3) & 3; - const isLayer3 = ((versionAndLayer >> 1) & 3) === 1; - if ((versionAndLayer & 0xe0) !== 0xe0 || !isLayer3) return undefined; - const kbps = (version === 3 ? MPEG1_LAYER3_KBPS : MPEG2_LAYER3_KBPS)[rates >> 4]; - const sampleRate = SAMPLE_RATES[version]?.[(rates >> 2) & 3]; - if (!kbps || !sampleRate) return undefined; - const samples = version === 3 ? 1152 : 576; - const padding = (rates >> 1) & 1; - return { - offset, - length: Math.floor(((samples / 8) * kbps * 1000) / sampleRate) + padding, - seconds: samples / sampleRate, - }; -} - -async function convertToWavChunks(buffer: ArrayBuffer, basename: string): Promise { - const audioContext = createAudioContext(); - if (!audioContext) return []; - - try { - const audioBuffer = await audioContext.decodeAudioData(buffer.slice(0)); - return renderWavChunks(audioBuffer, basename); - } catch (error) { - console.warn("Failed to convert audio buffer for transcription", error); - return []; - } finally { - try { - await audioContext.close(); - } catch (error) { - console.warn("Failed to close audio context", error); - } - } -} - -function createAudioContext(): AudioContext | null { - if (typeof window === "undefined") { - return null; - } - - const contextCtor = - window.AudioContext || - (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; - if (!contextCtor) { - return null; - } - - return new contextCtor(); -} - -function renderWavChunks(audioBuffer: AudioBuffer, basename: string): File[] { - const numChannels = audioBuffer.numberOfChannels; - const bytesPerFrame = numChannels * PCM_BYTES_PER_SAMPLE; - const availableBytesPerChunk = CHUNK_SIZE_BYTES - WAV_HEADER_SIZE; - const maxSamplesPerChunk = Math.max(1, Math.floor(availableBytesPerChunk / bytesPerFrame)); - const channelData = Array.from({ length: numChannels }, (_, channelIndex) => - audioBuffer.getChannelData(channelIndex), - ); - const files: File[] = []; - let chunkIndex = 0; - - for (let startSample = 0; startSample < audioBuffer.length; startSample += maxSamplesPerChunk) { - const endSample = Math.min(audioBuffer.length, startSample + maxSamplesPerChunk); - const wavBuffer = renderWavBuffer( - channelData, - audioBuffer.sampleRate, - startSample, - endSample, - ); - files.push( - new File([wavBuffer], `${basename}.part${chunkIndex}.wav`, { - type: "audio/wav", - }), - ); - chunkIndex++; - } - - return files; -} - -function renderWavBuffer( - channelData: Float32Array[], - sampleRate: number, - startSample: number, - endSample: number, -): ArrayBuffer { - const numChannels = channelData.length; - const sampleCount = Math.max(0, endSample - startSample); - const blockAlign = numChannels * PCM_BYTES_PER_SAMPLE; - const buffer = new ArrayBuffer(WAV_HEADER_SIZE + sampleCount * blockAlign); - const view = new DataView(buffer); - writeWavHeader(view, sampleRate, numChannels, sampleCount); - let offset = WAV_HEADER_SIZE; - - for (let i = 0; i < sampleCount; i++) { - for (let channel = 0; channel < numChannels; channel++) { - const sample = channelData[channel][startSample + i] ?? 0; - const clamped = Math.max(-1, Math.min(1, sample)); - const intSample = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; - view.setInt16(offset, Math.round(intSample), true); - offset += PCM_BYTES_PER_SAMPLE; - } - } - - return buffer; -} - -export function writeWavHeader( - view: DataView, - sampleRate: number, - numChannels: number, - sampleCount: number, -): void { - const blockAlign = numChannels * PCM_BYTES_PER_SAMPLE; - const byteRate = sampleRate * blockAlign; - const dataSize = sampleCount * blockAlign; - writeString(view, 0, "RIFF"); - view.setUint32(4, 36 + dataSize, true); - writeString(view, 8, "WAVE"); - writeString(view, 12, "fmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, numChannels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, byteRate, true); - view.setUint16(32, blockAlign, true); - view.setUint16(34, PCM_BYTES_PER_SAMPLE * 8, true); - writeString(view, 36, "data"); - view.setUint32(40, dataSize, true); -} - -function writeString(view: DataView, offset: number, str: string): void { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); - } -} diff --git a/src/services/diarization/deepgramProvider.test.ts b/src/services/diarization/deepgramProvider.test.ts deleted file mode 100644 index 5dfc07ee..00000000 --- a/src/services/diarization/deepgramProvider.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -// @vitest-environment jsdom -import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { RequestUrlResponse } from "obsidian"; - -const nativeRequest = vi.hoisted(() => vi.fn()); - -vi.mock("../../utility/networkRequest", async () => { - const actual = await vi.importActual( - "../../utility/networkRequest", - ); - const fetchJsonWithTimeout: typeof actual.fetchJsonWithTimeout = (url, options) => - actual.fetchJsonWithTimeout(url, { ...options, request: nativeRequest }); - - return { ...actual, fetchJsonWithTimeout: vi.fn(fetchJsonWithTimeout) }; -}); - -import { diarizeWithDeepgram } from "./deepgramProvider"; -import type { DiarizationAudio } from "./types"; -import { NetworkError, fetchJsonWithTimeout } from "../../utility/networkRequest"; - -const audio: DiarizationAudio = { - buffer: new ArrayBuffer(8), - mimeType: "audio/mpeg", - extension: "mp3", - basename: "episode", -}; - -function stubResponse(overrides: Partial<{ status: number; json: unknown; text: string }>) { - const json = overrides.json ?? {}; - return { - status: 200, - json, - text: overrides.text ?? JSON.stringify(json), - arrayBuffer: new ArrayBuffer(0), - headers: {}, - ...overrides, - } satisfies RequestUrlResponse; -} - -function deferred() { - let resolve!: (value: T | PromiseLike) => void; - const promise = new Promise((resolvePromise) => { - resolve = resolvePromise; - }); - return { promise, resolve }; -} - -async function settlesAfterMicrotasks(promise: Promise): Promise { - let settled = false; - void promise.then( - () => { - settled = true; - }, - () => { - settled = true; - }, - ); - - for (let turn = 0; turn < 10 && !settled; turn++) await Promise.resolve(); - return settled; -} - -beforeEach(() => { - nativeRequest.mockReset(); - vi.mocked(fetchJsonWithTimeout).mockClear(); -}); - -describe("diarizeWithDeepgram (#168)", () => { - it("posts the whole audio buffer with token auth and parses the response", async () => { - nativeRequest.mockResolvedValue( - stubResponse({ - json: { - results: { - utterances: [ - { speaker: 0, transcript: "Hello.", start: 0, end: 1 }, - { speaker: 1, transcript: "Hi.", start: 1, end: 2 }, - ], - }, - }, - }), - ); - - const segments = await diarizeWithDeepgram({ - audio, - apiKey: "dg-key", - onProgress: () => {}, - }); - - expect(segments).toEqual([ - { speaker: "1", text: "Hello.", start: 0, end: 1 }, - { speaker: "2", text: "Hi.", start: 1, end: 2 }, - ]); - - expect(fetchJsonWithTimeout).toHaveBeenCalledWith( - expect.stringContaining("api.deepgram.com/v1/listen"), - { - method: "POST", - headers: { Authorization: "Token dg-key" }, - contentType: "audio/mpeg", - body: audio.buffer, - timeoutMs: 30 * 60_000, - maxRequestBodyBytes: 2 * 1024 * 1024 * 1024, - maxResponseBytes: 16 * 1024 * 1024, - signal: undefined, - }, - ); - - const call = nativeRequest.mock.calls[0][0]; - expect(call.method).toBe("POST"); - expect(call.url).toContain("api.deepgram.com/v1/listen"); - expect(call.url).toContain("diarize=true"); - expect(call.headers.Authorization).toBe("Token dg-key"); - expect(call.contentType).toBe("audio/mpeg"); - expect(call.body).toBe(audio.buffer); - }); - - it("reports only a safe HTTP status on a non-2xx response", async () => { - nativeRequest.mockResolvedValue( - stubResponse({ status: 401, json: { err_msg: "Invalid credentials" } }), - ); - - const error = await diarizeWithDeepgram({ - audio, - apiKey: "bad", - onProgress: () => {}, - }).catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(NetworkError); - expect((error as NetworkError).status).toBe(401); - expect(String(error)).toContain("HTTP 401"); - expect(String(error)).not.toContain("Invalid credentials"); - }); - - it("redacts the API key and a native transport error", async () => { - const marker = "private-deepgram-marker"; - nativeRequest.mockRejectedValue(new Error(`native failure with ${marker}`)); - - const error = await diarizeWithDeepgram({ - audio, - apiKey: marker, - onProgress: () => {}, - }).catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(NetworkError); - expect((error as NetworkError).code).toBe("transport-failure"); - expect(String(error)).not.toContain(marker); - expect(String(error)).not.toContain("native failure"); - }); - - it("aborts promptly and ignores a late HTTP failure from the native transport", async () => { - const controller = new AbortController(); - const abortError = new DOMException("private lifecycle detail", "AbortError"); - const response = deferred(); - nativeRequest.mockReturnValue(response.promise); - const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); - const pending = diarizeWithDeepgram({ - audio, - apiKey: "bad", - onProgress: () => {}, - signal: controller.signal, - }); - const outcome = pending.then( - () => undefined, - (error: unknown) => error, - ); - await vi.waitFor(() => expect(nativeRequest).toHaveBeenCalledOnce()); - - try { - controller.abort(abortError); - const settledPromptly = await settlesAfterMicrotasks(outcome); - response.resolve(stubResponse({ status: 401, json: { err_msg: "late failure" } })); - const result = await outcome; - - expect(settledPromptly).toBe(true); - expect(result).toBeInstanceOf(NetworkError); - expect((result as NetworkError).code).toBe("aborted"); - expect(String(result)).not.toContain(abortError.message); - expect(consoleError).not.toHaveBeenCalled(); - } finally { - consoleError.mockRestore(); - } - }); -}); diff --git a/src/services/diarization/deepgramProvider.ts b/src/services/diarization/deepgramProvider.ts deleted file mode 100644 index a99741a8..00000000 --- a/src/services/diarization/deepgramProvider.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { DiarizationAudio, DiarizedSegment } from "./types"; -import { parseDeepgramSegments } from "./segments"; -import { fetchJsonWithTimeout } from "../../utility/networkRequest"; - -const DEEPGRAM_LISTEN_URL = "https://api.deepgram.com/v1/listen"; -const DEEPGRAM_REQUEST_TIMEOUT_MS = 30 * 60_000; -const MAX_DEEPGRAM_AUDIO_BYTES = 2 * 1024 * 1024 * 1024; -const MAX_DEEPGRAM_RESPONSE_BYTES = 16 * 1024 * 1024; - -/** - * Query params for the pre-recorded request. `diarize=true` is the broadly - * available diarizer; `smart_format`/`punctuate` make the text readable and - * `utterances=true` returns ready-made speaker turns the parser prefers. Sent as - * a single whole-file request, so speaker identity stays consistent across the - * entire episode (no cross-chunk drift). - */ -const DEEPGRAM_QUERY = "model=nova-3&diarize=true&punctuate=true&smart_format=true&utterances=true"; - -/** - * Diarize with Deepgram's pre-recorded speech-to-text API (issue #168). - * - * Posts the whole original (compressed) audio in one synchronous request via - * the shared bounded network boundary. No chunking is needed because Deepgram - * accepts large files server-side, which keeps speaker labels consistent for - * the full episode. - */ -export async function diarizeWithDeepgram(opts: { - audio: DiarizationAudio; - apiKey: string; - onProgress: (message: string) => void; - signal?: AbortSignal; -}): Promise { - const { audio, apiKey, onProgress, signal } = opts; - onProgress("Diarizing with Deepgram..."); - - const response = await fetchJsonWithTimeout( - `${DEEPGRAM_LISTEN_URL}?${DEEPGRAM_QUERY}`, - { - method: "POST", - headers: { Authorization: `Token ${apiKey}` }, - contentType: audio.mimeType, - body: audio.buffer, - timeoutMs: DEEPGRAM_REQUEST_TIMEOUT_MS, - maxRequestBodyBytes: MAX_DEEPGRAM_AUDIO_BYTES, - maxResponseBytes: MAX_DEEPGRAM_RESPONSE_BYTES, - signal, - }, - ); - - return parseDeepgramSegments(response); -} diff --git a/src/services/diarization/index.ts b/src/services/diarization/index.ts deleted file mode 100644 index cb29394e..00000000 --- a/src/services/diarization/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; -import type { CredentialKind } from "src/types/Credentials"; - -export * from "./types"; -export * from "./segments"; -export { diarizeWithOpenAI } from "./openaiProvider"; -export { diarizeWithDeepgram } from "./deepgramProvider"; - -/** - * Whether the credentials the active transcription mode needs are present. - * - * Diarization via Deepgram needs the dedicated Deepgram credential; every other - * mode (plain Whisper, or OpenAI diarization) reuses the OpenAI credential. - */ -export function requiredTranscriptionKeyPresent( - settings: IPodNotesSettings, - hasCredential: (kind: CredentialKind) => boolean, -): boolean { - const diarization = settings.transcript?.diarization; - if (diarization?.enabled && diarization.provider === "deepgram") { - return hasCredential("deepgram"); - } - return hasCredential("openai"); -} diff --git a/src/services/diarization/openaiProvider.test.ts b/src/services/diarization/openaiProvider.test.ts deleted file mode 100644 index 58c74ca8..00000000 --- a/src/services/diarization/openaiProvider.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { rateLimited } from "../../../tests/mocks/openaiFailures"; -import { corsBlockedFetch, openAIMock, sentForm } from "../../../tests/mocks/openaiRequestUrl"; -import { diarizeWithOpenAI } from "./openaiProvider"; - -vi.mock("obsidian", async (importOriginal) => ({ - ...(await importOriginal()), - requestUrl: (await import("../../../tests/mocks/openaiRequestUrl")).openAIRequestUrl, -})); - -function chunk(name: string): File { - return new File([new ArrayBuffer(8)], name, { type: "audio/mpeg" }); -} - -const diarized = (speaker: string, text: string) => async () => - Response.json({ - segments: [{ type: "transcript.text.segment", id: "1", speaker, start: 0, end: 1, text }], - }); - -const sentFormOf = (call: number) => sentForm(openAIMock.mock.calls[call][0]); - -beforeEach(() => { - vi.stubGlobal("fetch", corsBlockedFetch); - // In Obsidian's renderer `window` is the global object; the code under test uses its timers. - vi.stubGlobal("window", globalThis); -}); - -afterEach(() => { - vi.useRealTimers(); - vi.unstubAllGlobals(); - openAIMock.mockReset(); -}); - -describe("diarizeWithOpenAI (#168)", () => { - it("concatenates parsed segments across chunks in order", async () => { - openAIMock - .mockImplementationOnce(diarized("A", "First.")) - .mockImplementationOnce(diarized("B", "Second.")); - const chunks = [chunk("a.mp3"), chunk("b.mp3")]; - - const segments = await diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: chunks, - maxRetries: 2, - onProgress: () => {}, - signal: new AbortController().signal, - }); - - expect(segments).toEqual([ - { speaker: "A", text: "First.", start: 0, end: 1 }, - { speaker: "B", text: "Second.", start: 0, end: 1 }, - ]); - expect(openAIMock).toHaveBeenCalledTimes(2); - expect([...(await sentFormOf(0)).entries()]).toEqual([ - ["model", "gpt-4o-transcribe-diarize"], - ["file", expect.objectContaining({ name: "a.mp3" })], - ["response_format", "diarized_json"], - ["chunking_strategy", "auto"], - ]); - expect(((await sentFormOf(1)).get("file") as File).name).toBe("b.mp3"); - }); - - it("keeps a partial transcript with a marker when only some chunks fail", async () => { - openAIMock - .mockImplementationOnce(diarized("A", "Good chunk.")) - .mockImplementation(async () => new Response("Bad Gateway", { status: 502 })); - - const segments = await diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: [chunk("a.mp3"), chunk("b.mp3")], - maxRetries: 1, - onProgress: () => {}, - signal: new AbortController().signal, - }); - - expect(segments).toEqual([ - { speaker: "A", text: "Good chunk.", start: 0, end: 1 }, - { speaker: "?", text: "[Error diarizing chunk 2]" }, - ]); - }); - - it("throws (writes no transcript) when every chunk fails", async () => { - openAIMock.mockImplementation(async () => - Response.json( - { error: { message: "Incorrect API key provided: sk-test." } }, - { status: 401 }, - ), - ); - - await expect( - diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: [chunk("a.mp3"), chunk("b.mp3")], - maxRetries: 1, - onProgress: () => {}, - signal: new AbortController().signal, - }), - ).rejects.toThrow( - new Error( - "OpenAI diarization failed for every chunk: 401 Incorrect API key provided: sk-test.", - ), - ); - }); - - it("waits out the server's Retry-After and recovers the chunk", async () => { - vi.useFakeTimers(); - const sentAt: number[] = []; - openAIMock.mockImplementation(async () => { - sentAt.push(Date.now()); - return Date.now() - sentAt[0] < 4000 - ? rateLimited({ "retry-after": "4" }) - : diarized("A", "Recovered.")(); - }); - - const outcome = diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: [chunk("a.mp3")], - maxRetries: 3, - onProgress: () => {}, - signal: new AbortController().signal, - }).catch((error: unknown) => error); - await vi.runAllTimersAsync(); - - expect(sentAt.map((time) => time - sentAt[0])).toEqual([0, 4000]); - expect(await outcome).toEqual([{ speaker: "A", text: "Recovered.", start: 0, end: 1 }]); - }); - - it("aborts while waiting out the server's Retry-After", async () => { - vi.useFakeTimers(); - const controller = new AbortController(); - const abortReason = new Error("plugin unloaded"); - openAIMock.mockImplementation(async () => rateLimited({ "retry-after": "30" })); - - let outcome: unknown = "pending"; - void diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: [chunk("a.mp3")], - maxRetries: 3, - onProgress: () => {}, - signal: controller.signal, - }).then( - () => { - outcome = "resolved"; - }, - (error: unknown) => { - outcome = error; - }, - ); - await vi.advanceTimersByTimeAsync(29_000); - expect(openAIMock).toHaveBeenCalledOnce(); - - controller.abort(abortReason); - await vi.advanceTimersByTimeAsync(0); - - expect(outcome).toBe(abortReason); - expect(openAIMock).toHaveBeenCalledOnce(); - }); - - it("stops waiting for an in-flight request on abort without retrying or logging a failure", async () => { - const controller = new AbortController(); - const abortError = new DOMException("plugin unloaded", "AbortError"); - openAIMock.mockImplementation(() => new Promise(() => {})); - const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); - - try { - const pending = diarizeWithOpenAI({ - apiKey: "sk-test", - chunkFiles: [chunk("a.mp3")], - maxRetries: 3, - onProgress: () => {}, - signal: controller.signal, - }); - await vi.waitFor(() => expect(openAIMock).toHaveBeenCalledOnce()); - - controller.abort(abortError); - - await expect(pending).rejects.toBe(abortError); - expect(openAIMock).toHaveBeenCalledTimes(1); - expect(consoleError).not.toHaveBeenCalled(); - } finally { - consoleError.mockRestore(); - } - }); -}); diff --git a/src/services/diarization/openaiProvider.ts b/src/services/diarization/openaiProvider.ts deleted file mode 100644 index cbc705c2..00000000 --- a/src/services/diarization/openaiProvider.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { toError } from "../../utility/toError"; -import { canRetry, createTranscription, retryDelayMs } from "../openaiTranscription"; -import { type DiarizedSegment, OPENAI_DIARIZE_MODEL } from "./types"; -import { parseOpenAIDiarizedSegments } from "./segments"; - -/** - * Diarize with OpenAI's `gpt-4o-transcribe-diarize` model (issue #168). - * - * Each chunk stays under ~20 MB (a conservative margin under OpenAI's 25 MB - * request cap) and 600 s, which the model answers inside the 10 min timeout. `chunking_strategy: "auto"` is required - * for audio longer than 30s and lets the model segment within a chunk. - * Speaker labels are assigned per request, so on a multi-chunk (long) episode - * the labels can differ across chunk boundaries — the caller documents this; a - * single-chunk episode (the common case) is fully consistent. - * - * Chunks are processed in order and their segments concatenated, so the - * transcript reads start-to-finish. A single failed chunk contributes an error - * marker segment rather than discarding an otherwise-good transcript — but if - * EVERY chunk fails (e.g. an invalid key or unsupported model) the function - * throws so the caller writes no file and the run stays retryable, instead of - * saving a transcript made only of error markers. - */ -export async function diarizeWithOpenAI(opts: { - apiKey: string; - chunkFiles: File[]; - maxRetries: number; - onProgress: (message: string) => void; - signal: AbortSignal; -}): Promise { - const { apiKey, chunkFiles, maxRetries, onProgress, signal } = opts; - throwIfAborted(signal); - const segments: DiarizedSegment[] = []; - let failedChunks = 0; - let lastError: unknown; - - for (let index = 0; index < chunkFiles.length; index++) { - throwIfAborted(signal); - onProgress(`Diarizing with OpenAI... chunk ${index + 1}/${chunkFiles.length}`); - const file = chunkFiles[index]; - - let attempt = 0; - while (true) { - throwIfAborted(signal); - try { - const result = await createTranscription( - apiKey, - { - model: OPENAI_DIARIZE_MODEL, - file, - response_format: "diarized_json", - chunking_strategy: "auto", - }, - signal, - ); - throwIfAborted(signal); - // Each chunk's start/end are relative to that chunk, so the - // concatenated segments' timestamps are not episode-absolute. The - // rendered transcript does not surface timestamps today; if it ever - // does, offset each chunk by the cumulative prior-chunk duration here. - segments.push(...parseOpenAIDiarizedSegments(result)); - break; - } catch (error) { - throwIfAborted(signal); - attempt++; - if (attempt >= maxRetries || !canRetry(error)) { - console.error( - `OpenAI diarization failed for chunk ${index + 1} after ${attempt} attempt(s):`, - error, - ); - failedChunks++; - lastError = error; - segments.push({ - speaker: "?", - text: `[Error diarizing chunk ${index + 1}]`, - }); - break; - } - await waitForRetry(retryDelayMs(error, 1000 * attempt), signal); - } - } - } - - if (chunkFiles.length > 0 && failedChunks === chunkFiles.length) { - const detail = lastError instanceof Error ? lastError.message : String(lastError); - throw new Error(`OpenAI diarization failed for every chunk: ${detail}`); - } - - return segments; -} - -function throwIfAborted(signal: AbortSignal): void { - if (!signal.aborted) return; - throw signal.reason ?? new DOMException("OpenAI diarization was aborted.", "AbortError"); -} - -function waitForRetry(delayMs: number, signal: AbortSignal): Promise { - throwIfAborted(signal); - - return new Promise((resolve, reject) => { - const timeout = window.setTimeout(() => { - signal.removeEventListener("abort", onAbort); - resolve(); - }, delayMs); - const onAbort = () => { - window.clearTimeout(timeout); - signal.removeEventListener("abort", onAbort); - reject(toError(signal.reason, "OpenAI diarization was aborted.")); - }; - - signal.addEventListener("abort", onAbort, { once: true }); - if (signal.aborted) onAbort(); - }); -} diff --git a/src/services/diarization/requiredTranscriptionKeyPresent.test.ts b/src/services/diarization/requiredTranscriptionKeyPresent.test.ts deleted file mode 100644 index b1dd5f7e..00000000 --- a/src/services/diarization/requiredTranscriptionKeyPresent.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DEFAULT_SETTINGS } from "src/constants"; -import type { IPodNotesSettings } from "src/types/IPodNotesSettings"; -import { requiredTranscriptionKeyPresent } from "./index"; - -function settings(overrides: Partial = {}): IPodNotesSettings { - return structuredClone({ ...DEFAULT_SETTINGS, ...overrides }); -} - -function withDiarization(provider: "openai" | "deepgram", enabled: boolean): IPodNotesSettings { - const s = settings(); - s.transcript.diarization.enabled = enabled; - s.transcript.diarization.provider = provider; - return s; -} - -const has = - (...kinds: Array<"openai" | "deepgram">) => - (kind: "openai" | "deepgram") => - kinds.includes(kind); - -describe("requiredTranscriptionKeyPresent (#168)", () => { - it("requires the OpenAI key when diarization is off", () => { - expect(requiredTranscriptionKeyPresent(settings(), has("openai"))).toBe(true); - expect(requiredTranscriptionKeyPresent(settings(), has())).toBe(false); - }); - - it("requires the OpenAI key for the OpenAI diarization provider", () => { - expect( - requiredTranscriptionKeyPresent(withDiarization("openai", true), has("openai")), - ).toBe(true); - expect( - requiredTranscriptionKeyPresent(withDiarization("openai", true), has("deepgram")), - ).toBe(false); - }); - - it("requires the Deepgram key for the Deepgram diarization provider", () => { - expect( - requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has("deepgram")), - ).toBe(true); - // An OpenAI key does not satisfy the Deepgram provider. - expect( - requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has("openai")), - ).toBe(false); - }); - - it("falls back to the OpenAI key when Deepgram is selected but diarization is off", () => { - // provider=deepgram only matters when diarization is enabled. - expect( - requiredTranscriptionKeyPresent(withDiarization("deepgram", false), has("openai")), - ).toBe(true); - }); - - it("treats dangling SecretStorage references as absent", () => { - expect(requiredTranscriptionKeyPresent(settings(), has())).toBe(false); - expect(requiredTranscriptionKeyPresent(withDiarization("deepgram", true), has())).toBe( - false, - ); - }); -}); diff --git a/src/services/diarization/segments.test.ts b/src/services/diarization/segments.test.ts deleted file mode 100644 index d52f0fe3..00000000 --- a/src/services/diarization/segments.test.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DEFAULT_SETTINGS } from "src/constants"; -import { - mergeAdjacentSpeakers, - parseDeepgramSegments, - parseOpenAIDiarizedSegments, - renderDiarizedTranscript, -} from "./segments"; - -describe("parseOpenAIDiarizedSegments (#168)", () => { - it("maps diarized_json segments to normalized turns", () => { - const payload = { - segments: [ - { - type: "transcript.text.segment", - id: "1", - speaker: "A", - start: 0, - end: 2, - text: "Hello there.", - }, - { - type: "transcript.text.segment", - id: "2", - speaker: "B", - start: 2, - end: 4, - text: "Hi!", - }, - ], - }; - - expect(parseOpenAIDiarizedSegments(payload)).toEqual([ - { speaker: "A", text: "Hello there.", start: 0, end: 2 }, - { speaker: "B", text: "Hi!", start: 2, end: 4 }, - ]); - }); - - it("merges consecutive same-speaker segments into one turn", () => { - const payload = { - segments: [ - { speaker: "A", start: 0, end: 1, text: "One." }, - { speaker: "A", start: 1, end: 2, text: "Two." }, - { speaker: "B", start: 2, end: 3, text: "Three." }, - ], - }; - - expect(parseOpenAIDiarizedSegments(payload)).toEqual([ - { speaker: "A", text: "One. Two.", start: 0, end: 2 }, - { speaker: "B", text: "Three.", start: 2, end: 3 }, - ]); - }); - - it("skips empty/malformed entries instead of throwing", () => { - const payload = { - segments: [ - { speaker: "A", text: " " }, - null, - { speaker: "A", text: "Real." }, - { text: "No speaker field" }, - ], - }; - - expect(parseOpenAIDiarizedSegments(payload)).toEqual([ - { speaker: "A", text: "Real.", start: undefined, end: undefined }, - { speaker: "?", text: "No speaker field", start: undefined, end: undefined }, - ]); - }); - - it("returns [] for non-object or shapeless payloads", () => { - expect(parseOpenAIDiarizedSegments(null)).toEqual([]); - expect(parseOpenAIDiarizedSegments("nope")).toEqual([]); - expect(parseOpenAIDiarizedSegments({})).toEqual([]); - expect(parseOpenAIDiarizedSegments({ segments: "x" })).toEqual([]); - }); -}); - -describe("parseDeepgramSegments (#168)", () => { - it("prefers utterances and presents speakers 1-based", () => { - const payload = { - results: { - utterances: [ - { speaker: 0, transcript: "Hello there.", start: 0, end: 2 }, - { speaker: 1, transcript: "Hi!", start: 2, end: 3 }, - ], - }, - }; - - expect(parseDeepgramSegments(payload)).toEqual([ - { speaker: "1", text: "Hello there.", start: 0, end: 2 }, - { speaker: "2", text: "Hi!", start: 2, end: 3 }, - ]); - }); - - it("falls back to grouping words when utterances are absent", () => { - const payload = { - results: { - channels: [ - { - alternatives: [ - { - words: [ - { - word: "hello", - punctuated_word: "Hello", - speaker: 0, - start: 0, - end: 1, - }, - { - word: "there", - punctuated_word: "there.", - speaker: 0, - start: 1, - end: 2, - }, - { - word: "hi", - punctuated_word: "Hi!", - speaker: 1, - start: 2, - end: 3, - }, - ], - }, - ], - }, - ], - }, - }; - - expect(parseDeepgramSegments(payload)).toEqual([ - { speaker: "1", text: "Hello there.", start: 0, end: 2 }, - { speaker: "2", text: "Hi!", start: 2, end: 3 }, - ]); - }); - - it("returns [] for a results object with neither utterances nor words", () => { - expect(parseDeepgramSegments({ results: {} })).toEqual([]); - expect(parseDeepgramSegments({})).toEqual([]); - expect(parseDeepgramSegments(null)).toEqual([]); - }); -}); - -describe("mergeAdjacentSpeakers (#168)", () => { - it("does not mutate the input array", () => { - const input = [ - { speaker: "A", text: "One." }, - { speaker: "A", text: "Two." }, - ]; - const copy = structuredClone(input); - mergeAdjacentSpeakers(input); - expect(input).toEqual(copy); - }); -}); - -describe("renderDiarizedTranscript (#168)", () => { - const DEFAULT_SPEAKER_TEMPLATE = DEFAULT_SETTINGS.transcript.diarization.speakerTemplate; - - it("renders speaker turns separated by blank lines", () => { - const body = renderDiarizedTranscript( - [ - { speaker: "A", text: "Hello there." }, - { speaker: "B", text: "Hi!" }, - ], - DEFAULT_SPEAKER_TEMPLATE, - ); - - expect(body).toBe("**A:** Hello there.\n\n**B:** Hi!"); - }); - - it("honors a custom speaker template", () => { - const body = renderDiarizedTranscript( - [{ speaker: "1", text: "Hi." }], - "Speaker {{speaker}}: ", - ); - - expect(body).toBe("Speaker 1: Hi."); - }); - - it("falls back to the default template when given an empty one", () => { - const body = renderDiarizedTranscript([{ speaker: "A", text: "Hi." }], ""); - expect(body).toBe("**A:** Hi."); - }); - - it("returns an empty string for no segments", () => { - expect(renderDiarizedTranscript([], DEFAULT_SPEAKER_TEMPLATE)).toBe(""); - }); -}); diff --git a/src/services/diarization/segments.ts b/src/services/diarization/segments.ts deleted file mode 100644 index cfd6fccd..00000000 --- a/src/services/diarization/segments.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { DEFAULT_SETTINGS } from "src/constants"; -import { formatSpeakerLabel } from "src/services/publishedTranscript/render"; -import type { DiarizedSegment } from "./types"; - -/** - * Pure parsing/rendering for diarized transcripts (issue #168). - * - * The provider-specific network code lives in `openaiProvider.ts` / - * `deepgramProvider.ts`; everything here is side-effect free so the risky shape - * handling (untrusted JSON -> normalized segments -> markdown) is unit-testable. - */ - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - -function cleanText(text: unknown): string { - return typeof text === "string" ? text.trim() : ""; -} - -/** - * Parse OpenAI `diarized_json` output (from `gpt-4o-transcribe-diarize`) into - * normalized segments. The payload is `{ segments: [{ speaker, text, start, - * end, ... }] }` where `speaker` is a capital-letter label (`A`, `B`, ...). - * Malformed/empty entries are skipped rather than throwing so one bad segment - * can't sink an otherwise-good transcript. - */ -export function parseOpenAIDiarizedSegments(payload: unknown): DiarizedSegment[] { - if (!isRecord(payload) || !Array.isArray(payload.segments)) return []; - - const segments: DiarizedSegment[] = []; - for (const raw of payload.segments) { - if (!isRecord(raw)) continue; - const text = cleanText(raw.text); - if (!text) continue; - const speaker = - typeof raw.speaker === "string" && raw.speaker.trim() ? raw.speaker.trim() : "?"; - segments.push({ - speaker, - text, - start: typeof raw.start === "number" ? raw.start : undefined, - end: typeof raw.end === "number" ? raw.end : undefined, - }); - } - return mergeAdjacentSpeakers(segments); -} - -/** - * Parse Deepgram pre-recorded output into normalized segments. Prefers the - * top-level `results.utterances[]` (already grouped into speaker turns when the - * request sets `utterances=true`); otherwise falls back to grouping the - * per-word `results.channels[0].alternatives[0].words[]` by their integer - * `speaker`. Deepgram speakers are 0-based; we present them 1-based so a label - * never reads "Speaker 0". - */ -export function parseDeepgramSegments(payload: unknown): DiarizedSegment[] { - if (!isRecord(payload) || !isRecord(payload.results)) return []; - const results = payload.results; - - if (Array.isArray(results.utterances)) { - const segments: DiarizedSegment[] = []; - for (const raw of results.utterances) { - if (!isRecord(raw)) continue; - const text = cleanText(raw.transcript); - if (!text) continue; - segments.push({ - speaker: speakerLabelFromIndex(raw.speaker), - text, - start: typeof raw.start === "number" ? raw.start : undefined, - end: typeof raw.end === "number" ? raw.end : undefined, - }); - } - return mergeAdjacentSpeakers(segments); - } - - return mergeAdjacentSpeakers(groupDeepgramWords(results)); -} - -/** Deepgram fallback: stitch per-word objects into contiguous speaker turns. */ -function groupDeepgramWords(results: Record): DiarizedSegment[] { - const channels = results.channels; - if (!Array.isArray(channels) || !isRecord(channels[0])) return []; - const alternatives = channels[0].alternatives; - if (!Array.isArray(alternatives) || !isRecord(alternatives[0])) return []; - const words = alternatives[0].words; - if (!Array.isArray(words)) return []; - - const segments: DiarizedSegment[] = []; - for (const raw of words) { - if (!isRecord(raw)) continue; - const token = - typeof raw.punctuated_word === "string" - ? raw.punctuated_word - : typeof raw.word === "string" - ? raw.word - : ""; - if (!token) continue; - const speaker = speakerLabelFromIndex(raw.speaker); - const last = segments[segments.length - 1]; - if (last && last.speaker === speaker) { - last.text += ` ${token}`; - if (typeof raw.end === "number") last.end = raw.end; - } else { - segments.push({ - speaker, - text: token, - start: typeof raw.start === "number" ? raw.start : undefined, - end: typeof raw.end === "number" ? raw.end : undefined, - }); - } - } - return segments; -} - -function speakerLabelFromIndex(speaker: unknown): string { - return typeof speaker === "number" && Number.isFinite(speaker) ? String(speaker + 1) : "?"; -} - -/** - * Collapse runs of the same speaker into one turn. Providers can emit several - * back-to-back segments for the same speaker (e.g. one per sentence); merging - * them yields readable paragraphs and a stable shape for rendering. - */ -export function mergeAdjacentSpeakers(segments: DiarizedSegment[]): DiarizedSegment[] { - const merged: DiarizedSegment[] = []; - for (const segment of segments) { - const last = merged[merged.length - 1]; - if (last && last.speaker === segment.speaker) { - last.text = `${last.text} ${segment.text}`.trim(); - if (segment.end !== undefined) last.end = segment.end; - } else { - merged.push({ ...segment }); - } - } - return merged; -} - -/** - * Render normalized segments into the markdown body that fills `{{transcript}}`. - * Each turn is ``, turns separated by a blank line so they - * read as paragraphs. Returns an empty string for no segments so the caller can - * detect a diarization that produced nothing. - */ -export function renderDiarizedTranscript( - segments: DiarizedSegment[], - speakerTemplate: string, -): string { - const template = speakerTemplate || DEFAULT_SETTINGS.transcript.diarization.speakerTemplate; - return segments - .map((segment) => `${formatSpeakerLabel(template, segment.speaker)}${segment.text}`) - .join("\n\n"); -} diff --git a/src/services/diarization/types.ts b/src/services/diarization/types.ts deleted file mode 100644 index 78ec6add..00000000 --- a/src/services/diarization/types.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Speaker diarization adds "who spoke when" labels to a transcript (issue #168). - * - * OpenAI Whisper (`whisper-1`) — the plugin's default transcription backend — - * produces no speaker labels, so diarization is an opt-in path that routes the - * audio to a diarization-capable provider instead. Two providers are supported: - * - * - `openai`: the `gpt-4o-transcribe-diarize` model on the same - * `/v1/audio/transcriptions` endpoint the plugin already uses, reusing the - * user's existing OpenAI key. Convenient, but each request takes at most - * ~20 MB and 600 s of audio, so a long episode is split into chunks whose - * speaker labels are not guaranteed to line up across chunk boundaries (the - * labels are assigned per request). - * - `deepgram`: a single whole-file request to Deepgram's pre-recorded API, - * which keeps speaker identity consistent across the entire episode but needs - * its own API key. - */ -export type DiarizationProviderId = "openai" | "deepgram"; - -export const DIARIZATION_PROVIDERS: readonly DiarizationProviderId[] = ["openai", "deepgram"]; - -/** The fixed OpenAI model that performs diarization (no other OpenAI model does). */ -export const OPENAI_DIARIZE_MODEL = "gpt-4o-transcribe-diarize"; - -/** - * The most audio, in seconds, in one diarization request. OpenAI accepts up to 1400 s, but the model - * takes up to 0.58 s per second of audio, so 600 s answers in under 6 min, inside the 10 min request timeout. - */ -export const OPENAI_DIARIZE_PART_SECONDS = 600; - -/** - * One contiguous turn of speech attributed to a single speaker. `speaker` is the - * provider's label, normalized to a short human-facing token (OpenAI emits `A`, - * `B`, ...; Deepgram emits 0-based integers which we present 1-based as `1`, - * `2`, ...). `start`/`end` are seconds when the provider reports them. - */ -export interface DiarizedSegment { - speaker: string; - text: string; - start?: number; - end?: number; -} - -/** The whole, original (compressed) episode audio handed to a diarization provider. */ -export interface DiarizationAudio { - buffer: ArrayBuffer; - mimeType: string; - extension: string; - basename: string; -} diff --git a/src/services/openaiTranscription.test.ts b/src/services/openaiTranscription.test.ts deleted file mode 100644 index 901fff35..00000000 --- a/src/services/openaiTranscription.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { OPENAI_FAILURES, OPENAI_REJECTIONS, rateLimited } from "../../tests/mocks/openaiFailures"; -import { corsBlockedFetch, openAIMock, sentForm } from "../../tests/mocks/openaiRequestUrl"; -import { createTranscription, retryDelayMs } from "./openaiTranscription"; - -vi.mock("obsidian", async (importOriginal) => ({ - ...(await importOriginal()), - requestUrl: (await import("../../tests/mocks/openaiRequestUrl")).openAIRequestUrl, -})); - -const neverAnswers = () => new Promise(() => {}); - -const send = (signal = new AbortController().signal) => - createTranscription("sk-test", { model: "whisper-1" }, signal); - -const status500 = (body: string) => async () => new Response(body, { status: 500 }); - -const EDGE_FAILURES: typeof OPENAI_FAILURES = [ - [ - "500 top-level message", - status500('{"message":"Overloaded","type":"server_error"}'), - "500 Overloaded", - ], - [ - "500 empty error.message", - status500('{"error":{"message":"","type":"server_error"}}'), - '500 {"message":"","type":"server_error"}', - ], - [ - "500 non-string error.message", - status500('{"error":{"message":{"reason":"x"}}}'), - '500 {"reason":"x"}', - ], - ["500 error without message", status500('{"error":{"code":"busy"}}'), '500 {"code":"busy"}'], - ["500 error is a string", status500('{"error":"busy"}'), '500 "busy"'], - ["500 JSON number", status500("42"), "500 status code (no body)"], - ["500 JSON null", status500("null"), "500 null"], -]; - -beforeEach(() => { - vi.stubGlobal("fetch", corsBlockedFetch); - // In Obsidian's renderer `window` is the global object; the code under test uses its timers. - vi.stubGlobal("window", globalThis); -}); - -afterEach(() => { - vi.useRealTimers(); - vi.unstubAllGlobals(); - openAIMock.mockReset(); -}); - -describe("createTranscription", () => { - it("posts multipart form data that parses back to the same fields and file bytes", async () => { - openAIMock.mockImplementation(async () => Response.json({ text: "Hi." })); - const bytes = new Uint8Array(70_000).map((_, i) => (i * 31) % 256); - const file = new File([bytes], "episode.mp3", { type: "audio/mp3" }); - - await createTranscription( - "sk-test", - { - model: "gpt-4o-transcribe-diarize", - file, - response_format: "diarized_json", - chunking_strategy: "auto", - }, - new AbortController().signal, - ); - - expect(openAIMock).toHaveBeenCalledOnce(); - const [request] = openAIMock.mock.calls[0]; - expect(request.url).toBe("https://api.openai.com/v1/audio/transcriptions"); - expect(request.method).toBe("POST"); - expect(request.headers).toEqual({ Authorization: "Bearer sk-test" }); - expect(request.contentType).toMatch(/^multipart\/form-data; boundary=/); - const form = await sentForm(request); - expect([...form.keys()]).toEqual(["model", "file", "response_format", "chunking_strategy"]); - expect(form.get("model")).toBe("gpt-4o-transcribe-diarize"); - expect(form.get("response_format")).toBe("diarized_json"); - expect(form.get("chunking_strategy")).toBe("auto"); - const sent = form.get("file") as File; - expect(sent.name).toBe("episode.mp3"); - expect(sent.type).toBe("audio/mp3"); - expect(new Uint8Array(await sent.arrayBuffer())).toEqual(bytes); - }); - - it("resolves with the parsed JSON body", async () => { - const payload = { - text: "Hi.", - segments: [{ type: "transcript.text.segment", speaker: "A", start: 0, end: 1 }], - }; - openAIMock.mockImplementation(async () => Response.json(payload)); - - await expect(send()).resolves.toEqual(payload); - }); - - it.each([...OPENAI_FAILURES, ...OPENAI_REJECTIONS, ...EDGE_FAILURES])( - "%s rejects with the openai SDK's message after one send", - async (_name, respond, message) => { - openAIMock.mockImplementation(respond); - - await expect(send()).rejects.toHaveProperty("message", message); - expect(openAIMock).toHaveBeenCalledOnce(); - }, - ); - - it("keeps the openai SDK's message for a 429 with retry headers", async () => { - openAIMock.mockImplementation(async () => - rateLimited({ "retry-after": "4", "retry-after-ms": "4000" }), - ); - - await expect(send()).rejects.toHaveProperty( - "message", - "429 Rate limit reached for requests.", - ); - }); - - it.each<[name: string, headers: Record, delayMs: number]>([ - ["retry-after-ms", { "retry-after-ms": "1500" }, 1500], - ["retry-after seconds", { "retry-after": "4" }, 4000], - ["retry-after HTTP date", { "retry-after": "Thu, 01 Jan 2026 00:00:04 GMT" }, 4000], - ["both headers", { "retry-after-ms": "1500", "retry-after": "4" }, 1500], - ["retry-after over 60 s", { "retry-after": "61" }, 1234], - ["negative retry-after-ms", { "retry-after-ms": "-5" }, 1234], - [ - "negative retry-after-ms with retry-after", - { "retry-after-ms": "-5", "retry-after": "4" }, - 1234, - ], - ["unparseable retry-after", { "retry-after": "soon" }, 1234], - ["no retry header", {}, 1234], - ["retry-after zero", { "retry-after": "0" }, 0], - ])("%s gives the same retry delay as openai-node", async (_name, headers, delayMs) => { - vi.useFakeTimers({ now: new Date("2026-01-01T00:00:00Z") }); - openAIMock.mockImplementation(async () => rateLimited(headers)); - - const error = await send().catch((caught: unknown) => caught); - - expect(retryDelayMs(error, 1234)).toBe(delayMs); - }); - - it("stops waiting for a stalled request after 10 minutes", async () => { - vi.useFakeTimers(); - openAIMock.mockImplementation(neverAnswers); - let settled = false; - const pending = send().finally(() => { - settled = true; - }); - const assertion = expect(pending).rejects.toThrow(new Error("Request timed out.")); - - await vi.advanceTimersByTimeAsync(10 * 60 * 1000 - 1); - expect(settled).toBe(false); - await vi.advanceTimersByTimeAsync(1); - - await assertion; - }); - - it("stops waiting on abort and rejects with the caller's abort reason", async () => { - openAIMock.mockImplementation(neverAnswers); - const controller = new AbortController(); - const reason = new DOMException( - "PodNotes was unloaded during transcription.", - "AbortError", - ); - const pending = send(controller.signal); - await vi.waitFor(() => expect(openAIMock).toHaveBeenCalledOnce()); - - controller.abort(reason); - - await expect(pending).rejects.toBe(reason); - }); -}); diff --git a/src/services/openaiTranscription.ts b/src/services/openaiTranscription.ts deleted file mode 100644 index 42d48a8d..00000000 --- a/src/services/openaiTranscription.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { - TimeoutError, - requestWithTimeout, - type NetworkBinaryResponse, -} from "../utility/networkRequest"; - -const TRANSCRIPTIONS_URL = "https://api.openai.com/v1/audio/transcriptions"; -const REQUEST_TIMEOUT_MS = 10 * 60 * 1000; -const MAX_REQUEST_BYTES = 25 * 1024 * 1024; -const MAX_SERVER_RETRY_DELAY_MS = 60 * 1000; -const EVERY_HTTP_STATUS = Array.from({ length: 500 }, (_, index) => 100 + index); - -// OpenAI's responses do not allow Obsidian's origin, so the renderer's fetch -// cannot read them. requestUrl has no CORS check but cannot cancel an upload, -// so a timeout or abort only stops waiting and drops the late response. -export async function createTranscription( - apiKey: string, - fields: Record, - signal: AbortSignal, -): Promise { - const form = new FormData(); - for (const [name, value] of Object.entries(fields)) form.append(name, value); - const multipart = new Request(TRANSCRIPTIONS_URL, { method: "POST", body: form }); - - let response: NetworkBinaryResponse; - try { - response = await requestWithTimeout(TRANSCRIPTIONS_URL, { - method: "POST", - headers: { Authorization: `Bearer ${apiKey}` }, - contentType: multipart.headers.get("content-type") ?? undefined, - body: await multipart.arrayBuffer(), - timeoutMs: REQUEST_TIMEOUT_MS, - maxRequestBodyBytes: MAX_REQUEST_BYTES, - acceptedStatuses: EVERY_HTTP_STATUS, - signal, - }); - } catch (error) { - if (signal.aborted) throw signal.reason; - throw new Error(error instanceof TimeoutError ? "Request timed out." : "Connection error."); - } - - const text = new TextDecoder().decode(response.arrayBuffer); - if (response.status < 200 || response.status >= 300) { - throw new OpenAIStatusError( - `${response.status} ${errorDetail(text) || "status code (no body)"}`, - response.status, - serverRetryDelayMs(new Headers(response.headers)), - ); - } - return JSON.parse(text); -} - -export function retryDelayMs(error: unknown, fallbackMs: number): number { - return (error instanceof OpenAIStatusError ? error.retryAfterMs : undefined) ?? fallbackMs; -} - -/** Whether sending the request again can succeed. Statuses follow openai-node's retry rule. */ -export function canRetry(error: unknown): boolean { - if (!(error instanceof OpenAIStatusError)) return true; - const { status } = error; - return status === 408 || status === 409 || status === 429 || status >= 500; -} - -class OpenAIStatusError extends Error { - constructor( - message: string, - readonly status: number, - readonly retryAfterMs: number | undefined, - ) { - super(message); - } -} - -function serverRetryDelayMs(headers: Headers): number | undefined { - let delayMs = parseFloat(headers.get("retry-after-ms") ?? ""); - const retryAfter = headers.get("retry-after"); - if (Number.isNaN(delayMs) && retryAfter) { - const seconds = parseFloat(retryAfter); - delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000; - } - return delayMs >= 0 && delayMs <= MAX_SERVER_RETRY_DELAY_MS ? delayMs : undefined; -} - -// Mirrors openai-node's makeStatusError and APIError.makeMessage, so failure -// notices read exactly as they did when PodNotes used the SDK. -function errorDetail(body: string): string | undefined { - let json: unknown; - try { - json = JSON.parse(body); - } catch {} - if (!json) return body; - if (typeof json !== "object") return undefined; - const error = "error" in json && json.error != null ? json.error : json; - const message = - typeof error === "object" && error !== null && "message" in error - ? error.message - : undefined; - if (message) return typeof message === "string" ? message : JSON.stringify(message); - return error ? JSON.stringify(error) : undefined; -} diff --git a/src/services/publishedTranscript/index.ts b/src/services/publishedTranscript/index.ts index ef77d7f1..108c4ca6 100644 --- a/src/services/publishedTranscript/index.ts +++ b/src/services/publishedTranscript/index.ts @@ -10,6 +10,7 @@ import { } from "src/utility/privateFeedUrl"; import { type ParsedTranscript, parseTranscript, type TranscriptFormat } from "./parse"; +export type { ParsedTranscript } from "./parse"; export { renderTranscriptBody, timestampStyle } from "./render"; const FORMATS_BY_PREFERENCE: ReadonlyArray = [ diff --git a/src/settingsMigrations.ts b/src/settingsMigrations.ts index 7ecac037..53e30a19 100644 --- a/src/settingsMigrations.ts +++ b/src/settingsMigrations.ts @@ -1,5 +1,4 @@ import { DEFAULT_SETTINGS } from "./constants"; -import { DIARIZATION_PROVIDERS, type DiarizationProviderId } from "./services/diarization/types"; import type { IPodNotesSettings } from "./types/IPodNotesSettings"; /** @@ -138,9 +137,11 @@ export function migrateTranscriptSettings( }; } -function sanitizeDiarizationProvider(value: unknown): DiarizationProviderId { - return DIARIZATION_PROVIDERS.includes(value as DiarizationProviderId) - ? (value as DiarizationProviderId) +function sanitizeDiarizationProvider( + value: unknown, +): IPodNotesSettings["transcript"]["diarization"]["provider"] { + return value === "openai" || value === "deepgram" + ? value : DEFAULT_SETTINGS.transcript.diarization.provider; } diff --git a/src/types/IPodNotesSettings.ts b/src/types/IPodNotesSettings.ts index 246e1011..801e4c6d 100644 --- a/src/types/IPodNotesSettings.ts +++ b/src/types/IPodNotesSettings.ts @@ -4,7 +4,6 @@ import type { PlayedEpisode } from "./PlayedEpisode"; import type { Playlist } from "./Playlist"; import type { Episode } from "./Episode"; import type DownloadedEpisode from "./DownloadedEpisode"; -import type { DiarizationProviderId } from "src/services/diarization/types"; import type { NextTrackAction, PreviousTrackAction } from "./Headset"; export interface IPodNotesSettings { @@ -67,14 +66,13 @@ export interface IPodNotesSettings { path: string; template: string; /** - * Opt-in speaker diarization for transcripts (issue #168). When `enabled` - * the audio is routed to a diarization-capable `provider` instead of plain - * Whisper, and the transcript is rendered as speaker-labeled turns using - * `speakerTemplate` (where `{{speaker}}` is the per-turn speaker label). + * Speaker labels for transcripts. When `enabled` and `provider` is `"deepgram"`, audio + * goes to Deepgram, which labels speakers. Otherwise it goes to OpenAI, which does not. + * `speakerTemplate` shapes each label, where `{{speaker}}` is the speaker's name. */ diarization: { enabled: boolean; - provider: DiarizationProviderId; + provider: "openai" | "deepgram"; speakerTemplate: string; }; }; diff --git a/src/ui/PodcastView/TranscriptPanel.svelte b/src/ui/PodcastView/TranscriptPanel.svelte index 5cea738e..e6d77efb 100644 --- a/src/ui/PodcastView/TranscriptPanel.svelte +++ b/src/ui/PodcastView/TranscriptPanel.svelte @@ -3,6 +3,7 @@ import type { Episode } from "src/types/Episode"; import { formatDuration } from "src/utility/formatDuration"; import { isLocalFile } from "src/utility/isLocalFile"; + import { transcriptionChoice } from "src/services/transcription"; import { transcriptNote } from "src/services/TranscriptionService"; import { getEpisodeMediaType } from "src/utility/mediaType"; import { icon } from "../obsidian/icon"; @@ -66,10 +67,7 @@ void $rowHasNote; return transcriptNote(app.vault, $settings, episode); }); - const provider = $derived.by(() => { - const { diarization } = $settings.transcript; - return diarization?.enabled && diarization.provider === "deepgram" ? "Deepgram" : "OpenAI"; - }); + const provider = $derived(transcriptionChoice($settings).label); $effect(() => { finding = false; diff --git a/src/ui/settings/PodNotesSettingsTab.ts b/src/ui/settings/PodNotesSettingsTab.ts index ad738ec1..2a31d478 100644 --- a/src/ui/settings/PodNotesSettingsTab.ts +++ b/src/ui/settings/PodNotesSettingsTab.ts @@ -82,7 +82,7 @@ interface TextField { } const DIARIZATION_PROVIDERS = { - openai: "OpenAI (gpt-4o-transcribe-diarize)", + openai: "OpenAI (no speaker labels)", deepgram: "Deepgram", }; @@ -376,7 +376,7 @@ export class PodNotesSettingsTab extends PluginSettingTab { (provider) => { diarization().provider = provider; }, - "Deepgram needs its own API key and keeps speaker labels consistent across long episodes.", + "Deepgram needs its own API key.", ), ({ enabled }) => enabled, ), diff --git a/tests/e2e/journey-resilience.test.ts b/tests/e2e/journey-resilience.test.ts index 1933844c..fed5cdb9 100644 --- a/tests/e2e/journey-resilience.test.ts +++ b/tests/e2e/journey-resilience.test.ts @@ -632,7 +632,7 @@ describe("resilience journeys", () => { sandbox.path("Transcripts/Journey Transcribe/Spoken Episode.md"), ); expect(transcript).toMatch( - /^# Spoken Episode\n\nPodcast: Journey Transcribe\nDate: .+\n\nJourney transcript opens here\.\n\nIt closes with a second sentence\.$/, + /^# Spoken Episode\n\nPodcast: Journey Transcribe\nDate: .+\n\nJourney transcript opens here\. It closes with a second sentence\.$/, ); const uploads = await transcriptionUploads(obsidian); @@ -640,7 +640,9 @@ describe("resilience journeys", () => { expect(uploads[0]).toMatchObject({ method: "POST", authorization: "Bearer sk-journey-test", - model: "whisper-1", + model: "gpt-transcribe", + responseFormat: "json", + fileName: "audio.wav", fileBytes: server.audio(show, spoken).length, riffWave: true, sha256: createHash("sha256").update(server.audio(show, spoken)).digest("hex"), @@ -801,7 +803,7 @@ describe("resilience journeys", () => { await sandbox.waitForExists(notePath, { timeoutMs: 30_000 }); expect(await readVaultText(obsidian, sandbox.path(notePath))).toMatch( - /^# Spoken Episode\n\nPodcast: Journey Missing Captions\nDate: .+\n\nPaid transcript opens here\.\n\nIt closes here\.$/, + /^# Spoken Episode\n\nPodcast: Journey Missing Captions\nDate: .+\n\nPaid transcript opens here\. It closes here\.$/, ); const uploads = await transcriptionUploads(obsidian); expect(uploads).toHaveLength(1); diff --git a/tests/e2e/journey/fixtures.ts b/tests/e2e/journey/fixtures.ts index 588f20dd..7bd14b65 100644 --- a/tests/e2e/journey/fixtures.ts +++ b/tests/e2e/journey/fixtures.ts @@ -461,6 +461,7 @@ export type TranscriptionUpload = Readonly<{ method: string; authorization: string | null; model: string | null; + responseFormat: string | null; fileName: string | null; fileBytes: number; riffWave: boolean; @@ -486,6 +487,7 @@ export async function installOpenAIStub(obsidian: ObsidianClient, text: string): method: (request.method ?? "GET").toUpperCase(), authorization: new Headers(request.headers).get("authorization"), model: form.get("model"), + responseFormat: form.get("response_format"), fileName: file instanceof File ? file.name : null, fileBytes: bytes.byteLength, riffWave: ascii(0, 4) === "RIFF" && ascii(8, 12) === "WAVE", diff --git a/tests/mocks/openaiFailures.ts b/tests/mocks/openaiFailures.ts deleted file mode 100644 index 196ba3a2..00000000 --- a/tests/mocks/openaiFailures.ts +++ /dev/null @@ -1,60 +0,0 @@ -const openAIError = (status: number, message: string) => async () => - Response.json({ error: { message, type: "error", param: null, code: null } }, { status }); - -type Failure = [name: string, respond: () => Promise, message: string]; - -/** Failures that sending the same request again can get past. */ -export const OPENAI_FAILURES: Failure[] = [ - [ - "429", - openAIError( - 429, - "You exceeded your current quota, please check your plan and billing details.", - ), - "429 You exceeded your current quota, please check your plan and billing details.", - ], - [ - "500", - openAIError( - 500, - "The server had an error while processing your request. Sorry about that!", - ), - "500 The server had an error while processing your request. Sorry about that!", - ], - ["502 text body", async () => new Response("Bad Gateway", { status: 502 }), "502 Bad Gateway"], - ["503 empty body", async () => new Response("", { status: 503 }), "503 status code (no body)"], - ["network", () => Promise.reject(new TypeError("Failed to fetch")), "Connection error."], -]; - -/** Failures that the same request always gets again. */ -export const OPENAI_REJECTIONS: Failure[] = [ - [ - "400", - openAIError( - 400, - "audio duration 1520.592 seconds is longer than 1400 seconds which is the maximum for this model", - ), - "400 audio duration 1520.592 seconds is longer than 1400 seconds which is the maximum for this model", - ], - [ - "401", - openAIError( - 401, - "Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", - ), - "401 Incorrect API key provided: sk-test. You can find your API key at https://platform.openai.com/account/api-keys.", - ], -]; - -export const rateLimited = (headers: Record) => - Response.json( - { - error: { - message: "Rate limit reached for requests.", - type: "requests", - param: null, - code: "rate_limit_exceeded", - }, - }, - { status: 429, headers }, - ); diff --git a/tests/mocks/openaiRequestUrl.ts b/tests/mocks/openaiRequestUrl.ts deleted file mode 100644 index ea5373cc..00000000 --- a/tests/mocks/openaiRequestUrl.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { RequestUrlParam, RequestUrlResponse } from "obsidian"; -import { vi } from "vitest"; - -/** OpenAI's side of Obsidian's requestUrl. Each test answers with a fetch Response. */ -export const openAIMock = vi.fn<(request: RequestUrlParam) => Promise>(); - -/** Stands in for Obsidian's requestUrl and answers with `openAIMock`. */ -export async function openAIRequestUrl(request: RequestUrlParam): Promise { - const response = await openAIMock(request); - const arrayBuffer = await response.arrayBuffer(); - const text = new TextDecoder().decode(arrayBuffer); - return { - status: response.status, - headers: Object.fromEntries(response.headers), - arrayBuffer, - text, - get json() { - return JSON.parse(text); - }, - }; -} - -/** The renderer's fetch as Obsidian runs it, where OpenAI's responses fail the CORS check. */ -export const corsBlockedFetch = () => Promise.reject(new TypeError("Failed to fetch")); - -/** The multipart form a request sent, parsed back by the platform. */ -export function sentForm(request: RequestUrlParam): Promise { - return new Response(request.body, { - headers: { "content-type": request.contentType ?? "" }, - }).formData(); -} diff --git a/tests/mocks/providerRequestUrl.ts b/tests/mocks/providerRequestUrl.ts new file mode 100644 index 00000000..a516c511 --- /dev/null +++ b/tests/mocks/providerRequestUrl.ts @@ -0,0 +1,47 @@ +import type { RequestUrlParam, RequestUrlResponse } from "obsidian"; +import { vi } from "vitest"; + +/** The providers' side of Obsidian's requestUrl. Each test answers with a fetch Response. */ +export const providerMock = vi.fn<(request: RequestUrlParam) => Promise>(); + +/** Stands in for Obsidian's requestUrl and answers with `providerMock`. */ +export async function providerRequestUrl(request: RequestUrlParam): Promise { + const response = await providerMock(request); + const arrayBuffer = await response.arrayBuffer(); + const text = new TextDecoder().decode(arrayBuffer); + return { + status: response.status, + headers: Object.fromEntries(response.headers), + arrayBuffer, + text, + get json() { + return JSON.parse(text); + }, + }; +} + +/** The renderer's fetch as Obsidian runs it, where a provider's responses fail the CORS check. */ +export const corsBlockedFetch = () => Promise.reject(new TypeError("Failed to fetch")); + +/** The multipart form a request sent, parsed back by the platform. */ +export function sentForm(request: RequestUrlParam): Promise { + return new Response(request.body, { + headers: { "content-type": request.contentType ?? "" }, + }).formData(); +} + +export function sentBytes(request: RequestUrlParam): Uint8Array { + return typeof request.body === "string" + ? new TextEncoder().encode(request.body) + : new Uint8Array(request.body ?? new ArrayBuffer(0)); +} + +/** Everything a provider can read in a request: its address, its headers and its body. */ +export function sentText(request: RequestUrlParam): string { + return [ + request.url, + request.contentType, + JSON.stringify(request.headers), + new TextDecoder().decode(sentBytes(request)), + ].join("\n"); +} diff --git a/tests/mocks/transcriptionService.ts b/tests/mocks/transcriptionService.ts index a2f36235..a8f3021a 100644 --- a/tests/mocks/transcriptionService.ts +++ b/tests/mocks/transcriptionService.ts @@ -1,34 +1,34 @@ import { TFile } from "obsidian"; +import { readable } from "svelte/store"; import { afterEach, beforeEach, vi } from "vitest"; import type PodNotes from "src/main"; import type { TranscriptLink } from "src/parser/feedDocumentParser"; +import type { FeedRepository } from "src/services/FeedRepository"; +import type { CredentialKind } from "src/types/Credentials"; import type { Episode } from "src/types/Episode"; -import { corsBlockedFetch, openAIMock, openAIRequestUrl } from "./openaiRequestUrl"; +import { corsBlockedFetch, providerMock, providerRequestUrl } from "./providerRequestUrl"; /** What each Notice opened with, oldest first. */ export const noticeMessages: string[] = []; -/** Stand-ins for the audio download and Deepgram. OpenAI answers through `openAIMock`. */ +/** Stands in for the audio download. The providers answer through `providerMock`. */ export const getEpisodeAudioBufferMock = vi.fn(); -export const diarizeWithDeepgramMock = vi.fn(); /** - * Resets the mocks above, `openAIMock` and `noticeMessages` before each test of the - * calling file, and blocks `fetch` as Obsidian's CORS check does for OpenAI. The file - * still mocks `obsidian`, `../downloadEpisode` and `./diarization` with these handles, - * because `vi.mock` applies only to its own file. + * Resets the mock above, `providerMock` and `noticeMessages` before each test of the + * calling file, and blocks `fetch` as Obsidian's CORS check does for a provider. The file + * still mocks `obsidian` and `../downloadEpisode` with these handles, because `vi.mock` + * applies only to its own file. */ export function installTranscriptionMocks(): void { beforeEach(() => { vi.clearAllMocks(); noticeMessages.length = 0; - openAIMock.mockReset(); + providerMock.mockReset(); vi.stubGlobal("fetch", corsBlockedFetch); - diarizeWithDeepgramMock.mockReset(); getEpisodeAudioBufferMock.mockResolvedValue({ buffer: new ArrayBuffer(1024), extension: "mp3", - basename: "episode", }); }); @@ -39,7 +39,7 @@ export function installTranscriptionMocks(): void { /** * The `obsidian` module whose Notice records its opening message in `noticeMessages` - * and whose requestUrl answers with `openAIMock`. + * and whose requestUrl answers with `providerMock`. */ export function mockedObsidian(obsidian: typeof import("obsidian")): typeof import("obsidian") { class RecordingNotice extends obsidian.Notice { @@ -51,7 +51,7 @@ export function mockedObsidian(obsidian: typeof import("obsidian")): typeof impo return { ...obsidian, Notice: RecordingNotice, - requestUrl: openAIRequestUrl as typeof obsidian.requestUrl, + requestUrl: providerRequestUrl as typeof obsidian.requestUrl, }; } @@ -70,6 +70,11 @@ export const mockEpisode: Episode = { export function createMockPlugin( overrides: { openAIKey?: string; + deepgramKey?: string; + /** Speaker labels are on, with this provider. */ + speakers?: "openai" | "deepgram"; + /** In place of a repository with no saved feed. */ + feeds?: FeedRepository; podcast?: Episode | null; existingTranscriptPath?: string | null; transcripts?: TranscriptLink[]; @@ -79,6 +84,9 @@ export function createMockPlugin( ): PodNotes { const { openAIKey = "test-api-key", + deepgramKey = "", + speakers, + feeds, podcast = mockEpisode, existingTranscriptPath = null, transcripts, @@ -88,6 +96,7 @@ export function createMockPlugin( const existing = (path: string) => path === existingTranscriptPath ? Object.assign(new TFile(), { path }) : null; + const keys: Record = { openai: openAIKey, deepgram: deepgramKey }; return { settings: { @@ -97,25 +106,27 @@ export function createMockPlugin( transcript: { path: "Transcripts/{{podcast}}/{{title}}.md", template: "# {{title}}\n\n{{transcript}}", + diarization: { + enabled: speakers !== undefined, + provider: speakers ?? "openai", + speakerTemplate: "", + }, }, download: { path: "Downloads", }, }, - feeds: { + feeds: feeds ?? { load: vi.fn(loadFeed), episodeMetadata: vi.fn(() => (transcripts ? { transcripts } : undefined)), itemUrl: vi.fn((_key: string, link: string) => new URL(link, feedUrl).href), + subscribe: readable(new Map()).subscribe, }, credentials: { - get: vi.fn((_settings, kind: "openai" | "deepgram") => - kind === "openai" ? openAIKey || null : null, - ), - has: vi.fn((_settings, kind: "openai" | "deepgram") => - kind === "openai" ? Boolean(openAIKey) : false, - ), - status: vi.fn((_settings, kind: "openai" | "deepgram") => - kind === "openai" && openAIKey ? "available" : "unconfigured", + get: vi.fn((_settings, kind: CredentialKind) => keys[kind] || null), + has: vi.fn((_settings, kind: CredentialKind) => Boolean(keys[kind])), + status: vi.fn((_settings, kind: CredentialKind) => + keys[kind] ? "available" : "unconfigured", ), }, api: { diff --git a/tests/tooling/transcriptionModels.test.ts b/tests/tooling/transcriptionModels.test.ts new file mode 100644 index 00000000..cf6fe269 --- /dev/null +++ b/tests/tooling/transcriptionModels.test.ts @@ -0,0 +1,28 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build } from "vite"; +import { expect, it, vi } from "vitest"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); + +it("main.js names gpt-transcribe and no model OpenAI shuts down on 2027-02-26", async () => { + vi.stubEnv("NODE_ENV", "production"); + try { + const result = await build({ + configFile: path.join(root, "vite.config.ts"), + configLoader: "bundle", + mode: "production", + logLevel: "silent", + build: { write: false }, + }); + if (!Array.isArray(result)) throw new Error("Expected library build outputs"); + const [chunk] = result.flatMap(({ output }) => output); + if (chunk?.type !== "chunk") throw new Error("Expected a JavaScript chunk"); + + // The retired ones are whisper-1 and the gpt-4o transcription models. + const named = chunk.code.match(/whisper-1|gpt-4o[\w-]*|gpt-transcribe/g) ?? []; + expect([...new Set(named)]).toEqual(["gpt-transcribe"]); + } finally { + vi.unstubAllEnvs(); + } +}, 120_000); From 2f9cd679c908b63aa662b562288ee4afee27bb94 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 22:24:34 +0100 Subject: [PATCH 09/26] test(verify): answer OpenAI and Deepgram as the engine reads them net.mjs answers OpenAI with { text } and Deepgram with the words of two speakers, and logs the content type of each request. fixtures.mjs serves a feed at priv3.supercast.com, an address that PodNotes keeps private. The transcripts recipe covers OpenAI in parts, and Deepgram by address and by upload. --- .claude/skills/verify/SKILL.md | 4 +++- .claude/skills/verify/features/transcripts.md | 15 ++++++++++++--- .claude/skills/verify/fixtures.mjs | 17 +++++++++++++++-- .claude/skills/verify/net.mjs | 16 +++++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/.claude/skills/verify/SKILL.md b/.claude/skills/verify/SKILL.md index 0ebf937c..26e341ce 100644 --- a/.claude/skills/verify/SKILL.md +++ b/.claude/skills/verify/SKILL.md @@ -266,7 +266,9 @@ its profile. It keeps the vault and `.obsidian-e2e-artifacts/`. Delete live in `screens/scenes.mjs`, the instance calls in `screens/instance.mjs`, and the contact sheet in `screens/sheet.mjs`. - `.claude/skills/verify/fixtures.mjs` runs as `feed`, `feed-b`, `transcripts`, - `library`, `retitle`, `requests`, `audio [vault path]` or `stop`. `library` + `private`, `library`, `retitle`, `requests`, `audio [vault path]` or `stop`. + `private` prints a feed at `priv3.supercast.com`, an address PodNotes keeps in + SecretStorage as it does a Supercast subscriber's. `library` prints the URL of a manifest of eight podcasts, one whose feed answers 404, and one more feed to add. Their episodes' media is silence as long as the feed says. `#213` has chapters and a WebVTT transcript, and `Signal & Noise` has a 10 s video episode that diff --git a/.claude/skills/verify/features/transcripts.md b/.claude/skills/verify/features/transcripts.md index 8d66aed4..7a23a67e 100644 --- a/.claude/skills/verify/features/transcripts.md +++ b/.claude/skills/verify/features/transcripts.md @@ -4,8 +4,9 @@ A user transcribes the current episode. When its feed publishes a transcript (``), PodNotes writes it as timestamped paragraphs with no API key and no call to OpenAI or Deepgram. Otherwise, and always through `Transcribe current episode from audio`, it sends the audio to the configured -provider with the user's key. The note goes to `Transcript path`, filled -from `Transcript template`, and opens. +provider with the user's key. That is Deepgram when `Speaker diarization` is on +with Deepgram as the provider, and OpenAI otherwise. The note goes to +`Transcript path`, filled from `Transcript template`, and opens. ## Sub-features @@ -14,6 +15,8 @@ from `Transcript template`, and opens. - `transcript-paid` keeps the audio path for an episode without a published transcript: the key notice without a key, the provider with one. - `transcript-failed` shows the fetch failure when a published transcript cannot be loaded, names the audio command when a key is set, and sends nothing to a provider. - `transcript-audio` transcribes the audio with the provider through `Transcribe current episode from audio`, even when a published transcript exists. +- `transcript-openai` sends OpenAI the audio as `gpt-transcribe` uploads of at most 24 MB and 2,070 s, one at a time, and writes paragraphs without timestamps or speaker labels. +- `transcript-deepgram` writes timestamped paragraphs with numbered speakers. Deepgram gets the address of an episode a saved public feed lists, and the audio itself for a private feed's episode, a local file, and an address that carries a credential or is not public. - `transcript-existing` refuses a second run while the note exists. ## How to get to it (user POV) @@ -35,6 +38,12 @@ Preconditions: - **Key.** A fake key is a precondition, not a user path: `CLI eval code='app.secretStorage.setSecret("podnotes-openai-api-key-424242", "sk-verify-fixture-not-real"), app.plugins.plugins.podnotes.settings.openAISecretId = "podnotes-openai-api-key-424242", app.plugins.plugins.podnotes.saveSettings(), "key set"'`. With it, `Untranscribed Episode` downloads its WAV and `$NET log` shows a `POST https://api.openai.com/v1/audio/transcriptions` answered by the fixture provider. - **Failed.** Play `Broken Transcript Episode` and run the command. The progress notice ends with `The published transcript could not be fetched: Network response returned HTTP 404.`, followed with a key by `Run "Transcribe current episode from audio" to transcribe it with OpenAI instead, which uses your API credit.` `$NET log` has the transcript GET (it records no status) and no provider call or `.wav`. - **Audio command.** On the same episode, run `UI key Ctrl+p`, `UI type .prompt-input 'Transcribe current episode from audio'` and `UI click .suggestion-item --text 'from audio'`. With a key, the WAV is fetched and OpenAI (the fixture provider) answers. +- **OpenAI in parts.** Write an MP3 longer than 2,070 s into the vault, for example 70 min in 4 MB: `ffmpeg -f lavfi -i sine=frequency=440:duration=4200 -ac 1 -ar 8000 -b:a 8k "$VAULT/Fixtures/Long Fixture.mp3"`. Play it ([local-files.md](./local-files.md)) and run the audio command. `$NET log` shows three `POST https://api.openai.com/v1/audio/transcriptions` entries of about 1.4 MB each, one after the other, and the note holds the fixture text three times with no timestamps. +- **Deepgram.** Set a fake Deepgram key as above, with the id `podnotes-deepgram-api-key-424242` and the setting `deepgramSecretId`. In Settings turn on `Speaker diarization` and pick `Deepgram` under `Diarization provider`. The fixture provider answers with two speakers, the second 12 s in, so the note reads `[00:00:00](...) **1:** This text came from the fixture provider.` and `[00:00:12](...) **2:** A second speaker answers.` + - Public. Play `Fixture Episode 1` of `PodNotes Fixture` and run the audio command. `$NET log` shows one `POST https://api.deepgram.com/v1/listen` with `contentType` `application/json` and under 100 `uploadBytes`, and no request for `/episode-1.wav` after the command. + - Private. `FEED=$(node .claude/skills/verify/fixtures.mjs private)` prints a feed at `priv3.supercast.com`. Add `PodNotes Private` and check that `data.json` saved it with a `urlSecretId`. Play its `Fixture Episode 1` and run the audio command. `$NET log` shows a `GET` for `/p/episode-1.wav`, then one `POST https://api.deepgram.com/v1/listen` with `contentType` `audio/wav` and the WAV's 960,044 `uploadBytes`. + - Local. Play `Fixtures/Local Fixture.wav` and run the audio command. The `POST` carries `audio/wav`, with no `GET` before it. +- **Speaker labels with OpenAI.** With `Speaker diarization` on and `OpenAI (no speaker labels)` as the provider, the progress notice ends with `Saved without speaker labels, because OpenAI does not label speakers. These providers do: Deepgram.` and stays until clicked. - **Proof.** Screenshot the opened note (`npm run capture:e2e-obsidian -- screenshot .obsidian-e2e-artifacts/transcripts/note.png`), keep the note text, `$NET log` and `fixtures.mjs requests` for every run, and read `data.json` to show no new field. ## Gotchas @@ -44,6 +53,6 @@ Preconditions: - `net.mjs` sees `window.fetch` and `requestUrl`. The player streams audio through the media element, which only `fixtures.mjs requests` shows. - A secret id must be `podnotes-openai-api-key` or `podnotes-openai-api-key-`. Any other id reads as missing on this device. -- Notices hide after about 5 s. Read them right after the command. They also cover the PodNotes view's head and the full player's collapse button, and `ui.mjs` refuses a covered click, so dismiss them first. +- The progress notice hides about 5 s after a transcript is saved. A failure, or a transcript saved with a warning, stays until clicked. Notices also cover the PodNotes view's head and the full player's collapse button, and `ui.mjs` refuses a covered click, so dismiss them first. - After editing `fixtures.mjs`, run `fixtures.mjs stop` before `fixtures.mjs transcripts`. A running server keeps serving the old routes. - Every fixture transcript cue falls inside the 60 s WAV, so its links seek within the audio. diff --git a/.claude/skills/verify/fixtures.mjs b/.claude/skills/verify/fixtures.mjs index c7f50477..230ddd22 100755 --- a/.claude/skills/verify/fixtures.mjs +++ b/.claude/skills/verify/fixtures.mjs @@ -5,6 +5,8 @@ // node .claude/skills/verify/fixtures.mjs feed start or reuse the feed server, print the feed URL // node .claude/skills/verify/fixtures.mjs feed-b same server, print a second feed whose episode titles repeat the first's // node .claude/skills/verify/fixtures.mjs transcripts same server, print a feed whose episodes publish transcripts +// node .claude/skills/verify/fixtures.mjs private same server, print a feed at an address PodNotes keeps +// private, as a Supercast subscriber's feed is // node .claude/skills/verify/fixtures.mjs library same server, print the URL of a JSON manifest of eight // podcasts as PodNotes saves them (Night Owl Science's feed // answers 404) and one more feed to add. Each episode's @@ -26,7 +28,7 @@ import { Readable, pipeline } from "node:stream"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { loadRunnerConfig, resolveProvisionOptions } from "obsidian-e2e/runner"; -import { FIXTURE_HOST } from "../../../scripts/e2e-instance.mjs"; +import { FIXTURE_HOST, SUPERCAST_FIXTURE_HOST } from "../../../scripts/e2e-instance.mjs"; const root = fileURLToPath(new URL("../../..", import.meta.url)); // Stable per worktree, so a feed saved in the vault keeps working across restarts. @@ -36,11 +38,14 @@ const base = `http://${host}:${port}`; const feedUrl = `${base}/feed.xml`; const feedBUrl = `${base}/feed-b.xml`; const transcriptsUrl = `${base}/transcripts.xml`; +const PRIVATE_ROUTE = "/feeds/verify-subscriber-token/feed.xml"; +const privateUrl = `http://${SUPERCAST_FIXTURE_HOST}:${port}${PRIVATE_ROUTE}`; const libraryUrl = `${base}/library.json`; const local = `http://127.0.0.1:${port}`; const FEED_TITLE = "PodNotes Fixture"; const FEED_B_TITLE = "PodNotes Fixture B"; const TRANSCRIPTS_TITLE = "PodNotes Transcripts"; +const PRIVATE_TITLE = "PodNotes Private"; const transcriptFixtures = path.join(root, "tests/fixtures/transcripts"); // Media made here outlives the server; node_modules/.cache is where tools keep caches. const CACHE = path.join(root, "node_modules/.cache/podnotes-fixtures"); @@ -548,6 +553,13 @@ function serve() { "/b/artwork.svg": ["image/svg+xml", Buffer.from(artwork(FEED_B_TITLE, "#0f766e"))], "/b/episode-1.wav": ["audio/wav", audio], "/b/episode-2.wav": ["audio/wav", audio], + [PRIVATE_ROUTE]: [ + "application/rss+xml", + Buffer.from(feedXml(audio.length, PRIVATE_TITLE, "/p")), + ], + "/p/artwork.svg": ["image/svg+xml", Buffer.from(artwork(PRIVATE_TITLE, "#be123c"))], + "/p/episode-1.wav": ["audio/wav", audio], + "/p/episode-2.wav": ["audio/wav", audio], "/transcripts.xml": ["application/rss+xml", Buffer.from(transcriptsFeedXml(audio.length))], "/t/artwork.svg": ["image/svg+xml", Buffer.from(artwork("Transcripts", "#b45309"))], "/t/buzzcast.html": ["text/html; charset=utf-8", transcript("buzzcast.html")], @@ -748,6 +760,7 @@ try { else if (verb === "feed") await startFeed(feedUrl); else if (verb === "feed-b") await startFeed(feedBUrl); else if (verb === "transcripts") await startFeed(transcriptsUrl); + else if (verb === "private") await startFeed(privateUrl); else if (verb === "library") await startFeed(libraryUrl); else if (verb === "retitle") await retitleFeed(); else if (verb === "requests") await printRequests(); @@ -755,7 +768,7 @@ try { else if (verb === "stop") await stopFeed(); else throw new Error( - "Usage: fixtures.mjs feed | feed-b | transcripts | library | retitle | requests | audio [vault-relative path] | stop", + "Usage: fixtures.mjs feed | feed-b | transcripts | private | library | retitle | requests | audio [vault-relative path] | stop", ); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); diff --git a/.claude/skills/verify/net.mjs b/.claude/skills/verify/net.mjs index d4ba0686..5b516dc9 100644 --- a/.claude/skills/verify/net.mjs +++ b/.claude/skills/verify/net.mjs @@ -32,6 +32,7 @@ const instance = resolveInstanceOptions( ); const CALL_TIMEOUT_MS = 15000; const FIXTURE_TRANSCRIPTION = "This text came from the fixture provider."; +const FIXTURE_REPLY = "A second speaker answers."; const PATH_HOSTS = [FIXTURE_HOST, "api.openai.com", "api.deepgram.com"]; /** @param {string} code */ @@ -75,6 +76,15 @@ const WATCH = `(() => { } }; const text = ${jsLiteral(FIXTURE_TRANSCRIPTION)}; + // Deepgram's words: one every 0.4 s, the reply 12 s in from a second speaker. + const words = (sentence, speaker, from) => sentence.split(" ").map((word, index) => ({ + word: word.toLowerCase().replace(/[^a-z]/g, ""), + punctuated_word: word, + start: from + index * 0.4, + end: from + index * 0.4 + 0.3, + speaker, + })); + const deepgram = { results: { channels: [{ alternatives: [{ words: [...words(text, 0, 0), ...words(${jsLiteral(FIXTURE_REPLY)}, 1, 12)] }] }] } }; const originalFetch = window.fetch; window.fetch = async function (input, init) { const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; @@ -88,12 +98,12 @@ const WATCH = `(() => { if (channel === "request-url" && typeof request?.url === "string") { const method = (request.method ?? "GET").toUpperCase(); const answer = request.url.startsWith("https://api.openai.com/") - ? { text, segments: [{ speaker: "A", text, start: 0, end: 1 }] } + ? { text } : request.url.startsWith("https://api.deepgram.com/") - ? { results: { utterances: [{ speaker: 0, transcript: text, start: 0, end: 1 }] } } + ? deepgram : null; if (answer) { - entries.push({ via: "requestUrl", method, url: logged(request.url), answeredBy: "fixture provider", uploadBytes: request.body?.byteLength ?? 0 }); + entries.push({ via: "requestUrl", method, url: logged(request.url), answeredBy: "fixture provider", contentType: request.contentType, uploadBytes: request.body?.byteLength ?? 0 }); const body = new TextEncoder().encode(JSON.stringify(answer)).buffer; queueMicrotask(() => ipc.emit(replyChannel, {}, { status: 200, headers: { "content-type": "application/json" }, body })); return; From 989185fc6e585a5bf023b9e7673ea87277ab681d Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 22:43:12 +0100 Subject: [PATCH 10/26] test(e2e): read the abort rejection through waitForLifecycle The marketplace review test called TranscriptionService.waitForRetry, which the engine replaced. An aborted run now rejects through waitForLifecycle, so the test aborts a pending wait there and expects the same Error. --- tests/e2e/marketplace-review-runtime.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/marketplace-review-runtime.test.ts b/tests/e2e/marketplace-review-runtime.test.ts index effa3998..c993fb89 100644 --- a/tests/e2e/marketplace-review-runtime.test.ts +++ b/tests/e2e/marketplace-review-runtime.test.ts @@ -382,7 +382,7 @@ describe("marketplace review runtime paths", () => { const { obsidian } = getContext(); const result = await evalJsonAsync<{ lifecycle: { isError: boolean; message: string }; - retryAbort: { isError: boolean; message: string }; + abort: { isError: boolean; message: string }; save: { isError: boolean; message: string }; }>( obsidian, @@ -410,23 +410,23 @@ describe("marketplace review runtime paths", () => { lifecycle = { isError: error instanceof Error, message: error?.message ?? String(error) }; } - const retryPromise = service.waitForRetry(10_000); + const pending = service.waitForLifecycle(new Promise(() => {})); service.lifetimeAbortController.abort("raw abort reason"); - let retryAbort; + let abort; try { - await retryPromise; - retryAbort = { isError: false, message: "resolved" }; + await pending; + abort = { isError: false, message: "resolved" }; } catch (error) { - retryAbort = { isError: error instanceof Error, message: error?.message ?? String(error) }; + abort = { isError: error instanceof Error, message: error?.message ?? String(error) }; } - return { lifecycle, retryAbort, save }; + return { abort, lifecycle, save }; })()`, ); expect(result).toEqual({ lifecycle: { isError: true, message: "PodNotes transcription failed." }, - retryAbort: { isError: true, message: "PodNotes transcription was aborted." }, + abort: { isError: true, message: "PodNotes transcription was aborted." }, save: { isError: true, message: "PodNotes could not save its settings." }, }); expect(await obsidian.dev.runtimeErrors()).toEqual([]); From 7d46678737b2478cf18d7cd80f9fbbd50ba2998e Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Fri, 13 Nov 2026 23:25:39 +0100 Subject: [PATCH 11/26] fix(transcription): stop sooner after an abort or a long Retry-After An aborted job no longer downloads the episode or decodes it into parts. Until now only its requests were refused. A 429 or a 5xx that asks to wait over a minute is sent once. A sooner send would only upload the audio to be refused again. --- src/services/TranscriptionService.test.ts | 80 +++++++++++++++++++++-- src/services/transcription/http.ts | 12 ++-- src/services/transcription/transcribe.ts | 6 +- 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 53ddb9f4..7e18b0cd 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -113,6 +113,12 @@ function savedFeedRepository( return feeds; } +const publicFeed: PodcastFeed = { + title: "Test Podcast", + url: "https://example.com/feed.xml", + artworkUrl: "", +}; + /** An hour of MP3 in 3.2 MB, which OpenAI takes in two parts of 26 min 48 s. */ const twoPartStream = MP3_STREAMS.mpeg25At8Kbps; const twoPartFrames = 44_668; @@ -224,6 +230,68 @@ describe("TranscriptionService", () => { expect(providerMock).not.toHaveBeenCalled(); }); + test("decodes no audio that arrives after unload", async () => { + const audioRequest = deferred<{ buffer: ArrayBuffer; extension: string }>(); + getEpisodeAudioBufferMock.mockReturnValue(audioRequest.promise); + // A count, because a failed assertion on the call would print its 24 MB argument. + let decodes = 0; + vi.stubGlobal( + "AudioContext", + class { + async decodeAudioData(): Promise { + decodes++; + throw new Error("Unsupported codec"); + } + async close(): Promise {} + }, + ); + const service = new TranscriptionService(createMockPlugin()); + const pending = ( + service as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + } + ).transcribeEpisode(audioJob); + await vi.waitFor(() => expect(getEpisodeAudioBufferMock).toHaveBeenCalledOnce()); + + service.dispose(); + await pending; + // Over OpenAI's size cap, so it would be decoded into WAV parts. + audioRequest.resolve({ buffer: new ArrayBuffer(24_000_001), extension: "ogg" }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(decodes).toBe(0); + }); + + test("downloads no audio when unloaded while it looks the episode up in its feed", async () => { + const lookup = deferred>(); + const load = vi.fn(() => lookup.promise); + const feeds = new FeedRepository({ + load, + resolveUrl: ({ url }) => url, + storage: new MemoryStorage(), + now: Date.now, + }); + feeds.setSubscriptions({ [publicFeed.title]: publicFeed }); + const service = new TranscriptionService( + createMockPlugin({ speakers: "deepgram", deepgramKey: "deepgram-key", feeds }), + ); + const pending = ( + service as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + } + ).transcribeEpisode(audioJob); + await vi.waitFor(() => expect(load).toHaveBeenCalledOnce()); + + service.dispose(); + await pending; + // The feed does not list the episode, so its audio would be uploaded. + lookup.resolve(feedDocument(publicFeed.title, [])); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + expect(providerMock).not.toHaveBeenCalled(); + }); + test.each([ ["OpenAI", {}, { text: "This must not be saved." }], [ @@ -557,6 +625,12 @@ describe("TranscriptionService", () => { () => Promise.reject(new TypeError("Failed to fetch")), "Could not reach OpenAI. Check your connection.", ], + [ + "429 that asks to wait over a minute", + 1, + async () => rateLimited({ "retry-after": "61" }), + "OpenAI is limiting how fast your key may be used. Try again in a few minutes.", + ], ["timeout", 1, neverAnswers, "OpenAI did not answer within 10 minutes."], ])("%s saves no note after %i of 3 sends", async (_name, sends, respond, shown) => { providerMock.mockImplementation(respond); @@ -571,7 +645,6 @@ describe("TranscriptionService", () => { test.each<[name: string, failure: () => Response, waitMs: number]>([ ["429 with Retry-After", () => rateLimited({ "retry-after": "4" }), 4000], ["429 with retry-after-ms", () => rateLimited({ "retry-after-ms": "1500" }), 1500], - ["429 asking for over a minute", () => rateLimited({ "retry-after": "61" }), 1000], ["503", () => new Response("", { status: 503 }), 1000], ])("%s is sent again after the wait and the note is saved", async (_name, fail, waitMs) => { vi.useFakeTimers(); @@ -673,11 +746,6 @@ describe("TranscriptionService", () => { }); describe("a Deepgram transcript", () => { - const publicFeed: PodcastFeed = { - title: "Test Podcast", - url: "https://example.com/feed.xml", - artworkUrl: "", - }; const audio = new Uint8Array(2048).map((_, index) => (index * 31) % 251); const useDeepgram = (feeds?: ReturnType) => { getEpisodeAudioBufferMock.mockResolvedValue({ diff --git a/src/services/transcription/http.ts b/src/services/transcription/http.ts index 885973a4..42ec7c2f 100644 --- a/src/services/transcription/http.ts +++ b/src/services/transcription/http.ts @@ -82,8 +82,9 @@ function kindOf(status: number): FailureKind { /** * A request goes out at most three times, and only again after a failure another send can * get past. A timeout is not one: Obsidian's requestUrl cannot cancel, so the provider may - * still be working, and a second send would be billed too. requestUrl is used because - * providers do not allow Obsidian's origin, which rules out `fetch`. + * still be working, and a second send would be billed too. Neither is an answer that asks + * to wait over a minute, because a sooner send would only upload the audio to be refused. + * requestUrl is used because providers do not allow Obsidian's origin, which rules out `fetch`. */ export function createSender( readFailure: Provider["failure"], @@ -134,7 +135,8 @@ export function createSender( const again = error instanceof TranscriptionError && RETRIED.has(error.kind) && - sends < MAX_SENDS; + sends < MAX_SENDS && + (error.retryAfterMs ?? 0) <= MAX_SERVER_RETRY_DELAY_MS; if (!again) throw error; await wait(error.retryAfterMs ?? 1000 * sends, signal); } @@ -161,7 +163,7 @@ async function encode({ }; } -/** The server's `Retry-After`, when it asks for a minute or less. */ +/** How long the server's `Retry-After` asks to wait. */ function serverRetryDelayMs(headers: Headers): number | undefined { let delayMs = parseFloat(headers.get("retry-after-ms") ?? ""); const retryAfter = headers.get("retry-after"); @@ -169,7 +171,7 @@ function serverRetryDelayMs(headers: Headers): number | undefined { const seconds = parseFloat(retryAfter); delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000; } - return delayMs >= 0 && delayMs <= MAX_SERVER_RETRY_DELAY_MS ? delayMs : undefined; + return delayMs >= 0 ? delayMs : undefined; } function wait(delayMs: number, signal: AbortSignal): Promise { diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts index a4b4d068..750a4937 100644 --- a/src/services/transcription/transcribe.ts +++ b/src/services/transcription/transcribe.ts @@ -62,8 +62,12 @@ export async function transcribe(job: TranscriptionJob): Promise { } if (parts.length === 0) { + // An aborted job downloads and decodes nothing more. The sender refuses its requests. + if (signal.aborted) throw signal.reason; onProgress("Fetching episode audio..."); - const files = await splitAudio(await getEpisodeAudioBuffer(episode), { + const audio = await getEpisodeAudioBuffer(episode); + if (signal.aborted) throw signal.reason; + const files = await splitAudio(audio, { maxBytes: model.maxUploadBytes, maxSeconds: model.maxSeconds, }); From df5ea095ee5f7c5203181e7bef13d3bd0c228e8f Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 15:56:07 +0100 Subject: [PATCH 12/26] fix(transcription): cut MP3 parts without a one-frame tail or an uncounted last frame Parts were counted from rounded seconds, so an episode a frame over a whole number of capped parts got equal parts one frame too long, and the time cap then split off a final part of a single frame. Parts are now counted in frames. A last frame followed by metadata, such as an ID3v1 tag, failed the next-frame check and was not counted, so the last part could run one frame over maxSeconds. A frame that continues the frames before it now counts. --- src/services/transcription/split.test.ts | 31 ++++++++++++++++++++++++ src/services/transcription/split.ts | 18 ++++++++++---- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index 64c8675b..bd1ceccd 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -108,6 +108,37 @@ describe("an MP3 over a cap", () => { expect(parts[1].start).toBeCloseTo(18_000 * stream.frameSeconds, 6); expect(joinsInto(parts, episode)).toBe(true); }); + + test("an MP3 a frame over a whole number of capped parts is cut into equal parts, not a one-frame tail", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + // 1,400 s holds 19,444 frames of 72 ms, so 58,333 frames need 4 parts. + const episode = mp3Bytes(stream, 58_333); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ + 14_584, 14_584, 14_584, 14_581, + ]); + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("counts the last frame before trailing metadata, so the last part stays under maxSeconds", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + // An ID3v1 tag after the last frame. A 72.01 s part holds 1,000 frames, so 2,001 need three parts. + const tag = new Uint8Array(128); + tag.set([0x54, 0x41, 0x47]); + const episode = Buffer.concat([mp3Bytes(stream, 2001), tag]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 72.01 }); + + const frames = parts.map( + (part, index) => + (part.bytes.length - (index === parts.length - 1 ? tag.length : 0)) / + stream.frameBytes, + ); + expect(frames).toEqual([667, 667, 667]); + expect(joinsInto(parts, episode)).toBe(true); + }); }); describe("audio over the size cap that cannot be cut where it is", () => { diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index f11ff12f..ba25f2e7 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -101,8 +101,13 @@ function mp3PartStarts( maxBytes: number, maxSeconds: number, ): { offset: number; seconds: number }[] { - const total = frames.reduce((seconds, frame) => seconds + frame.seconds, 0); - const parts = Math.max(Math.ceil(byteLength / maxBytes), Math.ceil(total / maxSeconds)); + // Frames in one stream last equally long, so counting frames keeps every part under + // maxSeconds without a one-frame tail from rounding. + const framesPerCap = frames.length > 0 ? Math.floor(maxSeconds / frames[0].seconds) : Infinity; + const parts = Math.max( + Math.ceil(byteLength / maxBytes), + Math.ceil(frames.length / framesPerCap), + ); const framesPerPart = Math.ceil(frames.length / parts); const starts = [{ offset: 0, seconds: 0 }]; let elapsed = 0; @@ -126,8 +131,9 @@ function mp3PartStarts( } /** - * The Layer III frames in `bytes`. A frame counts only when another frame or the - * end of the buffer follows it, so sync-like bytes in a tag or cover art don't. + * The Layer III frames in `bytes`. A frame counts when it continues the frames before it, + * or when another frame or the end of the buffer follows it, so sync-like bytes in a tag + * or cover art don't, and the last frame before trailing metadata does. */ function mp3Frames(bytes: Uint8Array): Mp3Frame[] { const frames: Mp3Frame[] = []; @@ -135,7 +141,9 @@ function mp3Frames(bytes: Uint8Array): Mp3Frame[] { while (offset + 4 <= bytes.length) { const frame = mp3FrameAt(bytes, offset); const next = frame ? offset + frame.length : 0; - if (frame && (next === bytes.length || mp3FrameAt(bytes, next))) { + const last = frames[frames.length - 1]; + const continues = last !== undefined && last.offset + last.length === offset; + if (frame && (continues || next === bytes.length || mp3FrameAt(bytes, next))) { frames.push(frame); offset = next; } else { From 3a14c53a341f248cae32244f6128dc98e8dabc9b Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 16:12:56 +0100 Subject: [PATCH 13/26] fix(transcription): drop the summed-seconds cap that split off one-frame parts Summing frame durations drifts past the cap: 25,000 frames of 24 ms sum to 600.0000000000218 s, so the per-frame seconds check cut each part a frame early and a 49,999-frame episode went as 24,999 + 24,999 + 1 frames. Parts are already counted in frames, which keeps them under maxSeconds, so the check goes. --- src/services/transcription/split.test.ts | 13 +++++++++++++ src/services/transcription/split.ts | 13 +++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index bd1ceccd..6dcb7cfb 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -122,6 +122,19 @@ describe("an MP3 over a cap", () => { expect(joinsInto(parts, episode)).toBe(true); }); + test("an MP3 one frame short of two full parts splits into two, with no one-frame tail", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // 600 s holds 25,000 frames of 24 ms. Summing 0.024 s 25,000 times overshoots 600. + const episode = mp3Bytes(stream, 49_999); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ + 25_000, 24_999, + ]); + expect(joinsInto(parts, episode)).toBe(true); + }); + test("counts the last frame before trailing metadata, so the last part stays under maxSeconds", async () => { const stream = MP3_STREAMS.mpeg25At8Kbps; // An ID3v1 tag after the last frame. A 72.01 s part holds 1,000 frames, so 2,001 need three parts. diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index ba25f2e7..71c8e699 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -101,9 +101,10 @@ function mp3PartStarts( maxBytes: number, maxSeconds: number, ): { offset: number; seconds: number }[] { - // Frames in one stream last equally long, so counting frames keeps every part under - // maxSeconds without a one-frame tail from rounding. - const framesPerCap = frames.length > 0 ? Math.floor(maxSeconds / frames[0].seconds) : Infinity; + // Frames in one stream last equally long, so parts are counted in frames: summing frame + // durations drifts past maxSeconds and would split off a one-frame tail. + const framesPerCap = + frames.length > 0 ? Math.floor(maxSeconds / frames[0].seconds + 1e-9) : Infinity; const parts = Math.max( Math.ceil(byteLength / maxBytes), Math.ceil(frames.length / framesPerCap), @@ -112,19 +113,15 @@ function mp3PartStarts( const starts = [{ offset: 0, seconds: 0 }]; let elapsed = 0; let partFrames = 0; - let partSeconds = 0; for (const frame of frames) { if ( partFrames === framesPerPart || - frame.offset + frame.length - starts[starts.length - 1].offset > maxBytes || - partSeconds + frame.seconds > maxSeconds + frame.offset + frame.length - starts[starts.length - 1].offset > maxBytes ) { starts.push({ offset: frame.offset, seconds: elapsed }); partFrames = 0; - partSeconds = 0; } partFrames++; - partSeconds += frame.seconds; elapsed += frame.seconds; } return starts; From 27d89588e1639e05ca05e5c5b19b5e63e3c16494 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 16:30:19 +0100 Subject: [PATCH 14/26] fix(transcription): split MP3 parts equally even when the size cap binds The planner fixed a frame count per part and checked the size cap separately. When a leading tag or a variable bitrate put an equal split over the cap, the size check cut a part short and the next kept the old frame count, leaving a one-frame part that cannot be decoded: 43,690 frames of a 320 kbps MP3 with a 512-byte tag went as 21,844 + 21,845 + 1. The planner now takes the fewest equal splits that fit both caps, adding a part when an equal split does not fit. --- src/services/transcription/split.test.ts | 21 ++++++++++- src/services/transcription/split.ts | 47 +++++++++++++----------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index 6dcb7cfb..f9413986 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -117,11 +117,30 @@ describe("an MP3 over a cap", () => { const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); expect(parts.map((part) => part.bytes.length / stream.frameBytes)).toEqual([ - 14_584, 14_584, 14_584, 14_581, + 14_584, 14_583, 14_583, 14_583, ]); expect(joinsInto(parts, episode)).toBe(true); }); + test("an MP3 that a leading tag pushes over the size cap splits into equal parts, with no one-frame tail", async () => { + const stream = MP3_STREAMS.mpeg1At320Kbps; + // 512 bytes of tag and 43,690 frames of 960 bytes come to 41,942,912 bytes, just under + // twice 20 MiB, but an equal split into two puts the tag's 512 bytes over the cap. + const tag = new Uint8Array(512); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x03, 0x76]); + const episode = Buffer.concat([tag, mp3Bytes(stream, 43_690)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect( + parts.map( + (part, index) => (part.bytes.length - (index === 0 ? 512 : 0)) / stream.frameBytes, + ), + ).toEqual([14_564, 14_563, 14_563]); + for (const part of parts) expect(part.bytes.length).toBeLessThanOrEqual(20 * MIB); + expect(joinsInto(parts, episode)).toBe(true); + }); + test("an MP3 one frame short of two full parts splits into two, with no one-frame tail", async () => { const stream = MP3_STREAMS.mpeg1At96Kbps; // 600 s holds 25,000 frames of 24 ms. Summing 0.024 s 25,000 times overshoots 600. diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index 71c8e699..1bc1daad 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -102,29 +102,34 @@ function mp3PartStarts( maxSeconds: number, ): { offset: number; seconds: number }[] { // Frames in one stream last equally long, so parts are counted in frames: summing frame - // durations drifts past maxSeconds and would split off a one-frame tail. - const framesPerCap = - frames.length > 0 ? Math.floor(maxSeconds / frames[0].seconds + 1e-9) : Infinity; - const parts = Math.max( - Math.ceil(byteLength / maxBytes), - Math.ceil(frames.length / framesPerCap), - ); - const framesPerPart = Math.ceil(frames.length / parts); - const starts = [{ offset: 0, seconds: 0 }]; - let elapsed = 0; - let partFrames = 0; - for (const frame of frames) { - if ( - partFrames === framesPerPart || - frame.offset + frame.length - starts[starts.length - 1].offset > maxBytes - ) { - starts.push({ offset: frame.offset, seconds: elapsed }); - partFrames = 0; + // durations drifts past maxSeconds. + const frameSeconds = frames[0].seconds; + const framesPerCap = Math.floor(maxSeconds / frameSeconds + 1e-9); + // The fewest parts of equal frame counts that fit both caps. A leading tag or a variable + // bitrate can put an equal split over maxBytes, and then one more part is tried, so no + // part is ever cut short and no one-frame tail is left. + for ( + let parts = Math.max( + Math.ceil(byteLength / maxBytes), + Math.ceil(frames.length / framesPerCap), + ); + ; + parts++ + ) { + const firsts = Array.from({ length: parts }, (_, index) => + Math.ceil((index * frames.length) / parts), + ); + const offsets = firsts.map((first, index) => (index === 0 ? 0 : frames[first].offset)); + const fits = offsets.every( + (offset, index) => (offsets[index + 1] ?? byteLength) - offset <= maxBytes, + ); + if (fits || parts >= frames.length) { + return firsts.map((first, index) => ({ + offset: offsets[index], + seconds: first * frameSeconds, + })); } - partFrames++; - elapsed += frame.seconds; } - return starts; } /** From da3baf9440c450322eba76c247b682313c91820c Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 16:44:28 +0100 Subject: [PATCH 15/26] fix(transcription): fill MP3 parts up to the size cap before adding one Equal splits put the odd frame first, so a 512-byte tag and 43,689 frames of a 320 kbps MP3 put the first of two parts 192 bytes over 20 MiB, and the planner sent three where two fit. Each cut now takes an equal share of the frames still to place and steps back while the part is over the size cap, so 21,844 + 21,845 goes in two. --- src/services/transcription/split.test.ts | 19 +++++++++++++++ src/services/transcription/split.ts | 31 ++++++++++++++---------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index f9413986..2a696012 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -122,6 +122,25 @@ describe("an MP3 over a cap", () => { expect(joinsInto(parts, episode)).toBe(true); }); + test("an MP3 whose equal split misses the size cap by its odd frame still goes in two parts", async () => { + const stream = MP3_STREAMS.mpeg1At320Kbps; + // 512 bytes of tag and 43,689 frames: 21,845 + 21,844 puts the first part 192 bytes + // over 20 MiB, but 21,844 + 21,845 fits. + const tag = new Uint8Array(512); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x03, 0x76]); + const episode = Buffer.concat([tag, mp3Bytes(stream, 43_689)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect( + parts.map( + (part, index) => (part.bytes.length - (index === 0 ? 512 : 0)) / stream.frameBytes, + ), + ).toEqual([21_844, 21_845]); + for (const part of parts) expect(part.bytes.length).toBeLessThanOrEqual(20 * MIB); + expect(joinsInto(parts, episode)).toBe(true); + }); + test("an MP3 that a leading tag pushes over the size cap splits into equal parts, with no one-frame tail", async () => { const stream = MP3_STREAMS.mpeg1At320Kbps; // 512 bytes of tag and 43,690 frames of 960 bytes come to 41,942,912 bytes, just under diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index 1bc1daad..40531f23 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -105,9 +105,10 @@ function mp3PartStarts( // durations drifts past maxSeconds. const frameSeconds = frames[0].seconds; const framesPerCap = Math.floor(maxSeconds / frameSeconds + 1e-9); - // The fewest parts of equal frame counts that fit both caps. A leading tag or a variable - // bitrate can put an equal split over maxBytes, and then one more part is tried, so no - // part is ever cut short and no one-frame tail is left. + const offsetOf = (first: number) => (first === 0 ? 0 : frames[first].offset); + // The fewest parts that fit both caps, each taking an equal share of the frames still to + // place and stepping back a frame at a time while it is over maxBytes, so parts differ + // by a frame or so and none is a one-frame tail. for ( let parts = Math.max( Math.ceil(byteLength / maxBytes), @@ -116,16 +117,20 @@ function mp3PartStarts( ; parts++ ) { - const firsts = Array.from({ length: parts }, (_, index) => - Math.ceil((index * frames.length) / parts), - ); - const offsets = firsts.map((first, index) => (index === 0 ? 0 : frames[first].offset)); - const fits = offsets.every( - (offset, index) => (offsets[index + 1] ?? byteLength) - offset <= maxBytes, - ); - if (fits || parts >= frames.length) { - return firsts.map((first, index) => ({ - offset: offsets[index], + const firsts = [0]; + for (let left = parts - 1; left > 0; left--) { + const first = firsts[firsts.length - 1]; + const share = Math.ceil((frames.length - first) / (left + 1)); + let next = Math.min(first + Math.min(share, framesPerCap), frames.length - left); + while (next > first + 1 && frames[next].offset - offsetOf(first) > maxBytes) next--; + firsts.push(next); + } + const last = firsts[firsts.length - 1]; + const lastFits = + frames.length - last <= framesPerCap && byteLength - offsetOf(last) <= maxBytes; + if (lastFits || parts >= frames.length) { + return firsts.map((first) => ({ + offset: offsetOf(first), seconds: first * frameSeconds, })); } From 25470c31c608778b128a6a5d580511adb9f1b71f Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 17:05:24 +0100 Subject: [PATCH 16/26] fix(transcription): cut MP3 parts as few as fit, moving cuts either way Cuts only moved back from an even split, so where the bitrate rose a part was added although moving the cut forward fit: 21,847 low-bitrate frames then 21,845 at 320 kbps and an ID3v1 tag went as three parts, not two. The planner now finds the fewest parts by filling each as far as it fits, then places each cut as close to an equal share as the parts before and after it allow. A property test checks the count against a brute-force minimum on 300 random variable-bitrate streams. --- src/services/transcription/split.test.ts | 75 ++++++++++++++++++++++++ src/services/transcription/split.ts | 71 ++++++++++++---------- 2 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index 2a696012..b41cc837 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -160,6 +160,81 @@ describe("an MP3 over a cap", () => { expect(joinsInto(parts, episode)).toBe(true); }); + test("an MP3 whose bitrate rises moves the cut forward rather than adding a part", async () => { + const low = MP3_STREAMS.mpeg1At96Kbps; + const high = MP3_STREAMS.mpeg1At320Kbps; + // 21,847 low-bitrate frames, then 21,845 high-bitrate frames and an ID3v1 tag. An even + // cut leaves 288 + 20,971,200 + 128 bytes after it, over 20 MiB; one frame later fits. + const tag = new Uint8Array(128); + tag.set([0x54, 0x41, 0x47]); + const episode = Buffer.concat([mp3Bytes(low, 21_847), mp3Bytes(high, 21_845), tag]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 1400 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + 21_847 * low.frameBytes, + 21_845 * high.frameBytes + tag.length, + ]); + expect(joinsInto(parts, episode)).toBe(true); + }); + + test("cuts every MP3 into the fewest parts that fit both caps", async () => { + // A seeded generator, so a failure names a reproducible case. + let seed = 532; + const random = () => { + seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648; + return seed / 2_147_483_648; + }; + const kinds = [MP3_STREAMS.mpeg1At96Kbps, MP3_STREAMS.mpeg1At320Kbps]; + for (let trial = 0; trial < 300; trial++) { + const kindsUsed: (typeof kinds)[number][] = []; + while (kindsUsed.length < 20 + Math.floor(random() * 100)) { + const kind = kinds[Math.floor(random() * 2)]; + for (let run = 1 + Math.floor(random() * 12); run > 0; run--) kindsUsed.push(kind); + } + const lead = random() < 0.5 ? 512 : 0; + const tail = new Uint8Array(random() < 0.5 ? 128 : 0); + if (tail.length) tail.set([0x54, 0x41, 0x47]); + const bytes = new Uint8Array( + Buffer.concat([ + new Uint8Array(lead), + ...kindsUsed.map((kind) => mp3Bytes(kind, 1)), + tail, + ]), + ); + const maxBytes = 2_000 + Math.floor(random() * 20_000); + const framesPerCap = 3 + Math.floor(random() * 40); + + // Brute force: the fewest parts, where a part is a run of frames within both caps. + const offsets = [lead]; + for (const kind of kindsUsed) + offsets.push(offsets[offsets.length - 1] + kind.frameBytes); + const startOf = (index: number) => (index === 0 ? 0 : offsets[index]); + const endOf = (index: number) => + index === kindsUsed.length ? bytes.length : offsets[index]; + const fewest: number[] = [0]; + for (let end = 1; end <= kindsUsed.length; end++) { + fewest[end] = Infinity; + for (let first = Math.max(0, end - framesPerCap); first < end; first++) { + if (endOf(end) - startOf(first) <= maxBytes) { + fewest[end] = Math.min(fewest[end], fewest[first] + 1); + } + } + } + if (fewest[kindsUsed.length] === Infinity) continue; + + const parts = await split(bytes, { maxBytes, maxSeconds: framesPerCap * 0.024 }); + + const context = `trial ${trial}`; + expect(parts.length, context).toBe(fewest[kindsUsed.length]); + for (const part of parts) { + expect(part.bytes.length, context).toBeGreaterThan(0); + expect(part.bytes.length, context).toBeLessThanOrEqual(maxBytes); + } + expect(joinsInto(parts, bytes), context).toBe(true); + } + }); + test("an MP3 one frame short of two full parts splits into two, with no one-frame tail", async () => { const stream = MP3_STREAMS.mpeg1At96Kbps; // 600 s holds 25,000 frames of 24 ms. Summing 0.024 s 25,000 times overshoots 600. diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index 40531f23..ab28f0a6 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -91,9 +91,10 @@ const SAMPLE_RATES = [ ]; /** - * Where each part of an MP3 starts, in bytes and in seconds of audio. Parts start at frame - * starts, there are as few as both caps allow, and they hold an equal number of frames, so - * no part is a few seconds long. The first part starts at byte 0 and keeps a leading tag. + * Where each part of an MP3 starts, in bytes and in seconds of audio: at frame starts, in as + * few parts as both caps allow, cut as close to equal frame counts as the size cap lets + * them be, so no part is a one-frame tail that cannot be decoded. The first part starts at + * byte 0 and keeps a leading tag. */ function mp3PartStarts( frames: Mp3Frame[], @@ -101,40 +102,46 @@ function mp3PartStarts( maxBytes: number, maxSeconds: number, ): { offset: number; seconds: number }[] { + const count = frames.length; // Frames in one stream last equally long, so parts are counted in frames: summing frame // durations drifts past maxSeconds. const frameSeconds = frames[0].seconds; const framesPerCap = Math.floor(maxSeconds / frameSeconds + 1e-9); - const offsetOf = (first: number) => (first === 0 ? 0 : frames[first].offset); - // The fewest parts that fit both caps, each taking an equal share of the frames still to - // place and stepping back a frame at a time while it is over maxBytes, so parts differ - // by a frame or so and none is a one-frame tail. - for ( - let parts = Math.max( - Math.ceil(byteLength / maxBytes), - Math.ceil(frames.length / framesPerCap), - ); - ; - parts++ - ) { - const firsts = [0]; - for (let left = parts - 1; left > 0; left--) { - const first = firsts[firsts.length - 1]; - const share = Math.ceil((frames.length - first) / (left + 1)); - let next = Math.min(first + Math.min(share, framesPerCap), frames.length - left); - while (next > first + 1 && frames[next].offset - offsetOf(first) > maxBytes) next--; - firsts.push(next); - } - const last = firsts[firsts.length - 1]; - const lastFits = - frames.length - last <= framesPerCap && byteLength - offsetOf(last) <= maxBytes; - if (lastFits || parts >= frames.length) { - return firsts.map((first) => ({ - offset: offsetOf(first), - seconds: first * frameSeconds, - })); - } + const startOf = (first: number) => (first === 0 ? 0 : frames[first].offset); + const endOf = (end: number) => (end === count ? byteLength : frames[end].offset); + const fits = (first: number, end: number) => + end - first <= framesPerCap && endOf(end) - startOf(first) <= maxBytes; + /** The end of the longest part that starts at `first` and fits. */ + const furthest = (first: number) => { + let end = Math.min(count, first + framesPerCap); + while (end > first + 1 && !fits(first, end)) end--; + return end; + }; + /** The start of the longest part that ends at `end` and fits. */ + const earliest = (end: number) => { + let first = Math.max(0, end - framesPerCap); + while (first < end - 1 && !fits(first, end)) first++; + return first; + }; + // Filling each part as far as it fits gives the fewest parts. + let parts = 0; + for (let first = 0; first < count; first = furthest(first)) parts++; + // lowest[index] is the earliest frame part `index` may start at and still leave the parts + // after it able to reach the end. + const lowest: number[] = []; + for (let index = parts - 1, end = count; index > 0; index--) { + end = earliest(end); + lowest[index] = end; + } + // Each cut aims at an equal share of the frames still to place, within what keeps this + // part and the ones after it fitting. + const firsts = [0]; + for (let index = 1; index < parts; index++) { + const first = firsts[index - 1]; + const share = first + Math.ceil((count - first) / (parts - index + 1)); + firsts.push(Math.min(Math.max(share, lowest[index]), furthest(first))); } + return firsts.map((first) => ({ offset: startOf(first), seconds: first * frameSeconds })); } /** From 520f2ddfc4bab5a7952937fc1db9aa1bddc5b90b Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 17:23:27 +0100 Subject: [PATCH 17/26] fix(transcription): sum each MP3 frame's duration when cutting parts Frame-like bytes inside a leading tag set the first frame's duration, and counting every part in that frame's length sent a 1,728 s episode of 72 ms frames up in one request. Parts now fit by the summed duration of their own frames, and each part's start is the sum of the frames before it. --- src/services/transcription/split.test.ts | 75 ++++++++++++++++++------ src/services/transcription/split.ts | 37 +++++++----- 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index b41cc837..ba898a43 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -185,12 +185,16 @@ describe("an MP3 over a cap", () => { seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648; return seed / 2_147_483_648; }; - const kinds = [MP3_STREAMS.mpeg1At96Kbps, MP3_STREAMS.mpeg1At320Kbps]; + const kinds = [ + MP3_STREAMS.mpeg1At96Kbps, + MP3_STREAMS.mpeg1At320Kbps, + MP3_STREAMS.mpeg25At8Kbps, + ]; for (let trial = 0; trial < 300; trial++) { - const kindsUsed: (typeof kinds)[number][] = []; - while (kindsUsed.length < 20 + Math.floor(random() * 100)) { - const kind = kinds[Math.floor(random() * 2)]; - for (let run = 1 + Math.floor(random() * 12); run > 0; run--) kindsUsed.push(kind); + const frames: (typeof kinds)[number][] = []; + while (frames.length < 20 + Math.floor(random() * 100)) { + const kind = kinds[Math.floor(random() * kinds.length)]; + for (let run = 1 + Math.floor(random() * 12); run > 0; run--) frames.push(kind); } const lead = random() < 0.5 ? 512 : 0; const tail = new Uint8Array(random() < 0.5 ? 128 : 0); @@ -198,43 +202,76 @@ describe("an MP3 over a cap", () => { const bytes = new Uint8Array( Buffer.concat([ new Uint8Array(lead), - ...kindsUsed.map((kind) => mp3Bytes(kind, 1)), + ...frames.map((kind) => mp3Bytes(kind, 1)), tail, ]), ); const maxBytes = 2_000 + Math.floor(random() * 20_000); - const framesPerCap = 3 + Math.floor(random() * 40); + const maxSeconds = (3 + Math.floor(random() * 40)) * 0.024; // Brute force: the fewest parts, where a part is a run of frames within both caps. const offsets = [lead]; - for (const kind of kindsUsed) + const elapsed = [0]; + for (const kind of frames) { offsets.push(offsets[offsets.length - 1] + kind.frameBytes); + elapsed.push(elapsed[elapsed.length - 1] + kind.frameSeconds); + } const startOf = (index: number) => (index === 0 ? 0 : offsets[index]); const endOf = (index: number) => - index === kindsUsed.length ? bytes.length : offsets[index]; + index === frames.length ? bytes.length : offsets[index]; + const fits = (first: number, end: number) => + endOf(end) - startOf(first) <= maxBytes && + elapsed[end] - elapsed[first] <= maxSeconds + 1e-6; const fewest: number[] = [0]; - for (let end = 1; end <= kindsUsed.length; end++) { + for (let end = 1; end <= frames.length; end++) { fewest[end] = Infinity; - for (let first = Math.max(0, end - framesPerCap); first < end; first++) { - if (endOf(end) - startOf(first) <= maxBytes) { - fewest[end] = Math.min(fewest[end], fewest[first] + 1); - } + for (let first = 0; first < end; first++) { + if (fits(first, end)) fewest[end] = Math.min(fewest[end], fewest[first] + 1); } } - if (fewest[kindsUsed.length] === Infinity) continue; + if (fewest[frames.length] === Infinity) continue; - const parts = await split(bytes, { maxBytes, maxSeconds: framesPerCap * 0.024 }); + const parts = await split(bytes, { maxBytes, maxSeconds }); const context = `trial ${trial}`; - expect(parts.length, context).toBe(fewest[kindsUsed.length]); + expect(parts.length, context).toBe(fewest[frames.length]); + let begin = 0; for (const part of parts) { - expect(part.bytes.length, context).toBeGreaterThan(0); - expect(part.bytes.length, context).toBeLessThanOrEqual(maxBytes); + const end = begin + part.bytes.length; + const first = begin === 0 ? 0 : offsets.indexOf(begin); + const last = end === bytes.length ? frames.length : offsets.indexOf(end); + expect(last, context).toBeGreaterThan(first); + expect(fits(first, last), context).toBe(true); + expect(part.start, context).toBeCloseTo(elapsed[first], 6); + begin = end; } expect(joinsInto(parts, bytes), context).toBe(true); } }); + test("frame-like bytes in a leading tag do not set the duration of every frame", async () => { + const stream = MP3_STREAMS.mpeg25At8Kbps; + const fake = MP3_STREAMS.mpeg1At96Kbps; + // An ID3v2.4 tag holding two MPEG-1 headers 288 bytes apart, which pass for two 24 ms + // frames, before 24,001 real 72 ms frames: 1,728 s of audio. + const tag = new Uint8Array(8192); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x3f, 0x76]); + tag.set(fake.header, 100); + tag.set(fake.header, 388); + const episode = Buffer.concat([tag, mp3Bytes(stream, 24_001)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + const seconds = parts.map( + (part, index) => + ((part.bytes.length - (index === 0 ? tag.length : 0)) / stream.frameBytes) * + stream.frameSeconds, + ); + expect(seconds).toHaveLength(3); + for (const part of seconds) expect(part).toBeLessThanOrEqual(600); + expect(joinsInto(parts, episode)).toBe(true); + }); + test("an MP3 one frame short of two full parts splits into two, with no one-frame tail", async () => { const stream = MP3_STREAMS.mpeg1At96Kbps; // 600 s holds 25,000 frames of 24 ms. Summing 0.024 s 25,000 times overshoots 600. diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index ab28f0a6..eb1b1b8b 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -103,25 +103,34 @@ function mp3PartStarts( maxSeconds: number, ): { offset: number; seconds: number }[] { const count = frames.length; - // Frames in one stream last equally long, so parts are counted in frames: summing frame - // durations drifts past maxSeconds. - const frameSeconds = frames[0].seconds; - const framesPerCap = Math.floor(maxSeconds / frameSeconds + 1e-9); + // elapsed[i] is the length of frames [0, i). Summing durations drifts by a few ulps, so + // the time limit gets a microsecond of slack. + const elapsed = [0]; + for (const frame of frames) elapsed.push(elapsed[elapsed.length - 1] + frame.seconds); const startOf = (first: number) => (first === 0 ? 0 : frames[first].offset); const endOf = (end: number) => (end === count ? byteLength : frames[end].offset); const fits = (first: number, end: number) => - end - first <= framesPerCap && endOf(end) - startOf(first) <= maxBytes; - /** The end of the longest part that starts at `first` and fits. */ + elapsed[end] - elapsed[first] <= maxSeconds + 1e-6 && + endOf(end) - startOf(first) <= maxBytes; + /** The end of the longest part that starts at `first` and fits, or one frame. */ const furthest = (first: number) => { - let end = Math.min(count, first + framesPerCap); - while (end > first + 1 && !fits(first, end)) end--; - return end; + let [low, high] = [first + 1, count]; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (fits(first, middle)) low = middle; + else high = middle - 1; + } + return low; }; - /** The start of the longest part that ends at `end` and fits. */ + /** The start of the longest part that ends at `end` and fits, or one frame. */ const earliest = (end: number) => { - let first = Math.max(0, end - framesPerCap); - while (first < end - 1 && !fits(first, end)) first++; - return first; + let [low, high] = [0, end - 1]; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (fits(middle, end)) high = middle; + else low = middle + 1; + } + return low; }; // Filling each part as far as it fits gives the fewest parts. let parts = 0; @@ -141,7 +150,7 @@ function mp3PartStarts( const share = first + Math.ceil((count - first) / (parts - index + 1)); firsts.push(Math.min(Math.max(share, lowest[index]), furthest(first))); } - return firsts.map((first) => ({ offset: startOf(first), seconds: first * frameSeconds })); + return firsts.map((first) => ({ offset: startOf(first), seconds: elapsed[first] })); } /** From aea5d1aa77ecd360456d5e897c3c59dbe131bc68 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 17:47:18 +0100 Subject: [PATCH 18/26] fix(transcription): skip ID3v2 tags before, between and after MP3 frames Audio frames inside a tag's declared size were counted and could be cut through, sending an undecodable first part, and frame-like bytes in a trailing tag added a part. --- src/services/transcription/split.test.ts | 103 +++++++++++++++++++++++ src/services/transcription/split.ts | 22 ++++- 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index ba898a43..3b8d7af9 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -249,6 +249,109 @@ describe("an MP3 over a cap", () => { } }); + test("audio inside a leading tag's declared size is not cut or counted", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // An ID3v2.4 tag whose 5,760,000-byte private payload is 480 s of MP3 frames, before + // 720 s of real audio: two requests of 360 s. + const payload = mp3Bytes(stream, 20_000); + const header = [0x49, 0x44, 0x33, 4, 0, 0]; + for (const shift of [21, 14, 7, 0]) header.push((payload.length >> shift) & 0x7f); + const tag = Buffer.concat([new Uint8Array(header), payload]); + const episode = Buffer.concat([tag, mp3Bytes(stream, 30_000)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + tag.length + 15_000 * stream.frameBytes, + 15_000 * stream.frameBytes, + ]); + expect(parts[1].start).toBeCloseTo(15_000 * stream.frameSeconds, 6); + }); + + test("an MP3 that fills the time limit exactly goes whole despite frame-like bytes in its tag", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + const tag = new Uint8Array(8192); + tag.set([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0x3f, 0x76]); + tag.set(stream.header, 100); + tag.set(stream.header, 388); + const episode = Buffer.concat([tag, mp3Bytes(stream, 25_000)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([episode.length]); + }); + + test("an MP3 that fills the time limit exactly goes whole despite frame-like bytes in a trailing tag", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // An ID3v2.4 tag with a footer after the last frame. + const tag = new Uint8Array(8202); + tag.set([0x49, 0x44, 0x33, 4, 0, 0x10, 0, 0, 0x3f, 0x76]); + tag.set(stream.header, 100); + tag.set(stream.header, 388); + tag.set([0x33, 0x44, 0x49, 4, 0, 0x10, 0, 0, 0x3f, 0x76], 8192); + const episode = Buffer.concat([mp3Bytes(stream, 25_000), tag]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([episode.length]); + }); + + test("audio inside a second leading tag is not cut or counted either", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // An empty ID3v2.4 tag, then one whose payload is 480 s of MP3 frames, before 720 s of + // real audio: two requests of 360 s. + const tagWith = (payload: Uint8Array) => { + const header = [0x49, 0x44, 0x33, 4, 0, 0]; + for (const shift of [21, 14, 7, 0]) header.push((payload.length >> shift) & 0x7f); + return Buffer.concat([new Uint8Array(header), payload]); + }; + const tags = Buffer.concat([tagWith(new Uint8Array(0)), tagWith(mp3Bytes(stream, 20_000))]); + const episode = Buffer.concat([tags, mp3Bytes(stream, 30_000)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + tags.length + 15_000 * stream.frameBytes, + 15_000 * stream.frameBytes, + ]); + }); + + test("audio inside a tag after a tag with a footer is not cut or counted", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // An empty ID3v2.4 tag whose flags declare a 10-byte footer the size leaves out, then a + // tag whose payload is 480 s of MP3 frames, before 720 s of real audio. + const footed = new Uint8Array(20); + footed.set([0x49, 0x44, 0x33, 4, 0, 0x10, 0, 0, 0, 0]); + footed.set([0x33, 0x44, 0x49, 4, 0, 0x10, 0, 0, 0, 0], 10); + const payload = mp3Bytes(stream, 20_000); + const header = [0x49, 0x44, 0x33, 4, 0, 0]; + for (const shift of [21, 14, 7, 0]) header.push((payload.length >> shift) & 0x7f); + const tags = Buffer.concat([footed, new Uint8Array(header), payload]); + const episode = Buffer.concat([tags, mp3Bytes(stream, 30_000)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + tags.length + 15_000 * stream.frameBytes, + 15_000 * stream.frameBytes, + ]); + }); + + test("tag-like bytes where no frame could start do not hide the audio after them", async () => { + const stream = MP3_STREAMS.mpeg1At96Kbps; + // 64 bytes of junk holding "ID3" and a size of 32 MiB, before 720 s of audio. + const junk = new Uint8Array(64); + junk.set([0x49, 0x44, 0x33, 4, 0, 0, 0x10, 0, 0, 0], 20); + const episode = Buffer.concat([junk, mp3Bytes(stream, 30_000)]); + + const parts = await split(episode, { maxBytes: 20 * MIB, maxSeconds: 600 }); + + expect(parts.map((part) => part.bytes.length)).toEqual([ + junk.length + 15_000 * stream.frameBytes, + 15_000 * stream.frameBytes, + ]); + }); + test("frame-like bytes in a leading tag do not set the duration of every frame", async () => { const stream = MP3_STREAMS.mpeg25At8Kbps; const fake = MP3_STREAMS.mpeg1At96Kbps; diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index eb1b1b8b..0937d144 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -154,21 +154,35 @@ function mp3PartStarts( } /** - * The Layer III frames in `bytes`. A frame counts when it continues the frames before it, - * or when another frame or the end of the buffer follows it, so sync-like bytes in a tag - * or cover art don't, and the last frame before trailing metadata does. + * The Layer III frames in `bytes`, skipping ID3v2 tags at the start or right after a frame or tag. + * A frame counts when it continues the frames before it, or when another frame or the end + * of the buffer follows it, so sync-like bytes in other metadata don't, and the last frame + * before trailing metadata does. */ function mp3Frames(bytes: Uint8Array): Mp3Frame[] { const frames: Mp3Frame[] = []; let offset = 0; + // Where the last frame or tag ended, where the next tag may start. + let boundary = 0; while (offset + 4 <= bytes.length) { + // An ID3v2 tag: "ID3", version, flags, then its size in 7 bits per byte, which leaves + // out the 10-byte header and the 10-byte footer that flag 0x10 adds. + const isTag = + bytes[offset] === 0x49 && bytes[offset + 1] === 0x44 && bytes[offset + 2] === 0x33; + if (isTag && offset === boundary) { + let size = 0; + for (let index = offset + 6; index < offset + 10; index++) + size = (size << 7) | (bytes[index] & 0x7f); + offset = boundary = offset + 10 + size + (bytes[offset + 5] & 0x10 ? 10 : 0); + continue; + } const frame = mp3FrameAt(bytes, offset); const next = frame ? offset + frame.length : 0; const last = frames[frames.length - 1]; const continues = last !== undefined && last.offset + last.length === offset; if (frame && (continues || next === bytes.length || mp3FrameAt(bytes, next))) { frames.push(frame); - offset = next; + offset = boundary = next; } else { offset++; } From d544390dfad842686d43eeea6186792acc6898a1 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:07:11 +0100 Subject: [PATCH 19/26] fix(transcription): split long audio in other formats under the length cap A file in a format other than MP3 under the size cap went whole whatever its length, so a long .m4a passed a model's maxSeconds. Its length now comes from its metadata through a media element, without decoding it, and audio over the cap is decoded into WAV parts that fit both caps. Audio of an unknown length still goes whole. --- src/services/transcription/split.test.ts | 51 +++++++++++++++++++++++- src/services/transcription/split.ts | 46 ++++++++++++++++++--- vitest.setup.dom.ts | 14 +++++++ 3 files changed, 104 insertions(+), 7 deletions(-) diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index 3b8d7af9..a1055819 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -407,7 +407,7 @@ describe("an MP3 over a cap", () => { }); }); -describe("audio over the size cap that cannot be cut where it is", () => { +describe("audio without MP3 frames to cut at", () => { const SAMPLE_RATE = 44_100; // jsdom has no Web Audio, so a decoder that returns silence of the given length stands in. @@ -475,6 +475,55 @@ describe("audio over the size cap that cannot be cut where it is", () => { }, ); + describe("an .m4a under the size cap", () => { + const stubSeconds = (seconds: number) => + vi.spyOn(HTMLMediaElement.prototype, "duration", "get").mockReturnValue(seconds); + const noDecoder = () => + stubAudioContext(() => { + throw new Error("decoded audio that goes whole"); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + test("but over maxSeconds is decoded into WAV parts no longer than maxSeconds", async () => { + stubSeconds(5); + stubAudioContext(() => new FakeAudioBuffer(5 * SAMPLE_RATE)); + + const parts = await split( + new Uint8Array(4096), + { maxBytes: MIB, maxSeconds: 2 }, + "m4a", + ); + + expect(parts.map((part) => (part.bytes.length - 44) / 2 / SAMPLE_RATE)).toEqual([ + 2, 2, 1, + ]); + expect(parts.map((part) => part.start)).toEqual([0, 2, 4]); + }); + + test("within maxSeconds goes whole, without decoding", async () => { + stubSeconds(1.5); + noDecoder(); + const episode = new Uint8Array(4096).fill(7); + + const parts = await split(episode, { maxBytes: MIB, maxSeconds: 2 }, "m4a"); + + expect(parts).toEqual([ + { name: "audio.m4a", type: "audio/mp4", start: 0, bytes: episode }, + ]); + }); + + test("whose length its metadata does not give goes whole", async () => { + noDecoder(); + const episode = new Uint8Array(4096).fill(7); + + const parts = await split(episode, { maxBytes: MIB, maxSeconds: 2 }, "m4a"); + + expect(parts.map((part) => part.name)).toEqual(["audio.m4a"]); + }); + }); + test("throws when no decoder is available", async () => { await expect(split(new Uint8Array(1025), { maxBytes: 1024 }, "flac")).rejects.toThrow( /Could not split flac audio/, diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index 0937d144..38da7437 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -64,8 +64,10 @@ export async function splitAudio( })); } - if (fits) return whole(); - const parts = await convertToWavParts(buffer, maxBytes); + // Other formats are measured by their metadata, without decoding them, and audio of an + // unknown length goes whole. + if (fits && !((await mediaSeconds(buffer, type)) > maxSeconds)) return whole(); + const parts = await convertToWavParts(buffer, maxBytes, maxSeconds); if (parts.length === 0) { throw new Error( `Could not split ${extension || "audio"} audio for transcription. This format must be decoded to WAV before splitting, but the audio decoder is unavailable or could not decode it.`, @@ -209,13 +211,35 @@ function mp3FrameAt(bytes: Uint8Array, offset: number): Mp3Frame | undefined { }; } -async function convertToWavParts(buffer: ArrayBuffer, maxBytes: number): Promise { +/** The audio's length from its metadata, or NaN when it has none or cannot be read. */ +async function mediaSeconds(buffer: ArrayBuffer, type: string): Promise { + if (typeof activeDocument === "undefined") return NaN; + const url = URL.createObjectURL(new Blob([buffer], { type })); + const audio = activeDocument.createElement("audio"); + try { + return await new Promise((resolve) => { + audio.onloadedmetadata = () => resolve(audio.duration); + audio.onerror = () => resolve(NaN); + audio.preload = "metadata"; + audio.src = url; + }); + } finally { + audio.removeAttribute("src"); + URL.revokeObjectURL(url); + } +} + +async function convertToWavParts( + buffer: ArrayBuffer, + maxBytes: number, + maxSeconds: number, +): Promise { const audioContext = createAudioContext(); if (!audioContext) return []; try { const audioBuffer = await audioContext.decodeAudioData(buffer.slice(0)); - return renderWavParts(audioBuffer, maxBytes); + return renderWavParts(audioBuffer, maxBytes, maxSeconds); } catch (error) { console.warn("Failed to convert audio buffer for transcription", error); return []; @@ -243,10 +267,20 @@ function createAudioContext(): AudioContext | null { return new contextCtor(); } -function renderWavParts(audioBuffer: AudioBuffer, maxBytes: number): AudioPart[] { +function renderWavParts( + audioBuffer: AudioBuffer, + maxBytes: number, + maxSeconds: number, +): AudioPart[] { const numChannels = audioBuffer.numberOfChannels; const bytesPerFrame = numChannels * PCM_BYTES_PER_SAMPLE; - const maxSamplesPerPart = Math.max(1, Math.floor((maxBytes - WAV_HEADER_SIZE) / bytesPerFrame)); + const maxSamplesPerPart = Math.max( + 1, + Math.min( + Math.floor((maxBytes - WAV_HEADER_SIZE) / bytesPerFrame), + Math.floor(maxSeconds * audioBuffer.sampleRate), + ), + ); const channelData = Array.from({ length: numChannels }, (_, channelIndex) => audioBuffer.getChannelData(channelIndex), ); diff --git a/vitest.setup.dom.ts b/vitest.setup.dom.ts index 286379a0..b87df16a 100644 --- a/vitest.setup.dom.ts +++ b/vitest.setup.dom.ts @@ -293,3 +293,17 @@ if (!Element.prototype.animate) { return animation as unknown as Animation; }; } + +// jsdom loads no media, so an element given an object URL would never fire +// `loadedmetadata`. Fire it, with jsdom's `duration` of NaN: the length is unknown +// unless a test stubs `duration`. +const mediaSrc = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, "src"); +Object.defineProperty(HTMLMediaElement.prototype, "src", { + ...mediaSrc, + set(this: HTMLMediaElement, value: string) { + mediaSrc?.set?.call(this, value); + if (value.startsWith("blob:")) { + queueMicrotask(() => this.dispatchEvent(new Event("loadedmetadata"))); + } + }, +}); From cc142ba342f5d07e650d76bf24f8f0b50a7c7a01 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:07:11 +0100 Subject: [PATCH 20/26] build(size): raise the gzip budget by the 118 bytes the length check measures --- AGENTS.md | 2 +- scripts/size-baseline.md | 11 ++++++++++- scripts/size.mjs | 2 +- scripts/size.test.ts | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4bbd8f5d..0340ab8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,145 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,178 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 56bd59fa..aede3f36 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 104,145 bytes. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 104,178 bytes. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -43,6 +43,15 @@ main.js raw: 320267 bytes; budget 330000 bytes; 9733 bytes remaining main.js gzip (level 9): 104145 bytes; budget 104145 bytes; 0 bytes remaining ``` +On 2026-11-14 it rose from 104,145 by the 118 bytes that measure audio in other +formats against each model's length cap, so a long `.m4a` under the size cap is +split instead of sent whole. With it: + +```text +main.js raw: 319136 bytes; budget 330000 bytes; 10864 bytes remaining +main.js gzip (level 9): 104178 bytes; budget 104178 bytes; 0 bytes remaining +``` + Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. Budgets use decimal bytes. The brief's 96,533-byte reference is unexplained. This report fixes the compressor instead of comparing compressed sizes across diff --git a/scripts/size.mjs b/scripts/size.mjs index 0da6e12e..8baf4dc0 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -10,7 +10,7 @@ try { if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ { name: "raw", bytes: bundle.length, budget: 330_000 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_145 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_178 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index f9fef310..78363b46 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -30,7 +30,7 @@ describe("bundle budget CLI", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 104145 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 104178 bytes; OVER by \d+ bytes/, ); expect(result.stdout).toContain("budget 330000 bytes; 220000 bytes remaining"); }); From a8c841736e715297bd313fd8ba6ef0403836103f Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:11:38 +0100 Subject: [PATCH 21/26] fix(transcription): upload episodes whose address is a signed URL A public feed's enclosure with AWS or Google Cloud signing parameters passed the feed credential check, so Deepgram got the whole signed address. A query parameter whose name holds signature, credential or token, or Akamai's hdnts, now uploads the audio instead. --- src/services/TranscriptionService.test.ts | 43 +++++++++++++++++++++++ src/services/transcription/audioSource.ts | 15 ++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 7e18b0cd..76382d97 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -851,6 +851,49 @@ describe("TranscriptionService", () => { expectUpload("SIGNED-TOKEN", "cdn.example.com"); }); + test.each([ + [ + "AWS", + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=CRED&X-Amz-Signature=SIGNED", + ], + [ + "Google Cloud", + "X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=CRED&X-Goog-Signature=SIGNED", + ], + ["CloudFront", "Expires=1900000000&Signature=SIGNED&Key-Pair-Id=CRED"], + ["Azure", "sv=2024-11-04&se=2030-01-01&sp=r&sig=SIGNED"], + ])( + "an episode whose address is a %s signed URL is uploaded, and no request holds the address", + async (_, query) => { + providerMock.mockImplementation(async () => Response.json(answers)); + const episode: Episode = { + ...mockEpisode, + streamUrl: `https://cdn.example.com/episode.mp3?${query}`, + }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [episode])); + + await transcribe(plugin, episode); + + expectUpload("SIGNED", "cdn.example.com"); + }, + ); + + test("a public episode whose address has tracking parameters goes by its address", async () => { + providerMock.mockImplementation(async () => Response.json(answers)); + const streamUrl = + "https://cdn.example.com/episode.mp3?aid=rss_feed&feed=AbC123&updated=1700000000&utm_source=podcast"; + const episode: Episode = { ...mockEpisode, streamUrl }; + const plugin = useDeepgram(savedFeedRepository(publicFeed, [episode])); + + await transcribe(plugin, episode); + + expect(getEpisodeAudioBufferMock).not.toHaveBeenCalled(); + const [request] = providerMock.mock.calls[0]; + expect(JSON.parse(new TextDecoder().decode(sentBytes(request)))).toEqual({ + url: streamUrl, + }); + }); + test("an episode on a private network is uploaded, and no request holds its address", async () => { providerMock.mockImplementation(async () => Response.json(answers)); const episode: Episode = { diff --git a/src/services/transcription/audioSource.ts b/src/services/transcription/audioSource.ts index f8030e58..e6d60f2f 100644 --- a/src/services/transcription/audioSource.ts +++ b/src/services/transcription/audioSource.ts @@ -19,10 +19,20 @@ export type PublicAudioUrl = string & { readonly [publicAudio]: true }; * - a saved public feed lists exactly this address as an item's enclosure. A private feed's * episode never passes, even with a clean-looking address, because publishers put the * subscriber's token in the path. Neither does a local file, which no feed lists; - * - the address carries no credential and names a public http(s) host. + * - the address carries no credential or signed-URL parameter and names a public http(s) host. * * Anything else resolves undefined and is uploaded, which is always correct. */ +/** + * A query parameter that signs a URL: AWS and Google Cloud's X-Amz- and X-Goog- credential, + * signature and token, and Akamai's tokens. Matched within the name, so an unknown scheme's + * signature uploads the audio. + */ +const SIGNED_URL_PARAM = /signature|credential|token|^hdnts$/i; + +const isSignedUrl = (address: string): boolean => + [...new URL(address).searchParams.keys()].some((name) => SIGNED_URL_PARAM.test(name)); + export async function publicAudioUrl( episode: Episode, feeds: Pick, @@ -36,7 +46,8 @@ export async function publicAudioUrl( return state.episodes.some((item) => item.streamUrl === address); }; - if (isCredentialBearingUrl(address) || !isFetchableUrl(address)) return undefined; + if (isCredentialBearingUrl(address) || !isFetchableUrl(address) || isSignedUrl(address)) + return undefined; if (listed() === undefined) return undefined; const loaded = await feeds.load([key], "cached"); // The cache keeps a long feed's newest items, so an older episode needs the whole feed. From 7b64684fcd90c454e3a1c13e640e0118b0ec896c Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:11:38 +0100 Subject: [PATCH 22/26] build(size): set the gzip budget to the 104,235 bytes the engine fixes measure --- AGENTS.md | 2 +- scripts/size-baseline.md | 13 +++++++------ scripts/size.mjs | 2 +- scripts/size.test.ts | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0340ab8c..0b5c15cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,178 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,235 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index aede3f36..6ab64c42 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 104,178 bytes. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 104,235 bytes. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -43,13 +43,14 @@ main.js raw: 320267 bytes; budget 330000 bytes; 9733 bytes remaining main.js gzip (level 9): 104145 bytes; budget 104145 bytes; 0 bytes remaining ``` -On 2026-11-14 it rose from 104,145 by the 118 bytes that measure audio in other -formats against each model's length cap, so a long `.m4a` under the size cap is -split instead of sent whole. With it: +On 2026-11-14 it rose from 104,145 by 175 bytes for the transcription engine: +118 measure audio in other formats against each model's length cap, so a long +`.m4a` under the size cap is split instead of sent whole, and 57 keep a signed +enclosure URL from going to a provider. With them: ```text -main.js raw: 319136 bytes; budget 330000 bytes; 10864 bytes remaining -main.js gzip (level 9): 104178 bytes; budget 104178 bytes; 0 bytes remaining +main.js raw: 319246 bytes; budget 330000 bytes; 10754 bytes remaining +main.js gzip (level 9): 104235 bytes; budget 104235 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index 8baf4dc0..afd2c9f5 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -10,7 +10,7 @@ try { if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ { name: "raw", bytes: bundle.length, budget: 330_000 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_178 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_235 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index 78363b46..f84ba34b 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -30,7 +30,7 @@ describe("bundle budget CLI", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 104178 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 104235 bytes; OVER by \d+ bytes/, ); expect(result.stdout).toContain("budget 330000 bytes; 220000 bytes remaining"); }); From 1f0b4c23127691336b6b6c162b3ceb72f955ffc7 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:16:48 +0100 Subject: [PATCH 23/26] fix(transcription): upload episodes signed with an Akamai hdnea token --- src/services/TranscriptionService.test.ts | 2 ++ src/services/transcription/audioSource.ts | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 76382d97..6f5dc239 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -862,6 +862,8 @@ describe("TranscriptionService", () => { ], ["CloudFront", "Expires=1900000000&Signature=SIGNED&Key-Pair-Id=CRED"], ["Azure", "sv=2024-11-04&se=2030-01-01&sp=r&sig=SIGNED"], + ["Akamai query token", "hdnts=exp=1900000000~acl=/*~hmac=SIGNED"], + ["Akamai cookie token", "hdnea=exp=1900000000~acl=/*~hmac=SIGNED"], ])( "an episode whose address is a %s signed URL is uploaded, and no request holds the address", async (_, query) => { diff --git a/src/services/transcription/audioSource.ts b/src/services/transcription/audioSource.ts index e6d60f2f..1c39e4b9 100644 --- a/src/services/transcription/audioSource.ts +++ b/src/services/transcription/audioSource.ts @@ -25,10 +25,10 @@ export type PublicAudioUrl = string & { readonly [publicAudio]: true }; */ /** * A query parameter that signs a URL: AWS and Google Cloud's X-Amz- and X-Goog- credential, - * signature and token, and Akamai's tokens. Matched within the name, so an unknown scheme's + * signature and token, and Akamai's hdnts and hdnea tokens. Matched within the name, so an unknown scheme's * signature uploads the audio. */ -const SIGNED_URL_PARAM = /signature|credential|token|^hdnts$/i; +const SIGNED_URL_PARAM = /signature|credential|token|^hdn(ts|ea)$/i; const isSignedUrl = (address: string): boolean => [...new URL(address).searchParams.keys()].some((name) => SIGNED_URL_PARAM.test(name)); From 37fdc08c8b2bff0862194a8b504ff3e84bcf98db Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:16:48 +0100 Subject: [PATCH 24/26] build(size): set the gzip budget to the 104,238 bytes the engine fixes measure --- AGENTS.md | 2 +- scripts/size-baseline.md | 10 +++++----- scripts/size.mjs | 2 +- scripts/size.test.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0b5c15cd..c01faa82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,235 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,238 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 6ab64c42..11479853 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 104,235 bytes. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 104,238 bytes. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -43,14 +43,14 @@ main.js raw: 320267 bytes; budget 330000 bytes; 9733 bytes remaining main.js gzip (level 9): 104145 bytes; budget 104145 bytes; 0 bytes remaining ``` -On 2026-11-14 it rose from 104,145 by 175 bytes for the transcription engine: +On 2026-11-14 it rose from 104,145 by 178 bytes for the transcription engine: 118 measure audio in other formats against each model's length cap, so a long -`.m4a` under the size cap is split instead of sent whole, and 57 keep a signed +`.m4a` under the size cap is split instead of sent whole, and 60 keep a signed enclosure URL from going to a provider. With them: ```text -main.js raw: 319246 bytes; budget 330000 bytes; 10754 bytes remaining -main.js gzip (level 9): 104235 bytes; budget 104235 bytes; 0 bytes remaining +main.js raw: 319251 bytes; budget 330000 bytes; 10749 bytes remaining +main.js gzip (level 9): 104238 bytes; budget 104238 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index afd2c9f5..cfce3eed 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -10,7 +10,7 @@ try { if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ { name: "raw", bytes: bundle.length, budget: 330_000 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_235 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_238 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index f84ba34b..29b8b4a9 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -30,7 +30,7 @@ describe("bundle budget CLI", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 104235 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 104238 bytes; OVER by \d+ bytes/, ); expect(result.stdout).toContain("budget 330000 bytes; 220000 bytes remaining"); }); From 79addc37423c1dcaa94c4d0aa1bc12c613ceef55 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:30:00 +0100 Subject: [PATCH 25/26] fix(transcription): stop waiting for audio metadata after 5 s or on unload A WebView may load no metadata for media it does not play, so the length probe could wait forever, and an unload could not stop it before a decode. The probe now reads the length as unknown after 5 s or when the job is aborted, and an aborted job decodes nothing. --- src/services/TranscriptionService.test.ts | 53 ++++++++++++++++++++++- src/services/transcription/split.test.ts | 44 ++++++++++++++++++- src/services/transcription/split.ts | 23 ++++++++-- src/services/transcription/transcribe.ts | 9 ++-- 4 files changed, 119 insertions(+), 10 deletions(-) diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index 6f5dc239..fac16c2a 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -1,5 +1,5 @@ // @vitest-environment jsdom -import { describe, expect, test, vi } from "vitest"; +import { describe, expect, onTestFinished, test, vi } from "vitest"; import { Notice, type RequestUrlParam } from "obsidian"; import { deferred, feedDocument, MemoryStorage } from "../../tests/mocks/feeds"; import { @@ -262,6 +262,57 @@ describe("TranscriptionService", () => { expect(decodes).toBe(0); }); + test("decodes no audio whose length its metadata gives after unload", async () => { + getEpisodeAudioBufferMock.mockResolvedValue({ + buffer: new ArrayBuffer(4096), + extension: "m4a", + }); + // Longer than OpenAI's length cap, so it would be decoded into WAV parts. + const duration = vi + .spyOn(HTMLMediaElement.prototype, "duration", "get") + .mockReturnValue(3000); + // The probe's element loads nothing until the test fires its metadata. + let probe: HTMLElement | undefined; + const createElement = document.createElement.bind(document); + const create = vi.spyOn(document, "createElement").mockImplementation((tag: string) => { + const element = createElement(tag); + if (tag === "audio") probe = element; + return element; + }); + const src = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, "src"); + Object.defineProperty(HTMLMediaElement.prototype, "src", { ...src, set() {} }); + onTestFinished(() => { + duration.mockRestore(); + create.mockRestore(); + if (src) Object.defineProperty(HTMLMediaElement.prototype, "src", src); + }); + let decodes = 0; + vi.stubGlobal( + "AudioContext", + class { + async decodeAudioData(): Promise { + decodes++; + throw new Error("Unsupported codec"); + } + async close(): Promise {} + }, + ); + const service = new TranscriptionService(createMockPlugin()); + const pending = ( + service as unknown as { + transcribeEpisode: (job: AudioJob) => Promise; + } + ).transcribeEpisode(audioJob); + await vi.waitFor(() => expect(probe).toBeDefined()); + + service.dispose(); + probe?.dispatchEvent(new Event("loadedmetadata")); + await pending; + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(decodes).toBe(0); + }); + test("downloads no audio when unloaded while it looks the episode up in its feed", async () => { const lookup = deferred>(); const load = vi.fn(() => lookup.promise); diff --git a/src/services/transcription/split.test.ts b/src/services/transcription/split.test.ts index a1055819..19169fe5 100644 --- a/src/services/transcription/split.test.ts +++ b/src/services/transcription/split.test.ts @@ -1,5 +1,5 @@ // @vitest-environment jsdom -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, describe, expect, onTestFinished, test, vi } from "vitest"; import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../../tests/mocks/mp3"; import { type PartCaps, splitAudio } from "./split"; @@ -514,6 +514,48 @@ describe("audio without MP3 frames to cut at", () => { ]); }); + const silentMedia = () => { + const src = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, "src"); + Object.defineProperty(HTMLMediaElement.prototype, "src", { ...src, set() {} }); + onTestFinished(() => { + if (src) Object.defineProperty(HTMLMediaElement.prototype, "src", src); + }); + }; + + test("whose metadata never loads goes whole after 5 s", async () => { + vi.useFakeTimers(); + onTestFinished(() => { + vi.useRealTimers(); + }); + silentMedia(); + noDecoder(); + const parts = splitAudio( + { buffer: new Uint8Array(4096).buffer, extension: "m4a" }, + { maxBytes: MIB, maxSeconds: 2 }, + ); + + await vi.advanceTimersByTimeAsync(5000); + + expect((await parts).map(({ file }) => file.name)).toEqual(["audio.m4a"]); + }); + + test("stops waiting for its metadata on abort and decodes nothing", async () => { + silentMedia(); + const decode = vi.fn(); + stubAudioContext(decode); + const controller = new AbortController(); + const parts = splitAudio( + { buffer: new Uint8Array(4096).buffer, extension: "m4a" }, + { maxBytes: MIB, maxSeconds: 2 }, + controller.signal, + ); + + controller.abort(new Error("unloaded")); + + await expect(parts).rejects.toThrow("unloaded"); + expect(decode).not.toHaveBeenCalled(); + }); + test("whose length its metadata does not give goes whole", async () => { noDecoder(); const episode = new Uint8Array(4096).fill(7); diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index 38da7437..0032c4a5 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -43,6 +43,7 @@ function getMimeType(fileExtension: string): string { export async function splitAudio( audio: { readonly buffer: ArrayBuffer; readonly extension: string }, { maxBytes, maxSeconds = Infinity }: PartCaps, + signal?: AbortSignal, ): Promise { const { buffer, extension } = audio; const type = getMimeType(extension); @@ -66,7 +67,9 @@ export async function splitAudio( // Other formats are measured by their metadata, without decoding them, and audio of an // unknown length goes whole. - if (fits && !((await mediaSeconds(buffer, type)) > maxSeconds)) return whole(); + const seconds = fits ? await mediaSeconds(buffer, type, signal) : NaN; + if (signal?.aborted) throw signal.reason; + if (fits && !(seconds > maxSeconds)) return whole(); const parts = await convertToWavParts(buffer, maxBytes, maxSeconds); if (parts.length === 0) { throw new Error( @@ -211,19 +214,31 @@ function mp3FrameAt(bytes: Uint8Array, offset: number): Mp3Frame | undefined { }; } -/** The audio's length from its metadata, or NaN when it has none or cannot be read. */ -async function mediaSeconds(buffer: ArrayBuffer, type: string): Promise { +/** + * The audio's length from its metadata, or NaN when it has none, cannot be read within 5 s + * (a WebView may load no media it does not play), or the job is aborted. + */ +async function mediaSeconds( + buffer: ArrayBuffer, + type: string, + signal: AbortSignal | undefined, +): Promise { if (typeof activeDocument === "undefined") return NaN; const url = URL.createObjectURL(new Blob([buffer], { type })); const audio = activeDocument.createElement("audio"); + let timer = 0; try { return await new Promise((resolve) => { + const unknown = () => resolve(NaN); audio.onloadedmetadata = () => resolve(audio.duration); - audio.onerror = () => resolve(NaN); + audio.onerror = unknown; + signal?.addEventListener("abort", unknown, { once: true }); + timer = window.setTimeout(unknown, 5000); audio.preload = "metadata"; audio.src = url; }); } finally { + window.clearTimeout(timer); audio.removeAttribute("src"); URL.revokeObjectURL(url); } diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts index 750a4937..3bd75ca6 100644 --- a/src/services/transcription/transcribe.ts +++ b/src/services/transcription/transcribe.ts @@ -67,10 +67,11 @@ export async function transcribe(job: TranscriptionJob): Promise { onProgress("Fetching episode audio..."); const audio = await getEpisodeAudioBuffer(episode); if (signal.aborted) throw signal.reason; - const files = await splitAudio(audio, { - maxBytes: model.maxUploadBytes, - maxSeconds: model.maxSeconds, - }); + const files = await splitAudio( + audio, + { maxBytes: model.maxUploadBytes, maxSeconds: model.maxSeconds }, + signal, + ); // One part at a time: an upload is held in memory whole, and a failure that ends the // job ends it before the next part is sent. for (const [index, { file, start }] of files.entries()) { From 3641fb8b5aae2158b4c7250bdba5de92c6ac3fa5 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Sat, 14 Nov 2026 20:30:00 +0100 Subject: [PATCH 26/26] build(size): set the gzip budget to the 104,290 bytes the engine fixes measure --- AGENTS.md | 2 +- scripts/size-baseline.md | 15 ++++++++------- scripts/size.mjs | 2 +- scripts/size.test.ts | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c01faa82..4b02ffd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ svelte-check's native API path passes when a tsconfig matches no files, which `check:a11y` in temporary workspaces and requires it to fail on that case and when the native compiler cannot start. -After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,238 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. +After `npm run build`, `npm run size` enforces 330,000 raw bytes and 104,290 gzip bytes at level 9 for root `main.js` in the Test workflow's Build and sizes worker. `npm run size:data` reports a synthetic v2 stress library with 50 feeds, 2,000 history entries, and 200 saved episodes from frozen NPR metadata, including duplicated snapshots. Its 1.5 MB document goal is advisory. Pass an additive candidate JSON path to report optional-field overhead without changing existing data. In the isolated Obsidian instance, `npm run count:writes -- start`, `report`, and `stop` record writes to PodNotes' exact `data.json` path. Stop the recorder before stopping the instance, because recorder commands start a stopped instance; plugin reloads invalidate its counts. Before opening a PR or cutting a release, run the CI-equivalent checks locally: diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 11479853..0e15e00e 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 104,238 bytes. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 104,290 bytes. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -43,14 +43,15 @@ main.js raw: 320267 bytes; budget 330000 bytes; 9733 bytes remaining main.js gzip (level 9): 104145 bytes; budget 104145 bytes; 0 bytes remaining ``` -On 2026-11-14 it rose from 104,145 by 178 bytes for the transcription engine: -118 measure audio in other formats against each model's length cap, so a long -`.m4a` under the size cap is split instead of sent whole, and 60 keep a signed -enclosure URL from going to a provider. With them: +On 2026-11-14 it rose from 104,145 to 104,290 for the transcription engine, +which measured 104,060 on its own. On top of it, 170 bytes measure audio in other +formats against each model's length cap, for at most 5 s and only while the job +runs, so a long `.m4a` under the size cap is split instead of sent whole, and 60 +keep a signed enclosure URL from going to a provider. With them: ```text -main.js raw: 319251 bytes; budget 330000 bytes; 10749 bytes remaining -main.js gzip (level 9): 104238 bytes; budget 104238 bytes; 0 bytes remaining +main.js raw: 319405 bytes; budget 330000 bytes; 10595 bytes remaining +main.js gzip (level 9): 104290 bytes; budget 104290 bytes; 0 bytes remaining ``` Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. diff --git a/scripts/size.mjs b/scripts/size.mjs index cfce3eed..0c5f9b1a 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -10,7 +10,7 @@ try { if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ { name: "raw", bytes: bundle.length, budget: 330_000 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_238 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 104_290 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index 29b8b4a9..9f90153c 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -30,7 +30,7 @@ describe("bundle budget CLI", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 104238 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 104290 bytes; OVER by \d+ bytes/, ); expect(result.stdout).toContain("budget 330000 bytes; 220000 bytes remaining"); });