diff --git a/.claude/skills/verify/SKILL.md b/.claude/skills/verify/SKILL.md index 9bb5b621..363eb1e6 100644 --- a/.claude/skills/verify/SKILL.md +++ b/.claude/skills/verify/SKILL.md @@ -226,7 +226,10 @@ its profile. It keeps the vault and `.obsidian-e2e-artifacts/`. Delete - `.claude/skills/verify/net.mjs` runs as `watch`, `log` or `unwatch`. It logs every `fetch` and `requestUrl` call PodNotes makes, without query strings, and answers ElevenLabs, Deepgram, OpenRouter and OpenAI with a fixture - provider while it watches. + provider while it watches. It decodes Grok's MP3 uploads with Chromium's + decoder and hears a tone under 500 Hz as one speaker and a higher tone as + another, and logs each upload's `decodedSeconds` and `heardSpeakers`. Audio + it cannot decode is answered with a 400. - `.claude/skills/verify/ui.mjs` sends trusted input through `obsidian dev:cdp`. Its verbs are `click [--text ] [--right]`, diff --git a/.claude/skills/verify/features/transcripts.md b/.claude/skills/verify/features/transcripts.md index 3e772f95..28512591 100644 --- a/.claude/skills/verify/features/transcripts.md +++ b/.claude/skills/verify/features/transcripts.md @@ -54,7 +54,7 @@ Preconditions: - **OpenRouter.** Choose `OpenRouter` with its fake key. The notes read as for ElevenLabs. - Scribe v2, public. Choose `ElevenLabs Scribe v2`, play `Fixture Episode 1` of `PodNotes Fixture` and run the audio command. `$NET log` shows the catalog `GET`, then one `POST https://openrouter.ai/api/v1/audio/transcriptions` with `contentType` `application/json`, under 1,000 `uploadBytes` and the episode's address as `sourceUrl`, and no request for `/episode-1.wav` after the command. - Grok STT 2.0. On `Recommended (Grok STT 2.0)`, play `Fixtures/Local Fixture.wav` and run the audio command. The `POST` is `application/json` with about 4/3 of the file's bytes, its audio in base64, and no `sourceUrl`. - - In parts. Write the 70 min MP3 from the OpenAI steps below, play it and run the audio command on Grok. `$NET log` shows three `POST https://openrouter.ai/api/v1/audio/transcriptions` entries of about 1.9 MB each, one after the other. The fixture answers each with two speakers, so the note numbers six, and the progress notice ends with `OpenRouter transcribed this episode in 3 parts, and speaker labels restart in each, so one person can have several labels.` + - In parts. Write a 70 min MP3 of two voices, a 300 Hz tone and a 700 Hz tone that take turns every 45 s: `ffmpeg -f lavfi -i "aevalsrc='0.5*sin(2*PI*if(lt(mod(t\,90)\,45)\,300\,700)*t)':s=8000:d=4200" -ac 1 -ar 8000 -b:a 8k "$VAULT/Fixtures/Two Voices.mp3"`. Play it and run the audio command on Grok. `$NET log` shows three `POST https://openrouter.ai/api/v1/audio/transcriptions` entries of about 1.9 MB each, one after the other, then the speaker reel, one of about 82 KB whose `decodedSeconds` is `61.56`: six 9 s clips with 1.5 s of silence between two. The fixture numbers the voices in the order they sound in each request, so the second part, which starts with the high tone, numbers them the other way. The note still numbers the low voice `1` and the high voice `2` throughout, the paragraph at `00:23:15` runs on across the first part boundary, and the progress notice ends with `Transcription completed and saved.` With `ElevenLabs Scribe v2` the parts go as multipart, no reel follows, each part's fixed answer numbers two more speakers, and the notice ends with `OpenRouter transcribed this episode in 3 parts, and speaker labels restart in each, so one person can have several labels.` - **OpenAI.** Choose `OpenAI` with its fake key. `Untranscribed Episode` downloads its WAV and `$NET log` shows a `POST https://api.openai.com/v1/audio/transcriptions` answered by the fixture provider. The note holds the fixture text with no timestamps. - In parts. Write an MP3 longer than 2,070 s into the vault, for example 70 min in 4 MB: `ffmpeg -f lavfi -i sine=frequency=440:duration=4200 -ac 1 -ar 8000 -b:a 8k "$VAULT/Fixtures/Long Fixture.mp3"`. Play it ([local-files.md](./local-files.md)) and run the audio command. `$NET log` shows three `POST https://api.openai.com/v1/audio/transcriptions` entries of about 1.4 MB each, one after the other, and the note holds the fixture text three times with no timestamps. - Speaker labels. With `Speaker labels` on, as in a library that labelled speakers with OpenAI on 2.24.0, the progress notice ends with `Saved without speaker labels, because OpenAI does not label speakers. These providers do: ElevenLabs, Deepgram, OpenRouter.` and stays until clicked. diff --git a/.claude/skills/verify/net.mjs b/.claude/skills/verify/net.mjs index 37eab3ca..56e4d16a 100644 --- a/.claude/skills/verify/net.mjs +++ b/.claude/skills/verify/net.mjs @@ -110,6 +110,42 @@ const WATCH = `(() => { const openRouterWords = (sentence, speaker, from) => words(sentence, speaker, from).map(({ punctuated_word, start, end }) => ({ word: punctuated_word, start, end, speaker, speaker_label: "speaker_" + speaker, confidence: 0.99 })); const openrouter = { text: text + " " + reply, words: [...openRouterWords(text, 0, 0), ...openRouterWords(reply, 1, 12)] }; const openrouterCatalog = { data: ${jsLiteral(OPENROUTER_MODELS)}.map((id) => ({ id, expiration_date: null })) }; + // Grok's MP3 uploads are heard as a diarizing model hears them: a tone under 500 Hz is one + // voice and a higher one another, labeled in the order they first sound in the request, with + // a word for each turn. Other OpenRouter requests get the fixed answer. + const hearOpenRouter = async (request, entry) => { + if (request.contentType !== "application/json") return openrouter; + const audio = JSON.parse(new TextDecoder().decode(request.body)).input_audio ?? {}; + if (audio.format !== "mp3" || !audio.data) return openrouter; + const bytes = Uint8Array.from(atob(audio.data), (char) => char.charCodeAt(0)); + const decoded = await new OfflineAudioContext(1, 1, 8000).decodeAudioData(bytes.buffer); + entry.decodedSeconds = Math.round(decoded.duration * 1000) / 1000; + const samples = decoded.getChannelData(0); + const step = decoded.sampleRate / 4; + const labels = new Map(); + const words = []; + let before; + for (let at = 0; at + step <= samples.length; at += step) { + let energy = 0; + let crossings = 0; + for (let index = at + 1; index < at + step; index++) { + energy += samples[index] * samples[index]; + if (samples[index] >= 0 !== samples[index - 1] >= 0) crossings++; + } + // A tone of f Hz crosses zero 2f times a second, so f is twice the crossings in 0.25 s. + const voice = energy / step < 0.001 ? undefined : crossings * 2 < 500 ? "Low" : "High"; + const [start, end] = [at / decoded.sampleRate, (at + step) / decoded.sampleRate]; + if (voice && voice === before) words[words.length - 1].end = end; + else if (voice) { + if (!labels.has(voice)) labels.set(voice, labels.size); + const speaker = labels.get(voice); + words.push({ word: voice + " voice.", start, end, speaker, speaker_label: "speaker_" + speaker, confidence: 0.99 }); + } + before = voice; + } + entry.heardSpeakers = labels.size; + return { text: words.map(({ word }) => word).join(" "), words }; + }; // A request that names an address in place of a file is a few hundred bytes: multipart // with source_url for ElevenLabs, JSON with input_audio.url for OpenRouter. const namedAddress = (request) => { @@ -130,6 +166,7 @@ const WATCH = `(() => { ipc.send = function (channel, replyChannel, request, ...rest) { if (channel === "request-url" && typeof request?.url === "string") { const method = (request.method ?? "GET").toUpperCase(); + const entry = { via: "requestUrl", method, url: logged(request.url) }; const answer = request.url.startsWith("https://api.openai.com/") ? { text } : request.url.startsWith("https://api.deepgram.com/") @@ -137,17 +174,22 @@ const WATCH = `(() => { : request.url.startsWith("https://api.elevenlabs.io/") ? elevenlabs : request.url === ${jsLiteral(OPENROUTER_TRANSCRIPTIONS)} - ? openrouter + ? hearOpenRouter(request, entry) : request.url.startsWith(${jsLiteral(OPENROUTER_CATALOG)}) ? openrouterCatalog : null; if (answer) { - entries.push({ via: "requestUrl", method, url: logged(request.url), answeredBy: "fixture provider", contentType: request.contentType, uploadBytes: request.body?.byteLength ?? 0, sourceUrl: namedAddress(request) }); - const body = new TextEncoder().encode(JSON.stringify(answer)).buffer; - queueMicrotask(() => ipc.emit(replyChannel, {}, { status: 200, headers: { "content-type": "application/json" }, body })); + entries.push(Object.assign(entry, { answeredBy: "fixture provider", contentType: request.contentType, uploadBytes: request.body?.byteLength ?? 0, sourceUrl: namedAddress(request) })); + // Audio the decoder refuses is answered as OpenRouter answers audio it cannot read. + Promise.resolve(answer) + .then((json) => ({ status: 200, json }), () => { + entry.decodedSeconds = "undecodable"; + return { status: 400, json: { error: { message: "Invalid audio file", code: 400 } } }; + }) + .then(({ status, json }) => ipc.emit(replyChannel, {}, { status, headers: { "content-type": "application/json" }, body: new TextEncoder().encode(JSON.stringify(json)).buffer })); return; } - entries.push({ via: "requestUrl", method, url: logged(request.url) }); + entries.push(entry); } return originalSend.call(ipc, channel, replyChannel, request, ...rest); }; diff --git a/scripts/size-baseline.md b/scripts/size-baseline.md index 1bb93801..005dce57 100644 --- a/scripts/size-baseline.md +++ b/scripts/size-baseline.md @@ -12,7 +12,7 @@ main.js raw: 307131 bytes; budget 330000 bytes; 22869 bytes remaining main.js gzip (level 9): 97161 bytes; budget 100000 bytes; 2839 bytes remaining ``` -The gzip budget is now 106,162 bytes. On 2026-10-14 it rose from 100,000 for +The gzip budget is now 106,970 bytes. On 2026-10-14 it rose from 100,000 for the full player's Transcript panel, which the owner approved at about 1.9 KB and which measured 2,576 bytes. The coordinator set 103,000 to keep the panel's Find and Follow chip and to leave room for fixes in flight. The @@ -54,6 +54,16 @@ main.js raw: 325002 bytes; budget 330000 bytes; 4998 bytes remaining main.js gzip (level 9): 106002 bytes; budget 106162 bytes; 160 bytes remaining ``` +Later on 2026-11-14 it rose from 106,162 for the speaker reel, which matches +speaker labels across an episode's parts. The branch it builds on measured +106,065 bytes. The reel and the label merge measured 808 more. The budget rose +by those 808 bytes. With them: + +```text +main.js raw: 327446 bytes; budget 330000 bytes; 2554 bytes remaining +main.js gzip (level 9): 106873 bytes; budget 106970 bytes; 97 bytes remaining +``` + Gzip uses Node's `gzipSync` at level 9, without a filename or timestamp. Budgets use decimal bytes. The brief's 96,533-byte reference is unexplained. This report fixes the compressor instead of comparing compressed sizes across diff --git a/scripts/size.mjs b/scripts/size.mjs index c7a49ba6..ecd93c8f 100644 --- a/scripts/size.mjs +++ b/scripts/size.mjs @@ -10,7 +10,7 @@ try { if (!bundle.length) throw new Error(`${path} is empty. Run npm run build first.`); const measurements = [ { name: "raw", bytes: bundle.length, budget: 330_000 }, - { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 106_162 }, + { name: "gzip (level 9)", bytes: gzipSync(bundle, { level: 9 }).length, budget: 106_970 }, ]; const rows = measurements.map(({ name, bytes, budget }) => { const overage = Math.max(0, bytes - budget); diff --git a/scripts/size.test.ts b/scripts/size.test.ts index 32177dd8..b3f1e93f 100644 --- a/scripts/size.test.ts +++ b/scripts/size.test.ts @@ -30,7 +30,7 @@ describe("bundle budget CLI", () => { const result = check("gzip.js", randomBytes(110_000)); expect(result.status).toBe(1); expect(result.stdout).toMatch( - /gzip \(level 9\): \d+ bytes; budget 106162 bytes; OVER by \d+ bytes/, + /gzip \(level 9\): \d+ bytes; budget 106970 bytes; OVER by \d+ bytes/, ); expect(result.stdout).toContain("budget 330000 bytes; 220000 bytes remaining"); }); diff --git a/src/services/TranscriptionService.test.ts b/src/services/TranscriptionService.test.ts index e61a2d4c..2271fa03 100644 --- a/src/services/TranscriptionService.test.ts +++ b/src/services/TranscriptionService.test.ts @@ -9,7 +9,7 @@ import { mockEpisode, noticeMessages, } from "../../tests/mocks/transcriptionService"; -import { MP3_STREAMS, mp3Bytes, mp3Seconds } from "../../tests/mocks/mp3"; +import { MP3_STREAMS, mp3Bytes, mp3Seconds, voicedMp3Bytes } from "../../tests/mocks/mp3"; import { providerMock, sentBytes, sentForm, sentText } from "../../tests/mocks/providerRequestUrl"; import { FeedRepository } from "./FeedRepository"; import { TranscriptionService } from "./TranscriptionService"; @@ -1272,7 +1272,8 @@ describe("TranscriptionService", () => { }); /** Answers the catalog with `listed`, and each transcription request with `answer`. */ const openRouter = ( - answer: (request: RequestUrlParam) => Response = () => Response.json(answers), + answer: (request: RequestUrlParam) => Response | Promise = () => + Response.json(answers), listed: unknown = EVERY_MODEL, ) => providerMock.mockImplementation(async (request) => @@ -1484,26 +1485,265 @@ describe("TranscriptionService", () => { expect(note).toContain(`Part ${partBytes.length}.`); }); - test("labels from two parts never name the same speaker, and the notice says labels restart in each", async () => { - getEpisodeAudioBufferMock.mockResolvedValue(twoPartAudio()); - openRouter(() => - Response.json({ words: [word("Hello.", 1, 1.5, 0), word("Hi.", 10, 10.5, 1)] }), - ); - const plugin = createMockPlugin({ - provider: "openrouter", - openRouterKey: "openrouter-key", - speakers: true, + describe("speaker labels across the parts of an episode", () => { + const stream = MP3_STREAMS.mpeg25At8KbpsCrcPadded; + const VOICES = ["", "Ann", "Bob", "Cy"]; + const [ANN, BOB, CY] = [1, 2, 3]; + /** The stream's header with no CRC and no padding, then nothing: a frame of silence. */ + const SILENCE = [0xff, 0xe3, 0x18, 0xc0, ...Array(68).fill(0)]; + /** + * Episode frames, start inclusive and end exclusive. 417 frames are 30.024 s, and the + * second part starts at frame 22,334, 1,608.048 s in. + */ + const TURNS = [ + { voice: ANN, from: 0, to: 417 }, + { voice: BOB, from: 417, to: 834 }, + { voice: BOB, from: 22_500, to: 22_917 }, + { voice: ANN, from: 22_917, to: 23_334 }, + ]; + const episode = (turns = TURNS, frames = twoPartFrames) => ({ + buffer: voicedMp3Bytes( + stream, + frames, + (frame) => + turns.find(({ from, to }) => frame >= from && frame < to)?.voice ?? 0, + ).buffer, + extension: "mp3", }); - await expect(finalNotice(plugin)).resolves.toBe( - "OpenRouter transcribed this episode in 2 parts, and speaker labels restart in each, so one person can have several labels.", - ); + type Frame = { readonly index: number; readonly voice: number } | "silence"; + /** The frames of a request's audio. Throws at the first byte where no frame of the stream starts. */ + const framesOf = (bytes: Uint8Array): Frame[] => { + const frames: Frame[] = []; + const starts = (at: number, expected: readonly number[]) => + expected.every((byte, index) => bytes[at + index] === byte); + for (let at = 0; at < bytes.length;) { + if (starts(at, stream.header)) { + const end = at + stream.frameBytes; + frames.push({ + voice: bytes[end - 3], + index: (bytes[end - 2] << 8) | bytes[end - 1], + }); + at = end; + } else if (starts(at, SILENCE)) { + frames.push("silence"); + at += SILENCE.length; + } else { + throw new Error(`No frame of the stream starts at byte ${at}.`); + } + } + return frames; + }; + const sentAudio = async (request: RequestUrlParam): Promise => + request.contentType === "application/json" + ? Buffer.from(sentJson(request).input_audio.data, "base64") + : new Uint8Array(await (await sentFile(request)).arrayBuffer()); + /** + * Answers as a model that hears who speaks in each frame: a word for each turn, + * named for its voice, with labels numbered in the order voices first speak in this request. + */ + const heard = (frames: readonly Frame[]) => { + const labels = new Map(); + const words: ReturnType[] = []; + let voiceBefore = 0; + frames.forEach((frame, at) => { + const voice = frame === "silence" ? 0 : frame.voice; + const [start, end] = [at, at + 1].map((n) => n * stream.frameSeconds); + if (voice !== 0 && voice === voiceBefore) words[words.length - 1].end = end; + else if (voice !== 0) { + if (!labels.has(voice)) labels.set(voice, labels.size); + words.push(word(VOICES[voice], start, end, labels.get(voice))); + } + voiceBefore = voice; + }); + return { words }; + }; + /** OpenRouter answering each request as `heard` does, and the reel with `reel` when given. */ + const hearing = (reel?: (frames: readonly Frame[]) => Response) => + openRouter(async (request) => { + const frames = framesOf(await sentAudio(request)); + const isReel = frames.length * stream.frameSeconds < 60; + return isReel && reel ? reel(frames) : Response.json(heard(frames)); + }); + const spoken = (note: string) => + [...note.matchAll(/\*\*(\d+):\*\* (\w+)/g)].map(([, n, text]) => `${n} ${text}`); + const grok = () => + createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + speakers: true, + }); + const RESTART = + "OpenRouter transcribed this episode in 2 parts, and speaker labels restart in each, so one person can have several labels."; + + test("Grok STT 2.0 numbers each voice once across two parts that label them in the other order", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(episode()); + hearing(); + const plugin = grok(); + + const notice = await finalNotice(plugin); + + expect(spoken(savedNote(plugin))).toEqual(["1 Ann", "2 Bob", "2 Bob", "1 Ann"]); + expect(notice).toBe("Transcription completed and saved."); + expect(sent()).toHaveLength(3); + }); - expect(sent()).toHaveLength(2); - const labels = [...savedNote(plugin).matchAll(/\*\*(\d+):\*\* (\w+)/g)].map( - ([, speaker, text]) => `${speaker} ${text}`, + test("the reel is the stream's own frames, 9 s from 0.2 s into each voice's longest turn in each part, with 1.5 s of silent frames between clips", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(episode()); + hearing(); + + await transcribe(grok()); + + expect(sent()).toHaveLength(3); + const reel = framesOf(await sentAudio(sent()[2])); + // A run of silent frames has no first index. + const indexes = reel.map((frame) => (frame === "silence" ? -1 : frame.index)); + const runs: { first: number; frames: number }[] = []; + indexes.forEach((index, at) => { + const previous = indexes[at - 1]; + const follows = + at > 0 && (index === -1 ? previous === -1 : index === previous + 1); + if (follows) runs[runs.length - 1].frames++; + else runs.push({ first: index, frames: 1 }); + }); + const gapFrames = Math.ceil(1.5 / stream.frameSeconds); + expect(runs.map(({ first, frames }) => (first === -1 ? frames : "clip"))).toEqual([ + "clip", + gapFrames, + "clip", + gapFrames, + "clip", + gapFrames, + "clip", + ]); + // 0.2 s is 2.8 frames, so each clip starts at the third frame of its turn and holds 125. + const clips = runs + .filter(({ first }) => first !== -1) + .map(({ first, frames }) => [first, frames]) + .sort(([a], [b]) => a - b); + expect(clips).toEqual(TURNS.map(({ from }) => [from + 3, 125])); + expect(reel.length * stream.frameSeconds).toBeCloseTo( + 4 * 9 + 3 * gapFrames * 0.072, + 6, + ); + }); + + test("a voice whose turns are all under 2.6 s gets no clip, keeps its own number, and the notice says labels restart in each", async () => { + // Cy speaks 2.016 s, after Ann in the second part. + getEpisodeAudioBufferMock.mockResolvedValue( + episode([...TURNS, { voice: CY, from: 23_334, to: 23_362 }]), + ); + hearing(); + const plugin = grok(); + + const notice = await finalNotice(plugin); + + expect(spoken(savedNote(plugin))).toEqual([ + "1 Ann", + "2 Bob", + "2 Bob", + "1 Ann", + "3 Cy", + ]); + expect(notice).toBe(RESTART); + expect(sent()).toHaveLength(3); + const reelVoices = framesOf(await sentAudio(sent()[2])).map((frame) => + frame === "silence" ? 0 : frame.voice, + ); + expect(new Set(reelVoices)).toEqual(new Set([0, ANN, BOB])); + }); + + test("a clip the reel answer gives no words keeps its own number, though one speaker talks in the clips on both sides of it", async () => { + // Clips go most talk first: Ann in the first part, Cy, then Ann in the second. + getEpisodeAudioBufferMock.mockResolvedValue( + episode([ + { voice: ANN, from: 0, to: 417 }, + { voice: CY, from: 22_500, to: 22_778 }, + { voice: ANN, from: 22_778, to: 22_917 }, + ]), + ); + hearing((frames) => + Response.json( + heard( + frames.map((frame) => + frame !== "silence" && frame.voice === CY ? "silence" : frame, + ), + ), + ), + ); + const plugin = grok(); + + const notice = await finalNotice(plugin); + + expect(sent()).toHaveLength(3); + expect(spoken(savedNote(plugin))).toEqual(["1 Ann", "2 Cy", "1 Ann"]); + expect(notice).toBe(RESTART); + }); + + test("a reel OpenRouter refuses keeps the parts' labels, saves the note and says labels restart", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(episode()); + hearing(() => + Response.json( + { error: { message: "Invalid audio SENTINEL", code: 400 } }, + { status: 400 }, + ), + ); + const plugin = grok(); + + await expect(finalNotice(plugin)).resolves.toBe(RESTART); + + expect(sent()).toHaveLength(3); + expect(spoken(savedNote(plugin))).toEqual(["1 Ann", "2 Bob", "3 Bob", "4 Ann"]); + }); + + test.each(["elevenlabs/scribe-v2", "microsoft/mai-transcribe-2"])( + "%s sends no reel: labels from two parts never name the same speaker, and the notice says labels restart in each", + async (model) => { + getEpisodeAudioBufferMock.mockResolvedValue(episode()); + hearing(); + const plugin = createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + speakers: true, + model, + }); + + await expect(finalNotice(plugin)).resolves.toBe(RESTART); + + expect(sent()).toHaveLength(2); + expect(spoken(savedNote(plugin))).toEqual(["1 Ann", "2 Bob", "3 Bob", "4 Ann"]); + }, ); - expect(labels).toEqual(["1 Hello", "2 Hi", "3 Hello", "4 Hi"]); + + test("with speaker labels off, Grok STT 2.0 sends no reel, even when its answers label speakers", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(episode()); + hearing(); + const plugin = createMockPlugin({ + provider: "openrouter", + openRouterKey: "openrouter-key", + speakers: false, + }); + + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription completed and saved.", + ); + + expect(sent()).toHaveLength(2); + expect(savedNote(plugin)).not.toContain("**"); + }); + + test("an episode in one part sends no reel", async () => { + getEpisodeAudioBufferMock.mockResolvedValue(episode(TURNS, 834)); + hearing(); + const plugin = grok(); + + await expect(finalNotice(plugin)).resolves.toBe( + "Transcription completed and saved.", + ); + + expect(sent()).toHaveLength(1); + expect(spoken(savedNote(plugin))).toEqual(["1 Ann", "2 Bob"]); + }); }); test("speaker labels asked for and none returned, as from a route that ignores the option, saves the note and says so", async () => { diff --git a/src/services/transcription/cues.ts b/src/services/transcription/cues.ts index e55520f9..e83558f1 100644 --- a/src/services/transcription/cues.ts +++ b/src/services/transcription/cues.ts @@ -32,9 +32,10 @@ function failedPartMarker(index: number): string { export function transcriptFromParts( parts: readonly TranscribedPart[], speakers: boolean, + sameSpeaker: SameSpeaker = new Map(), ): ParsedTranscript | undefined { if (!parts.some(({ transcript }) => transcript && textOf(transcript).trim())) return undefined; - if (isTimed(parts)) return timed(cuesOf(parts, speakers)); + if (isTimed(parts)) return timed(cuesOf(parts, speakers, sameSpeaker)); const text = parts.map(({ transcript }, index) => transcript ? textOf(transcript) : failedPartMarker(index), ); @@ -59,12 +60,26 @@ function toMilliseconds(seconds: number): number { return Math.round(seconds * 1000) / 1000; } -/** Speakers are numbered in the order they first speak. A provider's label names a speaker only within its own part. */ -function cuesOf(parts: readonly TimedPart[], speakers: boolean): TranscriptCue[] { +/** Names a speaker label of one part. A provider's label names a speaker only within its own part. */ +export function speakerKey(part: number, label: string): string { + return `${part}:${label}`; +} + +/** Speaker keys of labels heard as one person, each mapped to the key that stands for them all. */ +export type SameSpeaker = ReadonlyMap; + +/** Speakers are numbered in the order they first speak. */ +function cuesOf( + parts: readonly TimedPart[], + speakers: boolean, + sameSpeaker: SameSpeaker, +): TranscriptCue[] { const numbers = new Map(); - const numberOf = (label: string) => { - const number = numbers.get(label) ?? `${numbers.size + 1}`; - numbers.set(label, number); + const numberOf = (part: number, label: string) => { + const key = speakerKey(part, label); + const speaker = sameSpeaker.get(key) ?? key; + const number = numbers.get(speaker) ?? `${numbers.size + 1}`; + numbers.set(speaker, number); return number; }; return parts.flatMap(({ start, transcript }, index): TranscriptCue[] => @@ -74,7 +89,7 @@ function cuesOf(parts: readonly TimedPart[], speakers: boolean): TranscriptCue[] end: toMilliseconds(start + word.end), speaker: speakers && word.speaker !== undefined - ? numberOf(`${index}:${word.speaker}`) + ? numberOf(index, word.speaker) : undefined, text: word.text, })) diff --git a/src/services/transcription/providers/openrouter.ts b/src/services/transcription/providers/openrouter.ts index bf96d5b3..594c4946 100644 --- a/src/services/transcription/providers/openrouter.ts +++ b/src/services/transcription/providers/openrouter.ts @@ -80,6 +80,7 @@ const MODELS: NonEmpty = [ speakers: true, url: false, ...PART_CAPS, + mergeSpeakers: true, body: grok, }, { @@ -88,6 +89,8 @@ const MODELS: NonEmpty = [ speakers: true, url: true, ...PART_CAPS, + // Its reel heard 4 voices on a show with 10. + mergeSpeakers: false, body: topLevelDiarize, }, { @@ -96,6 +99,7 @@ const MODELS: NonEmpty = [ speakers: true, url: false, ...PART_CAPS, + mergeSpeakers: false, body: topLevelDiarize, }, ]; diff --git a/src/services/transcription/providers/types.ts b/src/services/transcription/providers/types.ts index ccca60b7..bdbcb62b 100644 --- a/src/services/transcription/providers/types.ts +++ b/src/services/transcription/providers/types.ts @@ -16,6 +16,12 @@ export interface ModelProfile { readonly maxUploadBytes: number; /** The most audio, in seconds, one request may carry. */ readonly maxSeconds?: number; + /** + * Matches speaker labels across an episode's parts with one more request, a minute or two + * of audio that holds a clip of each part's every voice. Only where measured, because a + * model can hear different voices in it as one. + */ + readonly mergeSpeakers?: boolean; } export interface Word { diff --git a/src/services/transcription/speakers.ts b/src/services/transcription/speakers.ts new file mode 100644 index 00000000..76cc6e1f --- /dev/null +++ b/src/services/transcription/speakers.ts @@ -0,0 +1,99 @@ +import { type SameSpeaker, speakerKey, type TranscribedPart } from "./cues"; +import type { PartTranscript, Word } from "./providers/types"; +import { cutReel, type Span } from "./split"; + +/** A shorter turn makes a clip too short for a model to know the voice. */ +const MIN_TURN_SECONDS = 2.6; +/** A clip leaves out this much at the ends of its turn, where word times are least sure. */ +const CLIP_INSET_SECONDS = 0.2; +const MAX_CLIP_SECONDS = 9; + +/** A speaker label's words in a row. Words without a label make turns of their own. */ +type Turn = Span & { readonly speaker?: string }; + +function turnsOf(words: readonly Word[]): Turn[] { + const turns: Turn[] = []; + for (const { speaker, start, end } of words) { + const last = turns[turns.length - 1]; + if (last && last.speaker === speaker) turns[turns.length - 1] = { ...last, end }; + else turns.push({ speaker, start, end }); + } + return turns; +} + +/** + * The merge reel, one MP3 with a clip of each part's every voice from its longest turn, for + * the model that labeled the parts to label again in one request. `match` reads that answer. + * Part labels whose clips it gives one speaker are one person, and a label with no clip + * stays its own. Undefined when fewer than two labels have a clip or the audio is not MP3. + */ +export function speakerReel( + parts: readonly TranscribedPart[], + audio: { readonly buffer: ArrayBuffer; readonly extension: string }, +): { readonly file: File; match(answer: PartTranscript): SameSpeaker } | undefined { + const voices = new Map(); + parts.forEach(({ start, transcript }, part) => { + if (!transcript || !("words" in transcript)) return; + for (const turn of turnsOf(transcript.words)) { + if (turn.speaker === undefined) continue; + const key = speakerKey(part, turn.speaker); + const voice = voices.get(key) ?? { talk: 0 }; + const seconds = turn.end - turn.start; + voice.talk += seconds; + if ( + seconds >= MIN_TURN_SECONDS && + (!voice.longest || seconds > voice.longest.end - voice.longest.start) + ) { + voice.longest = { start: start + turn.start, end: start + turn.end }; + } + voices.set(key, voice); + } + }); + // Most talk first, the order the reel was measured in. + const clipped = [...voices] + .flatMap(([key, { talk, longest }]) => (longest ? [{ key, talk, longest }] : [])) + .sort((a, b) => b.talk - a.talk); + if (clipped.length < 2) return undefined; + + const reel = cutReel( + audio, + clipped.map(({ longest }) => ({ + start: longest.start + CLIP_INSET_SECONDS, + end: Math.min( + longest.end - CLIP_INSET_SECONDS, + longest.start + CLIP_INSET_SECONDS + MAX_CLIP_SECONDS, + ), + })), + ); + if (!reel) return undefined; + + return { + file: reel.file, + match(answer) { + // Words and not turns, because one speaker's turn can span a clip the model gave no words. + const heard = "words" in answer ? answer.words : []; + const standsFor = new Map(); + const same = new Map(); + reel.clips.forEach((clip, index) => { + const credit = new Map(); + for (const { speaker, start, end } of heard) { + const overlap = Math.min(end, clip.end) - Math.max(start, clip.start); + if (speaker !== undefined && overlap > 0) { + credit.set(speaker, (credit.get(speaker) ?? 0) + overlap); + } + } + let speaker: string | undefined; + let most = 0; + for (const [label, seconds] of credit) { + if (seconds > most) [speaker, most] = [label, seconds]; + } + if (speaker === undefined) return; + const { key } = clipped[index]; + const first = standsFor.get(speaker) ?? key; + standsFor.set(speaker, first); + same.set(key, first); + }); + return same; + }, + }; +} diff --git a/src/services/transcription/split.ts b/src/services/transcription/split.ts index f11ff12f..7eeba639 100644 --- a/src/services/transcription/split.ts +++ b/src/services/transcription/split.ts @@ -74,6 +74,78 @@ export async function splitAudio( return parts; } +/** A stretch of audio, in seconds. */ +export interface Span { + readonly start: number; + readonly end: number; +} + +const REEL_GAP_SECONDS = 1.5; + +/** + * One MP3 of `clips` of an episode's MP3, in order, with 1.5 s of silence between two, and + * where each clip sits in it. A clip starts and ends at the first frame start at or after + * its own. Undefined for audio that is not MP3. + */ +export function cutReel( + audio: { readonly buffer: ArrayBuffer; readonly extension: string }, + clips: readonly Span[], +): { file: File; clips: Span[] } | undefined { + const bytes = new Uint8Array(audio.buffer); + const frames = getMimeType(audio.extension) === "audio/mp3" ? mp3Frames(bytes) : []; + if (frames.length === 0) return undefined; + + // Where each frame starts, then where the last one ends. + const edges: { offset: number; seconds: number }[] = []; + let elapsed = 0; + for (const frame of frames) { + edges.push({ offset: frame.offset, seconds: elapsed }); + elapsed += frame.seconds; + } + const last = frames[frames.length - 1]; + const end = { offset: last.offset + last.length, seconds: elapsed }; + edges.push(end); + // A frame's start is a sum of frame lengths, so it can fall short of its time by float noise. + const edgeAt = (seconds: number) => edges.find((edge) => edge.seconds + 1e-6 >= seconds) ?? end; + + const silence = silentFrames(bytes, frames[Math.floor(frames.length / 2)], REEL_GAP_SECONDS); + const pieces: Uint8Array[] = []; + const placed: Span[] = []; + let at = 0; + for (const clip of clips) { + if (placed.length > 0) { + pieces.push(silence.bytes); + at += silence.seconds; + } + const from = edgeAt(clip.start); + const to = edgeAt(clip.end); + pieces.push(bytes.subarray(from.offset, to.offset)); + placed.push({ start: at, end: at + to.seconds - from.seconds }); + at += to.seconds - from.seconds; + } + return { file: new File(pieces, "audio.reel.mp3", { type: "audio/mp3" }), clips: placed }; +} + +/** + * Frames of silence for at least `seconds`. Each is `frame`'s header with the protection bit + * set, so no CRC follows, and the padding bit cleared, then zeros, which a decoder reads as + * a frame with nothing to play. + */ +function silentFrames( + bytes: Uint8Array, + frame: Mp3Frame, + seconds: number, +): { bytes: Uint8Array; seconds: number } { + const header = bytes.slice(frame.offset, frame.offset + 4); + const length = frame.length - ((header[2] >> 1) & 1); + header[1] |= 1; + header[2] &= ~2; + const count = Math.ceil(seconds / frame.seconds); + const silence = new Uint8Array(length * count); + for (let index = 0; index < count; index++) silence.set(header, index * length); + return { bytes: silence, seconds: count * frame.seconds }; +} + interface Mp3Frame { offset: number; length: number; diff --git a/src/services/transcription/transcribe.ts b/src/services/transcription/transcribe.ts index a1c3b21d..c886610b 100644 --- a/src/services/transcription/transcribe.ts +++ b/src/services/transcription/transcribe.ts @@ -3,10 +3,11 @@ import type { FeedRepository } from "src/services/FeedRepository"; import type { ParsedTranscript } from "src/services/publishedTranscript/parse"; import type { Episode } from "src/types/Episode"; import { publicAudioUrl } from "./audioSource"; -import { type TranscribedPart, transcriptFromParts } from "./cues"; +import { type SameSpeaker, speakerKey, type TranscribedPart, transcriptFromParts } from "./cues"; import { createSender, failure, type FailureKind, TranscriptionError } from "./http"; import type { AudioInput } from "./providers/types"; import { PROVIDERS, type ProviderChoice, providersWithSpeakers } from "./registry"; +import { speakerReel } from "./speakers"; import { splitAudio } from "./split"; export interface TranscriptionJob { @@ -51,6 +52,7 @@ export async function transcribe(job: TranscriptionJob): Promise { const parts: TranscribedPart[] = []; let lastFailure: TranscriptionError | undefined; + let sameSpeaker: SameSpeaker | undefined; const url = model.url ? await publicAudioUrl(episode, job.feeds) : undefined; if (url) { @@ -89,9 +91,23 @@ export async function transcribe(job: TranscriptionJob): Promise { parts.push({ start }); } } + + const reel = + speakers && model.mergeSpeakers && files.length > 1 + ? speakerReel(parts, audio) + : undefined; + if (reel) { + onProgress(`${progress} matching speakers across parts`); + try { + sameSpeaker = reel.match(await transcribeOne({ file: reel.file })); + } catch (error) { + // The parts' labels stay apart, and the warning below says so. + if (!(error instanceof TranscriptionError)) throw error; + } + } } - const content = transcriptFromParts(parts, speakers); + const content = transcriptFromParts(parts, speakers, sameSpeaker); if (!content) throw lastFailure ?? failure("empty", context); const warnings: string[] = []; @@ -116,17 +132,21 @@ export async function transcribe(job: TranscriptionJob): Promise { `Saved without speaker labels, because ${choice.label} does not label speakers. These providers do: ${providersWithSpeakers().join(", ")}.`, ); } - // A request routed where the speaker option is ignored is answered without speakers. - const labeled = parts.some( - ({ transcript }) => - transcript && - "words" in transcript && - transcript.words.some((word) => word.speaker !== undefined), + const labels = new Set( + parts.flatMap(({ transcript }, part) => + transcript && "words" in transcript + ? transcript.words.flatMap(({ speaker }) => + speaker === undefined ? [] : [speakerKey(part, speaker)], + ) + : [], + ), ); - if (speakers && !labeled) { + // A request routed where the speaker option is ignored is answered without speakers. + if (speakers && labels.size === 0) { warnings.push(`Saved without speaker labels, because ${model.label} returned none.`); } - if (speakers && labeled && parts.length > 1) { + // A label the reel did not hear can name a voice that another part labels too. + if (speakers && parts.length > 1 && [...labels].some((label) => !sameSpeaker?.has(label))) { warnings.push( `${choice.label} transcribed this episode in ${parts.length} parts, and speaker labels restart in each, so one person can have several labels.`, ); diff --git a/tests/mocks/mp3.ts b/tests/mocks/mp3.ts index 180ee1d2..b1a2bc13 100644 --- a/tests/mocks/mp3.ts +++ b/tests/mocks/mp3.ts @@ -6,6 +6,12 @@ export const MP3_STREAMS = { mpeg1At320Kbps: { header: [0xff, 0xfb, 0xe4, 0xc4], frameBytes: 960, frameSeconds: 0.024 }, /** MPEG-2.5, 8 kbps, 8 kHz, mono: an hour fits in 3.6 MB. */ mpeg25At8Kbps: { header: [0xff, 0xe3, 0x18, 0xc0], frameBytes: 72, frameSeconds: 0.072 }, + /** The same, with a CRC and a padding byte in every frame. */ + mpeg25At8KbpsCrcPadded: { + header: [0xff, 0xe2, 0x1a, 0xc0], + frameBytes: 73, + frameSeconds: 0.072, + }, } as const; export type Mp3Stream = (typeof MP3_STREAMS)[keyof typeof MP3_STREAMS]; @@ -17,6 +23,23 @@ export function mp3Bytes(stream: Mp3Stream, frames: number): Uint8Array { return bytes; } +/** + * `frames` frames of `stream` whose last three bytes hold the voice `voiceOf` gives the + * frame (0 for none) and the frame's index, so a test can hear who speaks in a request and + * tell which of the episode's frames it holds. + */ +export function voicedMp3Bytes( + stream: Mp3Stream, + frames: number, + voiceOf: (frame: number) => number, +): Uint8Array { + const bytes = mp3Bytes(stream, frames); + for (let frame = 0; frame < frames; frame++) { + bytes.set([voiceOf(frame), frame >> 8, frame & 0xff], (frame + 1) * stream.frameBytes - 3); + } + return bytes; +} + /** The seconds of audio in `bytes` of `stream`, counted from its fixed frame size. */ export function mp3Seconds(stream: Mp3Stream, bytes: number): number { return (bytes / stream.frameBytes) * stream.frameSeconds;