diff --git a/app/api/songs/analyze/route.ts b/app/api/songs/analyze/route.ts index fe8557e1d..e041afe2b 100644 --- a/app/api/songs/analyze/route.ts +++ b/app/api/songs/analyze/route.ts @@ -3,6 +3,14 @@ import { NextResponse } from "next/server"; import { getCorsHeaders } from "@/lib/networking/getCorsHeaders"; import { postFlamingoGenerateHandler } from "@/lib/flamingo/postFlamingoGenerateHandler"; +/** + * `full_report` runs its 13 sections as one batch, one after another on one + * model container, so it can outlast the default function duration. Kept + * above the Modal endpoint's 600 s timeout so the API never abandons a report + * the model finishes. + */ +export const maxDuration = 800; + /** * OPTIONS handler for CORS preflight requests. * diff --git a/app/mcp/route.ts b/app/mcp/route.ts index aa89bd048..3d81f79da 100644 --- a/app/mcp/route.ts +++ b/app/mcp/route.ts @@ -12,8 +12,9 @@ import { createOAuthToolServices } from "@/lib/mcp/oauth/createOAuthToolServices import { validateMcpJsonBody } from "@/lib/mcp/validateMcpJsonBody"; import { verifyOAuthBearer } from "@/lib/oauth/verifyOAuthBearer"; -// Site generation can take several minutes, matching the HTTP generation route. -export const maxDuration = 300; +// Site generation and analyze_music's full_report (13 sections run one after +// another in one model batch) can take several minutes, matching their HTTP routes. +export const maxDuration = 800; const baseHandler = createMcpHandler( server => { diff --git a/lib/contracts/fixtures/published-docs.json b/lib/contracts/fixtures/published-docs.json index 6057cd178..ae206f818 100644 --- a/lib/contracts/fixtures/published-docs.json +++ b/lib/contracts/fixtures/published-docs.json @@ -1233,13 +1233,13 @@ "content/docs/source/api-reference/openapi/accounts.json": "1a5a04f9ec358a15b8339b7029c9fed5fe69c6f042c8b03b6b4f03f3aafdeeed", "content/docs/source/api-reference/openapi/content.json": "f790ee44d1b881709e393a65500bec23493e21a391c7b66555e0c78750f95239", "content/docs/source/api-reference/openapi/research.json": "f301a714e994e99b1dbcfc63ee18824537cf42813d8df3906ac7c2436a481ae0", - "content/docs/source/api-reference/openapi/releases.json": "85b1ca7af667bac8a8f67dd04b0bd986cbd03f8bd287066401de2132d825ace6", + "content/docs/source/api-reference/openapi/releases.json": "03be8efc610d3625bbfc3cf4e71723193e31c4d1cab9b1216b1fd9b3622ab8e4", "content/docs/source/api-reference/openapi/social.json": "5bd167a78cf6a17c46724486e2921882be20969ea32f3c0297f5b7e7ba9a30aa", "content/docs/source/api-reference/openapi/context.json": "7784ee1d8c4b6753cc47bf518f468effa58d50fb9d0289009edf0caad3d05a67", "content/docs/source/api-reference/openapi/projects.json": "b059cf8baa83a7f73a3c15db19f54fa56e46a3840e91bd42979b83550f543ace", "content/docs/source/api-reference/openapi/sessions.json": "85af56cc150374b28c25541c45db15849c71fdeb6288cf90dc2beb08f494e83e", "content/docs/source/api-reference/sites/overview.mdx": "1430730142df476e6cec9ea40aab751b0a48373d1479254da70bd8a545ae51b2", - "content/docs/source/credits.mdx": "2a0cdf8ddd408b7a7ebfe0abb0e185a23c66af4035790223c4f5f595c41a9b7c", + "content/docs/source/credits.mdx": "8f56d2221e238f4f0deafe0b516ac29a0afa6124fd58ccabfc96d029d58286f5", "content/docs/source/api-reference/admins/agent-signups.mdx": "a6b65f3ad1b0ead873b83cdccf2254cdd3afa1556ae94738502c9b1fc46954bc" }, "examples": { diff --git a/lib/flamingo/__tests__/callFlamingoGenerateBatch.test.ts b/lib/flamingo/__tests__/callFlamingoGenerateBatch.test.ts index dea5bbcbd..09c73c64d 100644 --- a/lib/flamingo/__tests__/callFlamingoGenerateBatch.test.ts +++ b/lib/flamingo/__tests__/callFlamingoGenerateBatch.test.ts @@ -62,6 +62,60 @@ describe("callFlamingoGenerateBatch", () => { }); }); + it("sends a prompt's clip window and its own settings only when the prompt has them", async () => { + mockFetch.mockResolvedValue({ ok: true, json: async () => RESULT }); + + await callFlamingoGenerateBatch({ + ...BODY, + prompts: [ + { + id: "climax", + prompt: "Describe this clip.", + max_new_tokens: 256, + clip: { start: 143.36, end: 158.1 }, + }, + { + id: "pitch", + prompt: "Pitch it.", + max_new_tokens: 512, + temperature: 0.6, + top_p: 0.9, + do_sample: true, + }, + { id: "key", prompt: "Key?", max_new_tokens: 32 }, + ], + }); + + expect(JSON.parse(mockFetch.mock.calls[0][1].body).prompts).toEqual([ + { + id: "climax", + prompt: "Describe this clip.", + max_new_tokens: 256, + clip_start: 143.36, + clip_end: 158.1, + }, + { + id: "pitch", + prompt: "Pitch it.", + max_new_tokens: 512, + temperature: 0.6, + top_p: 0.9, + do_sample: true, + }, + { id: "key", prompt: "Key?", max_new_tokens: 32 }, + ]); + }); + + it("accepts an answer that echoes both ends of its clip", async () => { + const clipped = { + answers: [{ id: "a", response: "x", elapsed_seconds: 1, clip_start: 2, clip_end: 9.5 }], + elapsed_seconds: 1, + }; + mockFetch.mockResolvedValue({ ok: true, json: async () => clipped }); + + await expect(callFlamingoGenerateBatch(BODY)).resolves.toEqual(clipped); + }); + it("passes an abort signal through when given one", async () => { mockFetch.mockResolvedValue({ ok: true, json: async () => RESULT }); const controller = new AbortController(); @@ -117,6 +171,20 @@ describe("callFlamingoGenerateBatch", () => { { answers: [{ id: "a", response: 42, elapsed_seconds: 1 }], elapsed_seconds: 1 }, ], ["an answer without seconds", { answers: [{ id: "a", response: "x" }], elapsed_seconds: 1 }], + [ + "an answer with only one end of its clip", + { + answers: [{ id: "a", response: "x", elapsed_seconds: 1, clip_start: 2 }], + elapsed_seconds: 1, + }, + ], + [ + "an answer whose clip end is not a number", + { + answers: [{ id: "a", response: "x", elapsed_seconds: 1, clip_start: 2, clip_end: "9" }], + elapsed_seconds: 1, + }, + ], ["null", null], ])("throws on an unexpected response shape: %s", async (_name, payload) => { mockFetch.mockResolvedValue({ ok: true, json: async () => payload }); diff --git a/lib/flamingo/__tests__/chargeForBatchAnswers.test.ts b/lib/flamingo/__tests__/chargeForBatchAnswers.test.ts new file mode 100644 index 000000000..8bd006412 --- /dev/null +++ b/lib/flamingo/__tests__/chargeForBatchAnswers.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { chargeForBatchAnswers } from "../chargeForBatchAnswers"; +import { chargeForFlamingoBatch } from "@/lib/flamingo/chargeForFlamingoBatch"; + +vi.mock("@/lib/flamingo/chargeForFlamingoBatch", () => ({ chargeForFlamingoBatch: vi.fn() })); + +describe("chargeForBatchAnswers", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(chargeForFlamingoBatch).mockResolvedValue(undefined); + }); + + it("charges one batch on the seconds of the answers that were used", async () => { + await chargeForBatchAnswers({ + accountId: "acc_1", + answers: [{ elapsed_seconds: 4.5 }, { elapsed_seconds: 2.25 }], + audioUrl: "https://example.com/song.mp3", + }); + + expect(chargeForFlamingoBatch).toHaveBeenCalledTimes(1); + expect(chargeForFlamingoBatch).toHaveBeenCalledWith({ + accountId: "acc_1", + elapsedSeconds: 6.75, + promptCount: 2, + audioUrl: "https://example.com/song.mp3", + }); + }); + + it("charges nothing when no answer was usable", async () => { + await chargeForBatchAnswers({ accountId: "acc_1", answers: [] }); + + expect(chargeForFlamingoBatch).not.toHaveBeenCalled(); + }); + + it("leaves the audio URL off the usage record when none is given", async () => { + await chargeForBatchAnswers({ accountId: "acc_1", answers: [{ elapsed_seconds: 3 }] }); + + expect(chargeForFlamingoBatch).toHaveBeenCalledWith({ + accountId: "acc_1", + elapsedSeconds: 3, + promptCount: 1, + }); + }); +}); diff --git a/lib/flamingo/__tests__/executeFullReport.failed.test.ts b/lib/flamingo/__tests__/executeFullReport.failed.test.ts index ba0ef1ba4..5f273654b 100644 --- a/lib/flamingo/__tests__/executeFullReport.failed.test.ts +++ b/lib/flamingo/__tests__/executeFullReport.failed.test.ts @@ -1,12 +1,15 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { executeFullReport } from "../executeFullReport"; -import { chargeForFlamingoCall } from "@/lib/flamingo/chargeForFlamingoCall"; -import { callFlamingoGenerate } from "@/lib/flamingo/callFlamingoGenerate"; +import { chargeForBatchAnswers } from "@/lib/flamingo/chargeForBatchAnswers"; +import { callFlamingoGenerateBatch } from "@/lib/flamingo/callFlamingoGenerateBatch"; import { getPreset } from "@/lib/flamingo/presets"; import { FULL_REPORT_SECTIONS } from "@/lib/flamingo/presets/fullReport"; +import type { FlamingoBatchAnswer } from "@/lib/flamingo/isFlamingoGenerateBatchResult"; -vi.mock("@/lib/flamingo/callFlamingoGenerate", () => ({ callFlamingoGenerate: vi.fn() })); -vi.mock("@/lib/flamingo/chargeForFlamingoCall", () => ({ chargeForFlamingoCall: vi.fn() })); +vi.mock("@/lib/flamingo/callFlamingoGenerateBatch", () => ({ + callFlamingoGenerateBatch: vi.fn(), +})); +vi.mock("@/lib/flamingo/chargeForBatchAnswers", () => ({ chargeForBatchAnswers: vi.fn() })); vi.mock("@/lib/flamingo/presets", () => ({ getPreset: vi.fn() })); const AUDIO = "https://example.com/song.mp3"; @@ -19,43 +22,86 @@ const plainPreset = (name: string) => ({ describe("executeFullReport — failed sections", () => { beforeEach(() => { vi.clearAllMocks(); - vi.mocked(chargeForFlamingoCall).mockResolvedValue(undefined); + vi.mocked(chargeForBatchAnswers).mockResolvedValue(undefined); vi.mocked(getPreset).mockImplementation(name => plainPreset(name) as never); vi.spyOn(console, "error").mockImplementation(() => {}); }); - it("lists no failed sections when every call succeeds", async () => { - vi.mocked(callFlamingoGenerate).mockResolvedValue({ response: "ok", elapsed_seconds: 1 }); + it("lists no failed sections when every section answers", async () => { + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: FULL_REPORT_SECTIONS.map(s => ({ + id: s.reportKey, + response: "ok", + elapsed_seconds: 1, + })), + elapsed_seconds: 14, + }); const { failed } = await executeFullReport(AUDIO, "acc_1"); expect(failed).toEqual([]); }); - it("lists the report keys of the sections that failed, in report order", async () => { - const failing = new Set([FULL_REPORT_SECTIONS[0].preset, FULL_REPORT_SECTIONS[4].preset]); - vi.mocked(callFlamingoGenerate).mockImplementation(async ({ prompt }) => { - const name = prompt.replace("prompt for ", ""); - if (failing.has(name)) throw new Error("Music analysis failed (status 500)"); - return { response: "ok", elapsed_seconds: 1 }; + it("lists a section the batch returned no answer for, in report order", async () => { + const missing = new Set([FULL_REPORT_SECTIONS[0].reportKey, FULL_REPORT_SECTIONS[4].reportKey]); + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: FULL_REPORT_SECTIONS.filter(s => !missing.has(s.reportKey)).map(s => ({ + id: s.reportKey, + response: "ok", + elapsed_seconds: 1, + })), + elapsed_seconds: 12, }); const { report, failed } = await executeFullReport(AUDIO, "acc_1"); expect(failed).toEqual([FULL_REPORT_SECTIONS[0].reportKey, FULL_REPORT_SECTIONS[4].reportKey]); expect(report[FULL_REPORT_SECTIONS[0].reportKey]).toEqual({ - error: "Section failed: Music analysis failed (status 500)", + error: "Section failed: the model returned no answer for this section", }); }); - it("lists every section when the model fails every call", async () => { - vi.mocked(callFlamingoGenerate).mockRejectedValue( - new Error("Music analysis failed (status 500)"), + it("counts a section whose parsed answer is only an { error } as failed and does not charge it", async () => { + // A model reply of {"error": "..."} parses into the same shape the report + // uses for a failed section, so it is reported as failed and not billed. + vi.mocked(getPreset).mockImplementation(name => + name === FULL_REPORT_SECTIONS[1].preset + ? ({ ...plainPreset(name), parseResponse: (raw: string) => JSON.parse(raw) } as never) + : (plainPreset(name) as never), ); + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: FULL_REPORT_SECTIONS.map((s, i) => ({ + id: s.reportKey, + response: i === 1 ? '{"error": "cannot analyze"}' : "ok", + elapsed_seconds: 1, + })), + elapsed_seconds: 14, + }); const { failed } = await executeFullReport(AUDIO, "acc_1"); + expect(failed).toEqual([FULL_REPORT_SECTIONS[1].reportKey]); + const [charge] = vi.mocked(chargeForBatchAnswers).mock.calls[0]; + expect((charge.answers as FlamingoBatchAnswer[]).map(a => a.id)).toEqual( + FULL_REPORT_SECTIONS.filter((_, i) => i !== 1).map(s => s.reportKey), + ); + }); + + it("fails every section, with the batch's error, and charges nothing when the batch fails", async () => { + vi.mocked(callFlamingoGenerateBatch).mockRejectedValue( + new Error("Music analysis failed (status 500)"), + ); + + const { report, failed } = await executeFullReport(AUDIO, "acc_1"); + expect(failed).toEqual(FULL_REPORT_SECTIONS.map(s => s.reportKey)); - expect(chargeForFlamingoCall).not.toHaveBeenCalled(); + expect(report[FULL_REPORT_SECTIONS[2].reportKey]).toEqual({ + error: "Section failed: Music analysis failed (status 500)", + }); + expect(chargeForBatchAnswers).toHaveBeenCalledWith({ + accountId: "acc_1", + answers: [], + audioUrl: AUDIO, + }); }); }); diff --git a/lib/flamingo/__tests__/executeFullReport.test.ts b/lib/flamingo/__tests__/executeFullReport.test.ts index 8f62a1b25..8d5363f0e 100644 --- a/lib/flamingo/__tests__/executeFullReport.test.ts +++ b/lib/flamingo/__tests__/executeFullReport.test.ts @@ -1,12 +1,15 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { executeFullReport } from "../executeFullReport"; -import { chargeForFlamingoCall } from "@/lib/flamingo/chargeForFlamingoCall"; -import { callFlamingoGenerate } from "@/lib/flamingo/callFlamingoGenerate"; +import { chargeForBatchAnswers } from "@/lib/flamingo/chargeForBatchAnswers"; +import { callFlamingoGenerateBatch } from "@/lib/flamingo/callFlamingoGenerateBatch"; import { getPreset } from "@/lib/flamingo/presets"; import { FULL_REPORT_SECTIONS } from "@/lib/flamingo/presets/fullReport"; +import type { FlamingoBatchAnswer } from "@/lib/flamingo/isFlamingoGenerateBatchResult"; -vi.mock("@/lib/flamingo/callFlamingoGenerate", () => ({ callFlamingoGenerate: vi.fn() })); -vi.mock("@/lib/flamingo/chargeForFlamingoCall", () => ({ chargeForFlamingoCall: vi.fn() })); +vi.mock("@/lib/flamingo/callFlamingoGenerateBatch", () => ({ + callFlamingoGenerateBatch: vi.fn(), +})); +vi.mock("@/lib/flamingo/chargeForBatchAnswers", () => ({ chargeForBatchAnswers: vi.fn() })); vi.mock("@/lib/flamingo/presets", () => ({ getPreset: vi.fn() })); const AUDIO = "https://example.com/song.mp3"; @@ -16,49 +19,65 @@ const plainPreset = (name: string) => ({ prompt: `prompt for ${name}`, params: { max_new_tokens: 64, temperature: 0.2, do_sample: false }, }); +/** One answer per section, keyed by report key, each taking `seconds`. */ +const answersFor = (seconds: number) => + FULL_REPORT_SECTIONS.map(s => ({ id: s.reportKey, response: "ok", elapsed_seconds: seconds })); -describe("executeFullReport — credits", () => { +describe("executeFullReport — one batch", () => { beforeEach(() => { vi.clearAllMocks(); - vi.mocked(chargeForFlamingoCall).mockResolvedValue(undefined); + vi.mocked(chargeForBatchAnswers).mockResolvedValue(undefined); vi.mocked(getPreset).mockImplementation(name => plainPreset(name) as never); }); - it("charges one deduction per section, each on that section's own seconds", async () => { - let n = 0; - vi.mocked(callFlamingoGenerate).mockImplementation(async () => ({ - response: "ok", - elapsed_seconds: ++n, - })); + it("asks every section in one batch on one download, each with its preset's own settings", async () => { + vi.mocked(getPreset).mockImplementation(name => + name === "song_description" + ? ({ + ...plainPreset(name), + params: { max_new_tokens: 256, temperature: 0.7, do_sample: true }, + } as never) + : (plainPreset(name) as never), + ); + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: answersFor(2), + elapsed_seconds: 30, + }); - const { report } = await executeFullReport(AUDIO, "acc_1"); + await executeFullReport(AUDIO, "acc_1"); - expect(Object.keys(report)).toHaveLength(N); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(N); - const seconds = vi - .mocked(chargeForFlamingoCall) - .mock.calls.map(([args]) => args.elapsedSeconds) - .sort((a, b) => a - b); - expect(seconds).toEqual(FULL_REPORT_SECTIONS.map((_, i) => i + 1)); - for (const [args] of vi.mocked(chargeForFlamingoCall).mock.calls) { - expect(args).toMatchObject({ accountId: "acc_1", audioUrl: AUDIO }); - } + expect(callFlamingoGenerateBatch).toHaveBeenCalledTimes(1); + const [batch] = vi.mocked(callFlamingoGenerateBatch).mock.calls[0]; + expect(batch).toMatchObject({ audio_url: AUDIO, top_p: 1, do_sample: false }); + expect(batch.prompts.map(p => p.id)).toEqual(FULL_REPORT_SECTIONS.map(s => s.reportKey)); + expect(batch.prompts.find(p => p.id === "description")).toEqual({ + id: "description", + prompt: "prompt for song_description", + max_new_tokens: 256, + temperature: 0.7, + do_sample: true, + }); + expect(batch.prompts.find(p => p.id === "overview")).toEqual({ + id: "overview", + prompt: "prompt for catalog_metadata", + max_new_tokens: 64, + temperature: 0.2, + do_sample: false, + }); }); - it("does not charge for a section whose model call failed", async () => { - let n = 0; - vi.mocked(callFlamingoGenerate).mockImplementation(async () => { - n += 1; - if (n === 1) throw new Error("Music analysis failed (status 503)"); - return { response: "ok", elapsed_seconds: 3 }; - }); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + it("assembles the report in narrative order and charges one batch for every section", async () => { + const answers = answersFor(2).reverse(); + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ answers, elapsed_seconds: 30 }); - const { report } = await executeFullReport(AUDIO, "acc_1"); + const { report, failed } = await executeFullReport(AUDIO, "acc_1"); - expect(Object.keys(report)).toHaveLength(N); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(N - 1); - errorSpy.mockRestore(); + expect(Object.keys(report)).toEqual(FULL_REPORT_SECTIONS.map(s => s.reportKey)); + expect(failed).toEqual([]); + expect(chargeForBatchAnswers).toHaveBeenCalledTimes(1); + const [charge] = vi.mocked(chargeForBatchAnswers).mock.calls[0]; + expect(charge).toMatchObject({ accountId: "acc_1", audioUrl: AUDIO }); + expect(charge.answers).toHaveLength(N); }); it("does not charge for a section whose response failed to parse", async () => { @@ -74,33 +93,33 @@ describe("executeFullReport — credits", () => { } as never) : (plainPreset(name) as never), ); - vi.mocked(callFlamingoGenerate).mockImplementation(async ({ prompt }) => ({ - response: "ok", - elapsed_seconds: prompt === plainPreset(FULL_REPORT_SECTIONS[0].preset).prompt ? 17 : 3, - })); + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: answersFor(3), + elapsed_seconds: 45, + }); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const { report } = await executeFullReport(AUDIO, "acc_1"); - expect(report[FULL_REPORT_SECTIONS[0].reportKey]).toMatchObject({ - error: expect.stringContaining("Section failed"), + expect(report[FULL_REPORT_SECTIONS[0].reportKey]).toEqual({ + error: "Section failed: not JSON", }); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(N - 1); - expect(vi.mocked(chargeForFlamingoCall).mock.calls.map(([args]) => args)).toEqual( - Array.from({ length: N - 1 }, () => ({ - accountId: "acc_1", - elapsedSeconds: 3, - audioUrl: AUDIO, - })), + const [charge] = vi.mocked(chargeForBatchAnswers).mock.calls[0]; + expect((charge.answers as FlamingoBatchAnswer[]).map(a => a.id)).toEqual( + FULL_REPORT_SECTIONS.slice(1).map(s => s.reportKey), ); errorSpy.mockRestore(); }); - it("does not persist temporary attachment URLs in any section's usage record", async () => { - vi.mocked(callFlamingoGenerate).mockResolvedValue({ response: "ok", elapsed_seconds: 3 }); + + it("does not persist temporary attachment URLs in the usage record", async () => { + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: answersFor(3), + elapsed_seconds: 45, + }); + await executeFullReport(AUDIO, "acc_1", false); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(N); - for (const [args] of vi.mocked(chargeForFlamingoCall).mock.calls) { - expect(args).not.toHaveProperty("audioUrl"); - } + + const [charge] = vi.mocked(chargeForBatchAnswers).mock.calls[0]; + expect(charge).not.toHaveProperty("audioUrl"); }); }); diff --git a/lib/flamingo/__tests__/minimumCreditsForAnalyzeRequest.test.ts b/lib/flamingo/__tests__/minimumCreditsForAnalyzeRequest.test.ts deleted file mode 100644 index edd62d003..000000000 --- a/lib/flamingo/__tests__/minimumCreditsForAnalyzeRequest.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { minimumCreditsForAnalyzeRequest } from "../minimumCreditsForAnalyzeRequest"; -import { FULL_REPORT_SECTIONS } from "@/lib/flamingo/presets/fullReport"; - -describe("minimumCreditsForAnalyzeRequest", () => { - // The gate asks for the least the request can possibly cost: one base fee per - // model call. It never tries to guess elapsed seconds up front. - it("requires one base fee for a custom prompt", () => { - expect(minimumCreditsForAnalyzeRequest({ prompt: "What key is this in?" })).toBe(50_000); - }); - - it("requires one base fee for a single preset", () => { - expect( - minimumCreditsForAnalyzeRequest({ - preset: "mood_tags", - audio_url: "https://example.com/song.mp3", - }), - ).toBe(50_000); - }); - - it("requires one base fee per section for full_report", () => { - expect(FULL_REPORT_SECTIONS).toHaveLength(13); - expect( - minimumCreditsForAnalyzeRequest({ - preset: "full_report", - audio_url: "https://example.com/song.mp3", - }), - ).toBe(650_000); - }); -}); diff --git a/lib/flamingo/__tests__/postFlamingoGenerateHandler.credits.test.ts b/lib/flamingo/__tests__/postFlamingoGenerateHandler.credits.test.ts index 8db8d6ef0..db9b00bad 100644 --- a/lib/flamingo/__tests__/postFlamingoGenerateHandler.credits.test.ts +++ b/lib/flamingo/__tests__/postFlamingoGenerateHandler.credits.test.ts @@ -59,7 +59,7 @@ describe("postFlamingoGenerateHandler — credit gate", () => { ); }); - it("gates full_report on one floor per section", async () => { + it("gates full_report on one base fee, since its sections run as one batch", async () => { vi.mocked(ensureCreditsOrShortCircuit).mockResolvedValue(null); vi.mocked(processAnalyzeMusicRequest).mockResolvedValue({ type: "success", @@ -74,7 +74,7 @@ describe("postFlamingoGenerateHandler — credit gate", () => { expect(ensureCreditsOrShortCircuit).toHaveBeenCalledWith({ accountId: "acc_1", - creditsToDeduct: 650_000, + creditsToDeduct: 50_000, }); }); diff --git a/lib/flamingo/__tests__/postFlamingoGenerateHandler.test.ts b/lib/flamingo/__tests__/postFlamingoGenerateHandler.test.ts index cf53d4288..04360c994 100644 --- a/lib/flamingo/__tests__/postFlamingoGenerateHandler.test.ts +++ b/lib/flamingo/__tests__/postFlamingoGenerateHandler.test.ts @@ -29,6 +29,10 @@ vi.mock("@/lib/flamingo/chargeForFlamingoCall", () => ({ chargeForFlamingoCall: vi.fn().mockResolvedValue(undefined), })); +vi.mock("@/lib/flamingo/chargeForBatchAnswers", () => ({ + chargeForBatchAnswers: vi.fn().mockResolvedValue(undefined), +})); + /** * Creates a mock NextRequest with a JSON body. * diff --git a/lib/flamingo/__tests__/presets.test.ts b/lib/flamingo/__tests__/presets.test.ts index 3456940fb..afb8e7827 100644 --- a/lib/flamingo/__tests__/presets.test.ts +++ b/lib/flamingo/__tests__/presets.test.ts @@ -39,6 +39,31 @@ describe("getAllPresets", () => { expect(["json", "text"]).toContain(preset.responseFormat); } }); + + // Answers stored as data must not change between two runs on the same song, + // so they decode greedily; only the three presets that write prose sample. + const PROSE_PRESETS = ["song_description", "playlist_pitch", "artist_development_notes"]; + + it("decodes greedily for every preset whose answer is data", () => { + const data = getAllPresets().filter(p => !PROSE_PRESETS.includes(p.name)); + expect(data).toHaveLength(10); + for (const preset of data) { + // Temperature 1.0 is the neutral value greedy decoding ignores; any other + // value makes transformers log an "invalid flag" notice on every call. + const { do_sample, temperature } = preset.params; + expect({ name: preset.name, do_sample, temperature }).toEqual({ + name: preset.name, + do_sample: false, + temperature: 1.0, + }); + } + }); + + it("keeps sampling for the presets that write prose", () => { + for (const name of PROSE_PRESETS) { + expect(getPreset(name)?.params.do_sample).toBe(true); + } + }); }); describe("PRESET_NAMES", () => { diff --git a/lib/flamingo/__tests__/processAnalyzeMusicRequest.credits.test.ts b/lib/flamingo/__tests__/processAnalyzeMusicRequest.credits.test.ts index b3f02a30f..149e9fbe9 100644 --- a/lib/flamingo/__tests__/processAnalyzeMusicRequest.credits.test.ts +++ b/lib/flamingo/__tests__/processAnalyzeMusicRequest.credits.test.ts @@ -58,7 +58,7 @@ describe("processAnalyzeMusicRequest — credits", () => { expect(callFlamingoGenerate).not.toHaveBeenCalled(); }); - it("hands the account to executeFullReport, which charges per section", async () => { + it("hands the account to executeFullReport, which charges the batch itself", async () => { vi.mocked(executeFullReport).mockResolvedValue({ report: {}, elapsed_seconds: 12, failed: [] }); await processAnalyzeMusicRequest( @@ -67,7 +67,7 @@ describe("processAnalyzeMusicRequest — credits", () => { ); expect(executeFullReport).toHaveBeenCalledWith("https://example.com/song.mp3", "acc_1", true); - // The single-call charge must not double up on top of the per-section ones. + // The single-call charge must not double up on top of the batch's. expect(chargeForFlamingoCall).not.toHaveBeenCalled(); }); it("omits private attachment URLs from single-call usage records", async () => { diff --git a/lib/flamingo/callFlamingoGenerateBatch.ts b/lib/flamingo/callFlamingoGenerateBatch.ts index e39dfc49c..f3d3db952 100644 --- a/lib/flamingo/callFlamingoGenerateBatch.ts +++ b/lib/flamingo/callFlamingoGenerateBatch.ts @@ -7,17 +7,39 @@ import { type FlamingoGenerateBatchResult, } from "@/lib/flamingo/isFlamingoGenerateBatchResult"; +/** + * One prompt as sent to Modal. `clip` (seconds) has the model hear only that + * stretch of the audio; `temperature`, `top_p` and `do_sample` override the + * batch's settings for this prompt alone. The public batch body's prompts + * carry neither, so they fit as they are. + */ +export interface FlamingoBatchPrompt { + id: string; + prompt: string; + max_new_tokens: number; + clip?: { start: number; end: number }; + temperature?: number; + top_p?: number; + do_sample?: boolean; +} + +/** A batch request: the public body's settings, with prompts that may carry clips and settings. */ +export type FlamingoBatchRequest = Omit & { + prompts: FlamingoBatchPrompt[]; +}; + /** * Calls the Music Flamingo /generate_batch endpoint on Modal: every prompt * answered in one request on one download of the audio. * - * @param params - The validated batch body (defaults applied). + * @param params - The validated batch body (defaults applied), or an internal + * request whose prompts carry clips or their own settings. * @param options - Optional abort signal. * @returns Every prompt's answer and the batch's total seconds. * @throws FlamingoServiceError if the Modal endpoint returns a non-OK status. */ export async function callFlamingoGenerateBatch( - params: FlamingoGenerateBatchBody, + params: FlamingoBatchRequest, options?: { signal?: AbortSignal }, ): Promise { const response = await fetch(FLAMINGO_GENERATE_BATCH_URL, { @@ -29,11 +51,17 @@ export async function callFlamingoGenerateBatch( }, body: JSON.stringify({ audio_url: params.audio_url, - prompts: params.prompts.map(({ id, prompt, max_new_tokens }) => ({ - id, - prompt, - max_new_tokens, - })), + prompts: params.prompts.map( + ({ id, prompt, max_new_tokens, clip, temperature, top_p, do_sample }) => ({ + id, + prompt, + max_new_tokens, + ...(clip ? { clip_start: clip.start, clip_end: clip.end } : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(top_p !== undefined ? { top_p } : {}), + ...(do_sample !== undefined ? { do_sample } : {}), + }), + ), temperature: params.temperature, top_p: params.top_p, do_sample: params.do_sample, diff --git a/lib/flamingo/chargeForBatchAnswers.ts b/lib/flamingo/chargeForBatchAnswers.ts new file mode 100644 index 000000000..20a30e800 --- /dev/null +++ b/lib/flamingo/chargeForBatchAnswers.ts @@ -0,0 +1,31 @@ +import { chargeForFlamingoBatch } from "@/lib/flamingo/chargeForFlamingoBatch"; +import type { FlamingoBatchAnswer } from "@/lib/flamingo/isFlamingoGenerateBatchResult"; + +/** + * Charges one batch for the answers the caller could use: one base price + * plus the seconds those answers took, as a single usage row. An answer the + * caller had to drop (unparseable, missing) costs nothing, and no usable + * answer at all means no charge, as when each section was its own call + * (recoupable/app#2061). Never throws, like `chargeForFlamingoBatch`. + * + * @param params - The account, the answers that were used, and the audio to + * record on the usage row (omit for temporary attachment URLs). + */ +export async function chargeForBatchAnswers({ + accountId, + answers, + audioUrl, +}: { + accountId: string; + answers: Pick[]; + audioUrl?: string; +}): Promise { + if (answers.length === 0) return; + const seconds = answers.reduce((total, answer) => total + answer.elapsed_seconds, 0); + await chargeForFlamingoBatch({ + accountId, + elapsedSeconds: seconds, + promptCount: answers.length, + ...(audioUrl ? { audioUrl } : {}), + }); +} diff --git a/lib/flamingo/executeFullReport.ts b/lib/flamingo/executeFullReport.ts index 54b3782d2..b27c32020 100644 --- a/lib/flamingo/executeFullReport.ts +++ b/lib/flamingo/executeFullReport.ts @@ -1,34 +1,26 @@ -import { callFlamingoGenerate } from "@/lib/flamingo/callFlamingoGenerate"; +import { callFlamingoGenerateBatch } from "@/lib/flamingo/callFlamingoGenerateBatch"; +import { chargeForBatchAnswers } from "@/lib/flamingo/chargeForBatchAnswers"; +import type { FlamingoBatchAnswer } from "@/lib/flamingo/isFlamingoGenerateBatchResult"; import { getPreset } from "@/lib/flamingo/presets"; import { FULL_REPORT_SECTIONS } from "@/lib/flamingo/presets/fullReport"; -import { chargeForFlamingoCall } from "@/lib/flamingo/chargeForFlamingoCall"; /** - * Result from a single preset within the full report. - */ -interface SectionResult { - /** The report key this section maps to */ - reportKey: string; - /** The parsed/processed response data */ - data: unknown; - /** Inference time for this section in seconds */ - elapsed_seconds: number; -} - -/** - * Executes the full_report preset by running all 13 individual presets - * in parallel against the same audio URL. + * Executes the full_report preset: all 13 sections asked in ONE batch + * request, so the audio is downloaded once and the per-request base price is + * paid once instead of 13 times. Each section keeps its preset's prompt, + * token limit, temperature and sampling choice. * - * Each preset is resolved to its prompt and generation params, sent to - * the Modal endpoint concurrently, and the results are combined into - * a single report object in narrative order. + * The batch runs the sections one after another on one container, so the + * report takes the sum of the sections' times rather than the longest. * - * Each section that returns is charged to `accountId` on its own seconds, - * so the usage page shows one row per section; a section that fails costs - * nothing (recoupable/app#2061). + * The account is charged once, for the sections that came back usable: one + * base price plus their seconds. A section whose answer is missing or fails + * to parse is reported as `{ error }`, listed in `failed`, and costs nothing; + * if the whole batch fails, every section fails and nothing is charged + * (recoupable/app#2061). * * @param audioUrl - Public URL to the audio file - * @param accountId - The account the section calls are charged to + * @param accountId - The account the batch is charged to * @param recordAudioUrl - False for temporary host attachment URLs * @returns Combined report object with all sections, total elapsed time, and * the report keys of the sections that failed (in report order) @@ -43,72 +35,83 @@ export async function executeFullReport( failed: string[]; }> { const startTime = Date.now(); - - // Build all preset calls in parallel - const promises: Promise[] = FULL_REPORT_SECTIONS.map(async section => { + const sections = FULL_REPORT_SECTIONS.flatMap(section => { const preset = getPreset(section.preset); - if (!preset) return null; + return preset ? [{ reportKey: section.reportKey, preset }] : []; + }); - try { - const result = await callFlamingoGenerate({ + let answers: FlamingoBatchAnswer[] = []; + let batchError: string | null = null; + try { + const result = await callFlamingoGenerateBatch({ + audio_url: audioUrl, + prompts: sections.map(({ reportKey, preset }) => ({ + id: reportKey, prompt: preset.prompt, - audio_url: audioUrl, max_new_tokens: preset.params.max_new_tokens, temperature: preset.params.temperature, - top_p: 1.0, do_sample: preset.params.do_sample, - }); - - // Apply post-processing if the preset defines one. A parse failure - // lands in the catch below as a failed section, so charge only after it. - const data = preset.parseResponse ? preset.parseResponse(result.response) : result.response; - - await chargeForFlamingoCall({ - accountId, - elapsedSeconds: result.elapsed_seconds, - ...(recordAudioUrl ? { audioUrl } : {}), - }); - - return { - reportKey: section.reportKey, - data, - elapsed_seconds: result.elapsed_seconds, - }; - } catch (error) { - console.error(`[WARN] Full report section "${section.preset}" failed:`, error); - return { - reportKey: section.reportKey, - data: { - error: `Section failed: ${error instanceof Error ? error.message : "unknown error"}`, - }, - elapsed_seconds: 0, - }; - } - }); - - // Execute all presets in parallel - const results = await Promise.all(promises); + })), + temperature: 1.0, + top_p: 1.0, + do_sample: false, + }); + answers = result.answers; + } catch (error) { + console.error("[WARN] Full report batch failed:", error); + batchError = error instanceof Error ? error.message : "unknown error"; + } - // Assemble report in narrative order (sections are already ordered) + // Assemble in narrative order (sections are already ordered) + const byKey = new Map(answers.map(answer => [answer.id, answer])); const report: Record = {}; const failed: string[] = []; - for (const result of results) { - if (result) { - report[result.reportKey] = result.data; - if (isFailedSection(result.data)) failed.push(result.reportKey); + const used: FlamingoBatchAnswer[] = []; + for (const { reportKey, preset } of sections) { + const answer = byKey.get(reportKey); + if (!answer) { + report[reportKey] = sectionError( + batchError ?? "the model returned no answer for this section", + ); + failed.push(reportKey); + continue; + } + try { + const data = preset.parseResponse ? preset.parseResponse(answer.response) : answer.response; + if (isErrorShape(data)) { + // A reply of {"error": "..."} would read as a failed section in the + // report, so it is one: listed in `failed` and not billed. + report[reportKey] = sectionError(data.error); + failed.push(reportKey); + continue; + } + report[reportKey] = data; + used.push(answer); + } catch (error) { + console.error(`[WARN] Full report section "${preset.name}" failed to parse:`, error); + report[reportKey] = sectionError(error instanceof Error ? error.message : "unknown error"); + failed.push(reportKey); } } + await chargeForBatchAnswers({ + accountId, + answers: used, + ...(recordAudioUrl ? { audioUrl } : {}), + }); + const totalElapsed = Math.round(((Date.now() - startTime) / 1000) * 100) / 100; return { report, elapsed_seconds: totalElapsed, failed }; } -/** - * A section that failed is the `{ error }` object the catch above produced; - * a preset that legitimately returns an object never uses that key alone. - */ -function isFailedSection(data: unknown): boolean { +/** The `{ error }` object a failed section reports in place of its data. */ +function sectionError(reason: string): { error: string } { + return { error: `Section failed: ${reason}` }; +} + +/** True for an object whose only key is a string `error`: the failed-section shape. */ +function isErrorShape(data: unknown): data is { error: string } { return ( typeof data === "object" && data !== null && diff --git a/lib/flamingo/isFlamingoGenerateBatchResult.ts b/lib/flamingo/isFlamingoGenerateBatchResult.ts index c3099c425..c091747e9 100644 --- a/lib/flamingo/isFlamingoGenerateBatchResult.ts +++ b/lib/flamingo/isFlamingoGenerateBatchResult.ts @@ -6,6 +6,9 @@ export interface FlamingoBatchAnswer { response: string; /** Inference time for this prompt alone, in seconds. */ elapsed_seconds: number; + /** The clip window the model heard, echoed when the prompt asked for one (seconds). */ + clip_start?: number; + clip_end?: number; } /** Response shape from the Music Flamingo /generate_batch endpoint on Modal. */ @@ -20,7 +23,8 @@ export interface FlamingoGenerateBatchResult { * * @param value - Unknown parsed JSON payload. * @returns True when the payload has a total `elapsed_seconds` and every - * answer carries a string `id`, a string `response` and numeric seconds. + * answer carries a string `id`, a string `response` and numeric seconds, + * plus either no clip window or both of its ends as numbers. */ export function isFlamingoGenerateBatchResult( value: unknown, @@ -33,10 +37,12 @@ export function isFlamingoGenerateBatchResult( return candidate.answers.every(answer => { if (!answer || typeof answer !== "object") return false; const a = answer as Record; + const clipped = a.clip_start !== undefined || a.clip_end !== undefined; return ( typeof a.id === "string" && typeof a.response === "string" && - typeof a.elapsed_seconds === "number" + typeof a.elapsed_seconds === "number" && + (!clipped || (typeof a.clip_start === "number" && typeof a.clip_end === "number")) ); }); } diff --git a/lib/flamingo/minimumCreditsForAnalyzeRequest.ts b/lib/flamingo/minimumCreditsForAnalyzeRequest.ts deleted file mode 100644 index d354a9f03..000000000 --- a/lib/flamingo/minimumCreditsForAnalyzeRequest.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; -import { FULL_REPORT_PRESET_NAME, FULL_REPORT_SECTIONS } from "@/lib/flamingo/presets/fullReport"; -import type { FlamingoGenerateBody } from "@/lib/flamingo/flamingoGenerateBodySchema"; - -/** - * The base price of an analyze request, for the pre-flight credit gate: one - * base fee per model call the request will make. `full_report` fans out to - * one call per section; everything else is a single call. The metered part, - * priced on the model's reported seconds, lands after the call. - * - * @param params - The validated request; only `preset` matters. - * @returns Credits the account must hold before the model is called. - */ -export function minimumCreditsForAnalyzeRequest(params: Partial): number { - const calls = params.preset === FULL_REPORT_PRESET_NAME ? FULL_REPORT_SECTIONS.length : 1; - return calls * FLAMINGO_BASE_CREDITS; -} diff --git a/lib/flamingo/postFlamingoGenerateHandler.ts b/lib/flamingo/postFlamingoGenerateHandler.ts index cd10f8737..7b722d927 100644 --- a/lib/flamingo/postFlamingoGenerateHandler.ts +++ b/lib/flamingo/postFlamingoGenerateHandler.ts @@ -5,7 +5,7 @@ import { validateFlamingoGenerateRequest } from "@/lib/flamingo/validateFlamingo import { processAnalyzeMusicRequest } from "@/lib/flamingo/processAnalyzeMusicRequest"; import { FlamingoServiceError } from "@/lib/flamingo/FlamingoServiceError"; import { ensureCreditsOrShortCircuit } from "@/lib/credits/ensureCreditsOrShortCircuit"; -import { minimumCreditsForAnalyzeRequest } from "@/lib/flamingo/minimumCreditsForAnalyzeRequest"; +import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; /** * Handler for POST /api/songs/analyze. @@ -31,7 +31,7 @@ export async function postFlamingoGenerateHandler(request: NextRequest): Promise try { short = await ensureCreditsOrShortCircuit({ accountId, - creditsToDeduct: minimumCreditsForAnalyzeRequest(body), + creditsToDeduct: FLAMINGO_BASE_CREDITS, }); } catch (err) { console.error("[postFlamingoGenerateHandler] credit gate failed:", err); diff --git a/lib/flamingo/presets/audienceProfile.ts b/lib/flamingo/presets/audienceProfile.ts index 125738a7d..e542fcedd 100644 --- a/lib/flamingo/presets/audienceProfile.ts +++ b/lib/flamingo/presets/audienceProfile.ts @@ -8,7 +8,7 @@ export const audienceProfilePreset: PresetConfig = { description: "Target audience analysis — demographics, platforms, listening contexts, comparable fanbases, and marketing hook.", prompt: `Based on the sound, production, lyrics, and overall vibe of this track, describe the target audience. Return ONLY a valid JSON object (use double quotes) with: "age_range" (string like "18-25"), "gender_skew" (string: "neutral", "female-leaning", or "male-leaning"), "lifestyle_tags" (array of 5 lifestyle descriptors, e.g. "journal-keeper", "iced coffee drinker", "late-night driver"), "listening_contexts" (array of 5 situations where someone would play this, e.g. "alone in bed at 2am", "rainy day studying"), "platforms" (array of platforms where this audience lives, e.g. "TikTok", "Spotify", "Pinterest"), "playlist_types" (array of 5 playlist names/types this fits, e.g. "sad girl autumn", "indie chill", "bedroom pop essentials"), "comparable_fanbases" (array of 3 artists whose fans would like this), "marketing_hook" (one sentence that captures why this audience connects with this track). No other text.`, - params: { max_new_tokens: 1024, temperature: 0.5, do_sample: true }, + params: { max_new_tokens: 1024, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/catalogMetadata.ts b/lib/flamingo/presets/catalogMetadata.ts index 0bcc4b2cd..1793b7479 100644 --- a/lib/flamingo/presets/catalogMetadata.ts +++ b/lib/flamingo/presets/catalogMetadata.ts @@ -8,7 +8,7 @@ export const catalogMetadataPreset: PresetConfig = { description: "Returns structured JSON metadata for catalog enrichment — genre, BPM, key, mood, instruments, similar artists.", prompt: `Analyze this track and return ONLY a valid JSON object (use double quotes). Fields: genre (string, primary genre), subgenres (array of up to 3), mood (array of up to 5 mood tags), tempo_bpm (integer), key (string like "C major"), time_signature (string like "4/4"), instruments (array), vocal_type (string or "none"), vocal_style (string), production_style (string), energy_level (1-10), danceability (1-10), lyrical_themes (array of up to 3), similar_artists (array of up to 3), description (one sentence). No other text.`, - params: { max_new_tokens: 1024, temperature: 0.3, do_sample: true }, + params: { max_new_tokens: 1024, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/contentAdvisory.ts b/lib/flamingo/presets/contentAdvisory.ts index 2e2a83f1b..08ba88ac3 100644 --- a/lib/flamingo/presets/contentAdvisory.ts +++ b/lib/flamingo/presets/contentAdvisory.ts @@ -8,7 +8,7 @@ export const contentAdvisoryPreset: PresetConfig = { description: "Brand safety analysis — explicit content flags, thematic advisory, and radio-friendliness.", prompt: `Analyze this track for brand safety. Return ONLY a valid JSON object (double quotes). Profanity means ONLY swear words (fuck, shit, damn, bitch, ass, hell, etc). The word "love" is NOT profanity. Fields: explicit (boolean, true ONLY if actual swear words are present), profanity_found (array of unique swear words found, empty array if none), themes (object with booleans: violence, drugs_alcohol, sexual_content, self_harm, political, religious, profanity, emotional_pain), brand_safety_rating (string: safe/caution/not_recommended), brand_safety_notes (string, one sentence), radio_friendly (boolean), content_summary (one sentence describing lyrical content). No other text.`, - params: { max_new_tokens: 256, temperature: 0.2, do_sample: true }, + params: { max_new_tokens: 256, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/lyricTranscription.ts b/lib/flamingo/presets/lyricTranscription.ts index 629c6dd16..c0582ee62 100644 --- a/lib/flamingo/presets/lyricTranscription.ts +++ b/lib/flamingo/presets/lyricTranscription.ts @@ -7,7 +7,7 @@ export const lyricTranscriptionPreset: PresetConfig = { label: "Lyric Transcription", description: "Transcribes complete lyrics with section headers (Verse, Chorus, Bridge, etc).", prompt: `Transcribe the complete lyrics of this song. Format with section headers in brackets like [Verse 1], [Chorus], [Bridge], [Outro]. Include every distinct lyric line. Break lines where the singer naturally pauses.`, - params: { max_new_tokens: 2048, temperature: 0.1, do_sample: true }, + params: { max_new_tokens: 2048, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "text", parseResponse: raw => condenseRepetitions(raw), diff --git a/lib/flamingo/presets/mixFeedback.ts b/lib/flamingo/presets/mixFeedback.ts index 095559fcf..57adfdbc1 100644 --- a/lib/flamingo/presets/mixFeedback.ts +++ b/lib/flamingo/presets/mixFeedback.ts @@ -7,7 +7,7 @@ export const mixFeedbackPreset: PresetConfig = { description: "Technical mix engineer critique — frequency balance, dynamics, stereo field, and specific processing suggestions.", prompt: `I already know what this song sounds like. I need a MIX ENGINEER's critique ONLY. Do not describe the genre, lyrics, mood, or song structure. Focus exclusively on the technical quality of the mix. Answer ONLY these questions: 1) Is the low end clean or muddy? Is the kick cutting through? Is the sub controlled? 2) Are the vocals sitting properly in the mid-range or are they fighting with other elements? Any boxy buildup around 300-500Hz? 3) Is there enough air above 10kHz? Any harshness in the 2-5kHz range? Sibilance issues? 4) How wide is the stereo image? Would this collapse in mono? 5) Is the dynamic range healthy or is it over-compressed? 6) What specific EQ, compression, or spatial processing would you change? List concrete fixes.`, - params: { max_new_tokens: 1024, temperature: 0.4, do_sample: true }, + params: { max_new_tokens: 1024, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "text", }; diff --git a/lib/flamingo/presets/moodTags.ts b/lib/flamingo/presets/moodTags.ts index 5cf6129d7..3fe96baf8 100644 --- a/lib/flamingo/presets/moodTags.ts +++ b/lib/flamingo/presets/moodTags.ts @@ -7,7 +7,7 @@ export const moodTagsPreset: PresetConfig = { label: "Mood Tags", description: "Returns mood/vibe/energy tags for playlist curation and music discovery.", prompt: `Listen to this track and return ONLY a valid JSON object (use double quotes) with two fields: "tags" (array of 8-10 mood/vibe/energy tags that could be used for playlist curation and music discovery, e.g. "dreamy", "late-night", "heartbreak", "chill"), and "primary_mood" (single word that best captures the overall feeling). No other text.`, - params: { max_new_tokens: 256, temperature: 0.3, do_sample: true }, + params: { max_new_tokens: 256, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/musicTheory.ts b/lib/flamingo/presets/musicTheory.ts index cbaf47119..a2a47762e 100644 --- a/lib/flamingo/presets/musicTheory.ts +++ b/lib/flamingo/presets/musicTheory.ts @@ -8,7 +8,7 @@ export const musicTheoryPreset: PresetConfig = { description: "Music theory analysis — key, chord progression, time signature, song structure, and harmonic features.", prompt: `Analyze this track from a music theory perspective. Return ONLY a valid JSON object (use double quotes) with: key (string), scale (string like "major" or "natural minor"), tempo_bpm (integer), time_signature (string), chord_progression (array of chord symbols for ONE cycle of the main repeating progression — do NOT repeat the cycle), song_sections (array of objects with "name" like "intro" and "bars" as integer count), duration_seconds (number), notable_harmonic_features (string). No other text.`, - params: { max_new_tokens: 512, temperature: 0.3, do_sample: true }, + params: { max_new_tokens: 512, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => { diff --git a/lib/flamingo/presets/sampleDetection.ts b/lib/flamingo/presets/sampleDetection.ts index ae0442dd5..2b6135cba 100644 --- a/lib/flamingo/presets/sampleDetection.ts +++ b/lib/flamingo/presets/sampleDetection.ts @@ -7,7 +7,7 @@ export const sampleDetectionPreset: PresetConfig = { description: "Identifies potential samples, interpolations, or strong musical references to existing songs.", prompt: `Analyze this track for any musical elements that sound like they could be samples, interpolations, or strong references to existing songs. For each potential match, describe: 1) what element sounds familiar (melody, chord progression, beat pattern, vocal cadence), 2) what it reminds you of (specific song/artist if possible), 3) your confidence level (high/medium/low). If nothing sounds sampled, say so.`, - params: { max_new_tokens: 512, temperature: 0.5, do_sample: true }, + params: { max_new_tokens: 512, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "text", }; diff --git a/lib/flamingo/presets/similarArtists.ts b/lib/flamingo/presets/similarArtists.ts index 92c029335..264785fa8 100644 --- a/lib/flamingo/presets/similarArtists.ts +++ b/lib/flamingo/presets/similarArtists.ts @@ -8,7 +8,7 @@ export const similarArtistsPreset: PresetConfig = { description: "Suggests 5 similar artists based on sound, production, and vocal qualities, with reasoning.", prompt: `Based on the sound, production style, vocal qualities, and genre of this track, suggest 5 similar artists. Return ONLY a valid JSON object (use double quotes) with a field "similar_artists" containing an array of objects, each with: "name" (artist name), "similarity" (one sentence explaining why they are similar). No other text.`, - params: { max_new_tokens: 512, temperature: 0.5, do_sample: true }, + params: { max_new_tokens: 512, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/syncBriefMatch.ts b/lib/flamingo/presets/syncBriefMatch.ts index 920755484..67adb06c0 100644 --- a/lib/flamingo/presets/syncBriefMatch.ts +++ b/lib/flamingo/presets/syncBriefMatch.ts @@ -8,7 +8,7 @@ export const syncBriefMatchPreset: PresetConfig = { description: "Sync licensing analysis — visual scene matches, energy curve, best sync moments, and placement suggestions.", prompt: `Analyze this track for sync licensing placement. Return ONLY a valid JSON object (use double quotes) with these fields: "visual_scenes" (array of 5 specific scenes this track would fit, e.g. "rainy window scene in a coming-of-age film"), "emotional_arc" (string describing how the emotion shifts across the track), "energy_curve" (string like "low-build-peak-fade" or "steady medium"), "best_sync_moments" (array of objects with "timestamp_range" like "0:00-0:30" and "description" of what's happening musically and why it works for sync), "genres_for_brief" (array of genre tags a music supervisor would search), "avoid_for" (array of contexts where this track would NOT work, e.g. "upbeat car commercial"). No other text.`, - params: { max_new_tokens: 1024, temperature: 0.4, do_sample: true }, + params: { max_new_tokens: 1024, temperature: 1.0, do_sample: false }, requiresAudio: true, responseFormat: "json", parseResponse: raw => parseJsonLike(raw), diff --git a/lib/flamingo/presets/types.ts b/lib/flamingo/presets/types.ts index 17bf41ce9..fbcfe2ae6 100644 --- a/lib/flamingo/presets/types.ts +++ b/lib/flamingo/presets/types.ts @@ -15,7 +15,15 @@ export interface PresetConfig { /** Generation parameters */ params: { max_new_tokens: number; + /** + * Only used when `do_sample` is true. Greedy presets keep the neutral 1.0, + * since any other value makes transformers log an ignored-flag notice. + */ temperature: number; + /** + * False (greedy) for presets whose answer is stored as data, so the same + * song gets the same answer on every run; true only for prose. + */ do_sample: boolean; }; /** Whether this preset requires an audio_url */ diff --git a/lib/flamingo/processAnalyzeMusicRequest.ts b/lib/flamingo/processAnalyzeMusicRequest.ts index c55e4ac7b..ff11f0dbf 100644 --- a/lib/flamingo/processAnalyzeMusicRequest.ts +++ b/lib/flamingo/processAnalyzeMusicRequest.ts @@ -56,10 +56,12 @@ export interface AnalyzeMusicContext { * Shared business logic for music analysis. * Used by both POST /api/songs/analyze and the analyze_music MCP tool. * - * Every successful model call is charged to `context.accountId` after it + * A successful single call is charged to `context.accountId` after it * returns, priced on the seconds the model reported (recoupable/app#2061). - * Callers verify `params.audio_url` (`verifyAudioUrl`) and gate the balance - * (`minimumCreditsForAnalyzeRequest`) before calling this. + * `full_report` is one batch request, charged only when at least one section + * came back usable, on those sections' seconds. Callers verify `params.audio_url` + * (`verifyAudioUrl`) and gate the balance on one base price + * (`FLAMINGO_BASE_CREDITS`) before calling this. * * @param params - Validated request parameters. * @param context - The account the model calls are charged to. diff --git a/lib/mcp/tools/flamingo/registerAnalyzeMusicTool.ts b/lib/mcp/tools/flamingo/registerAnalyzeMusicTool.ts index 409086506..da9b5dbc9 100644 --- a/lib/mcp/tools/flamingo/registerAnalyzeMusicTool.ts +++ b/lib/mcp/tools/flamingo/registerAnalyzeMusicTool.ts @@ -9,7 +9,7 @@ import { type FlamingoGenerateBody } from "@/lib/flamingo/flamingoGenerateBodySc import { analyzeMusicToolSchema, type AnalyzeMusicToolInput } from "./analyzeMusicToolSchema"; import { processAnalyzeMusicRequest } from "@/lib/flamingo/processAnalyzeMusicRequest"; import { checkCreditsAvailable } from "@/lib/credits/checkCreditsAvailable"; -import { minimumCreditsForAnalyzeRequest } from "@/lib/flamingo/minimumCreditsForAnalyzeRequest"; +import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; import { verifyAudioUrl } from "@/lib/flamingo/verifyAudioUrl"; /** @@ -78,7 +78,7 @@ export function registerAnalyzeMusicTool(server: McpServer): void { try { gate = await checkCreditsAvailable({ accountId, - creditsToDeduct: minimumCreditsForAnalyzeRequest(params), + creditsToDeduct: FLAMINGO_BASE_CREDITS, }); } catch (err) { console.error("[analyze_music] credit gate failed:", err); diff --git a/lib/sites/production/analyzeReleaseMusic.ts b/lib/sites/production/analyzeReleaseMusic.ts index dd795a438..8b3be4d37 100644 --- a/lib/sites/production/analyzeReleaseMusic.ts +++ b/lib/sites/production/analyzeReleaseMusic.ts @@ -3,7 +3,7 @@ import type { ReleaseContext } from "./schema"; import { verifyAudioUrl } from "@/lib/flamingo/verifyAudioUrl"; import { processAnalyzeMusicRequest } from "@/lib/flamingo/processAnalyzeMusicRequest"; import { flamingoGenerateBodySchema } from "@/lib/flamingo/flamingoGenerateBodySchema"; -import { minimumCreditsForAnalyzeRequest } from "@/lib/flamingo/minimumCreditsForAnalyzeRequest"; +import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; import { requireCredits } from "./requireCredits"; import { SiteError } from "../SiteError"; export async function analyzeReleaseMusic( @@ -31,7 +31,7 @@ export async function analyzeReleaseMusic( "Analyze this audio for a creative fan experience. Describe audible instrumentation, energy, mood, texture, rhythmic feel and changes within the supplied clip. Summarize clearly audible lyrical themes without quoting lyrics. State uncertainty and do not invent lyrics or extrapolate a preview into a full-song structure.", max_new_tokens: 1200, }); - await requireCredits(accountId, minimumCreditsForAnalyzeRequest(params)); + await requireCredits(accountId, FLAMINGO_BASE_CREDITS); try { const result = await processAnalyzeMusicRequest(params, { accountId }); if (result.type !== "success") return unavailable("Music analysis was unavailable."); diff --git a/lib/sites/production/analyzeSiteContextAudio.ts b/lib/sites/production/analyzeSiteContextAudio.ts index defbfbeb8..cc7cec914 100644 --- a/lib/sites/production/analyzeSiteContextAudio.ts +++ b/lib/sites/production/analyzeSiteContextAudio.ts @@ -3,7 +3,7 @@ import { callContextRpc } from "@/lib/supabase/context_requests/callContextRpc"; import { flamingoGenerateBodySchema } from "@/lib/flamingo/flamingoGenerateBodySchema"; import { processAnalyzeMusicRequest } from "@/lib/flamingo/processAnalyzeMusicRequest"; import { verifyAudioUrl } from "@/lib/flamingo/verifyAudioUrl"; -import { minimumCreditsForAnalyzeRequest } from "@/lib/flamingo/minimumCreditsForAnalyzeRequest"; +import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; import { requireCredits } from "./requireCredits"; import { generateProductionObject } from "./generateProductionObject"; import { authorizeSiteWorkspace } from "../authorizeSiteWorkspace"; @@ -29,7 +29,7 @@ export async function analyzeSiteContextAudio( const params = flamingoGenerateBodySchema.parse(input); if (!(await verifyAudioUrl(input.audio_url)).ok) throw new Error("Private audio URL unavailable"); - await requireCredits(accountId, minimumCreditsForAnalyzeRequest(params)); + await requireCredits(accountId, FLAMINGO_BASE_CREDITS); const result = await processAnalyzeMusicRequest(params, { accountId }); if ( result.type !== "success" || diff --git a/lib/songMoments/__tests__/describeMoment.test.ts b/lib/songMoments/__tests__/describeMoment.test.ts deleted file mode 100644 index 33bf47383..000000000 --- a/lib/songMoments/__tests__/describeMoment.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { describeMoment } from "../describeMoment"; -import { callFlamingoGenerate } from "@/lib/flamingo/callFlamingoGenerate"; - -vi.mock("@/lib/flamingo/callFlamingoGenerate", () => ({ callFlamingoGenerate: vi.fn() })); - -describe("describeMoment", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("has the model hear only the moment's clip and parses its words", async () => { - vi.mocked(callFlamingoGenerate).mockResolvedValue({ - response: - '{"description": "The chorus lands.", "emotion": ["euphoric"], "vocals": true, "best_for": ["ad"]}', - elapsed_seconds: 2.5, - }); - - const result = await describeMoment({ - audioUrl: "https://example.com/song.mp3", - start: 36.15, - end: 50.9, - }); - - expect(callFlamingoGenerate).toHaveBeenCalledWith( - expect.objectContaining({ audio_url: "https://example.com/song.mp3", do_sample: false }), - { clip: { start: 36.15, end: 50.9 } }, - ); - expect(result).toEqual({ - description: { - description: "The chorus lands.", - emotion: ["euphoric"], - best_for: ["ad"], - vocals: true, - }, - elapsed_seconds: 2.5, - }); - }); -}); diff --git a/lib/songMoments/__tests__/describeMoments.test.ts b/lib/songMoments/__tests__/describeMoments.test.ts new file mode 100644 index 000000000..efe60ed00 --- /dev/null +++ b/lib/songMoments/__tests__/describeMoments.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { describeMoments } from "../describeMoments"; +import { callFlamingoGenerateBatch } from "@/lib/flamingo/callFlamingoGenerateBatch"; + +vi.mock("@/lib/flamingo/callFlamingoGenerateBatch", () => ({ + callFlamingoGenerateBatch: vi.fn(), +})); + +const audioUrl = "https://example.com/song.mp3"; +const hook = { type: "vocal_hook" as const, start: 36.15, end: 50.9 }; +const climax = { type: "climax" as const, start: 143.36, end: 158.1 }; +const reply = + '{"description": "The chorus lands.", "emotion": ["euphoric"], "vocals": true, "best_for": ["ad"]}'; +const words = { + description: "The chorus lands.", + emotion: ["euphoric"], + best_for: ["ad"], + vocals: true, +}; + +describe("describeMoments", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("asks about every moment in one batch, each prompt hearing only its own clip", async () => { + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: [ + { + id: "vocal_hook", + response: reply, + elapsed_seconds: 2.5, + clip_start: 36.15, + clip_end: 50.9, + }, + { id: "climax", response: reply, elapsed_seconds: 3, clip_start: 143.36, clip_end: 158.1 }, + ], + elapsed_seconds: 7, + }); + + const described = await describeMoments({ audioUrl, moments: [hook, climax] }); + + expect(callFlamingoGenerateBatch).toHaveBeenCalledTimes(1); + const [batch] = vi.mocked(callFlamingoGenerateBatch).mock.calls[0]; + expect(batch).toMatchObject({ audio_url: audioUrl, do_sample: false }); + expect(batch.prompts).toEqual([ + expect.objectContaining({ id: "vocal_hook", clip: { start: 36.15, end: 50.9 } }), + expect.objectContaining({ id: "climax", clip: { start: 143.36, end: 158.1 } }), + ]); + expect(described.get("vocal_hook")).toEqual({ description: words, elapsed_seconds: 2.5 }); + expect(described.get("climax")).toEqual({ description: words, elapsed_seconds: 3 }); + }); + + it("drops an answer that does not echo its clip, so words never come from the whole song", async () => { + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: [ + { id: "vocal_hook", response: reply, elapsed_seconds: 9 }, + { id: "climax", response: reply, elapsed_seconds: 3, clip_start: 143.36, clip_end: 158.1 }, + ], + elapsed_seconds: 13, + }); + + const described = await describeMoments({ audioUrl, moments: [hook, climax] }); + + expect(described.has("vocal_hook")).toBe(false); + expect(described.has("climax")).toBe(true); + }); + + it("drops an answer whose reply holds nothing usable, so it is neither merged nor billed", async () => { + vi.mocked(callFlamingoGenerateBatch).mockResolvedValue({ + answers: [ + { + id: "vocal_hook", + response: "I cannot describe this clip.", + elapsed_seconds: 2, + clip_start: 36.15, + clip_end: 50.9, + }, + { id: "climax", response: reply, elapsed_seconds: 3, clip_start: 143.36, clip_end: 158.1 }, + ], + elapsed_seconds: 6, + }); + + const described = await describeMoments({ audioUrl, moments: [hook, climax] }); + + expect([...described.keys()]).toEqual(["climax"]); + }); + + it("lets the batch's failure through for the caller to handle", async () => { + vi.mocked(callFlamingoGenerateBatch).mockRejectedValue( + new Error("Music analysis failed (status 503)"), + ); + + await expect(describeMoments({ audioUrl, moments: [hook] })).rejects.toThrow("status 503"); + }); +}); diff --git a/lib/songMoments/__tests__/minimumCreditsForSongMomentsRequest.test.ts b/lib/songMoments/__tests__/minimumCreditsForSongMomentsRequest.test.ts index fc89fc9bf..c11ac04ef 100644 --- a/lib/songMoments/__tests__/minimumCreditsForSongMomentsRequest.test.ts +++ b/lib/songMoments/__tests__/minimumCreditsForSongMomentsRequest.test.ts @@ -2,18 +2,16 @@ import { describe, it, expect } from "vitest"; import { minimumCreditsForSongMomentsRequest } from "../minimumCreditsForSongMomentsRequest"; describe("minimumCreditsForSongMomentsRequest", () => { - it("gates on the measurement plus a model base fee for each of the seven moment types", () => { - // $0.01 measurement + 7 × $0.05 model base. - expect(minimumCreditsForSongMomentsRequest({ describe: true })).toBe(360_000); + it("gates on the measurement plus one model base fee, since every moment is one batch", () => { + // $0.01 measurement + $0.05 model base. + expect(minimumCreditsForSongMomentsRequest({ describe: true })).toBe(60_000); + expect(minimumCreditsForSongMomentsRequest({ describe: ["vocal_hook", "climax"] })).toBe( + 60_000, + ); }); it("gates on the measurement alone when nothing is described", () => { expect(minimumCreditsForSongMomentsRequest({ describe: false })).toBe(10_000); - }); - - it("gates on the listed moments only", () => { - expect(minimumCreditsForSongMomentsRequest({ describe: ["vocal_hook", "climax"] })).toBe( - 110_000, - ); + expect(minimumCreditsForSongMomentsRequest({ describe: [] })).toBe(10_000); }); }); diff --git a/lib/songMoments/__tests__/postSongMomentsHandler.test.ts b/lib/songMoments/__tests__/postSongMomentsHandler.test.ts index dc6c50637..884bc43f9 100644 --- a/lib/songMoments/__tests__/postSongMomentsHandler.test.ts +++ b/lib/songMoments/__tests__/postSongMomentsHandler.test.ts @@ -50,14 +50,14 @@ describe("postSongMomentsHandler", () => { vi.mocked(processSongMomentsRequest).mockResolvedValue(measured); }); - it("gates on the full possible price, then returns the measured song flat", async () => { + it("gates on the measurement plus one model base fee, then returns the measured song flat", async () => { const res = await postSongMomentsHandler( request({ audio_url: "https://example.com/song.mp3" }), ); expect(ensureCreditsOrShortCircuit).toHaveBeenCalledWith({ accountId: "acc_1", - creditsToDeduct: 360_000, + creditsToDeduct: 60_000, }); expect(processSongMomentsRequest).toHaveBeenCalledWith( { audio_url: "https://example.com/song.mp3", describe: true }, diff --git a/lib/songMoments/__tests__/processSongMomentsRequest.test.ts b/lib/songMoments/__tests__/processSongMomentsRequest.test.ts index 61dde3555..1db1a6e51 100644 --- a/lib/songMoments/__tests__/processSongMomentsRequest.test.ts +++ b/lib/songMoments/__tests__/processSongMomentsRequest.test.ts @@ -2,14 +2,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { processSongMomentsRequest } from "../processSongMomentsRequest"; import { callMeasureSongMoments } from "@/lib/songMoments/callMeasureSongMoments"; import { chargeForSongMeasure } from "@/lib/songMoments/chargeForSongMeasure"; -import { describeMoment } from "@/lib/songMoments/describeMoment"; -import { chargeForFlamingoCall } from "@/lib/flamingo/chargeForFlamingoCall"; +import { describeMoments } from "@/lib/songMoments/describeMoments"; +import { chargeForBatchAnswers } from "@/lib/flamingo/chargeForBatchAnswers"; import type { MeasuredSong } from "@/lib/songMoments/isMeasuredSong"; vi.mock("@/lib/songMoments/callMeasureSongMoments", () => ({ callMeasureSongMoments: vi.fn() })); vi.mock("@/lib/songMoments/chargeForSongMeasure", () => ({ chargeForSongMeasure: vi.fn() })); -vi.mock("@/lib/songMoments/describeMoment", () => ({ describeMoment: vi.fn() })); -vi.mock("@/lib/flamingo/chargeForFlamingoCall", () => ({ chargeForFlamingoCall: vi.fn() })); +vi.mock("@/lib/songMoments/describeMoments", () => ({ describeMoments: vi.fn() })); +vi.mock("@/lib/flamingo/chargeForBatchAnswers", () => ({ chargeForBatchAnswers: vi.fn() })); const audio = "https://example.com/song.mp3"; const hook = { @@ -39,15 +39,17 @@ const words = { best_for: ["ad"], vocals: true, }; +const described = (...types: ("vocal_hook" | "climax")[]) => + new Map(types.map(type => [type, { description: words, elapsed_seconds: 2 }])); describe("processSongMomentsRequest", () => { beforeEach(() => { vi.clearAllMocks(); vi.mocked(callMeasureSongMoments).mockResolvedValue(song); - vi.mocked(describeMoment).mockResolvedValue({ description: words, elapsed_seconds: 2 }); + vi.mocked(describeMoments).mockResolvedValue(described("vocal_hook", "climax")); }); - it("measures, charges the measurement, and describes and charges every moment", async () => { + it("measures, charges the measurement, and describes every moment in one charged batch", async () => { const result = await processSongMomentsRequest( { audio_url: audio, section_order: ["Verse", "Chorus"], describe: true }, { accountId: "acc_1" }, @@ -58,12 +60,12 @@ describe("processSongMomentsRequest", () => { section_order: ["Verse", "Chorus"], }); expect(chargeForSongMeasure).toHaveBeenCalledWith({ accountId: "acc_1", audioUrl: audio }); - expect(describeMoment).toHaveBeenCalledWith({ audioUrl: audio, start: 36.15, end: 50.9 }); - expect(describeMoment).toHaveBeenCalledTimes(2); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(2); - expect(chargeForFlamingoCall).toHaveBeenCalledWith({ + expect(describeMoments).toHaveBeenCalledTimes(1); + expect(describeMoments).toHaveBeenCalledWith({ audioUrl: audio, moments: [hook, climax] }); + expect(chargeForBatchAnswers).toHaveBeenCalledTimes(1); + expect(chargeForBatchAnswers).toHaveBeenCalledWith({ accountId: "acc_1", - elapsedSeconds: 2, + answers: [...described("vocal_hook", "climax").values()], audioUrl: audio, }); expect(result.moments[0]).toEqual({ ...hook, ...words }); @@ -79,31 +81,31 @@ describe("processSongMomentsRequest", () => { expect(callMeasureSongMoments).toHaveBeenCalledWith({ audio_url: audio, tempo_hint: 86 }); }); - it("describes nothing when describe is false", async () => { + it("describes nothing, and charges no model time, when describe is false", async () => { const result = await processSongMomentsRequest( { audio_url: audio, describe: false }, { accountId: "acc_1" }, ); - expect(describeMoment).not.toHaveBeenCalled(); - expect(chargeForFlamingoCall).not.toHaveBeenCalled(); + expect(describeMoments).not.toHaveBeenCalled(); + expect(chargeForBatchAnswers).toHaveBeenCalledWith(expect.objectContaining({ answers: [] })); expect(result.moments).toEqual([hook, climax]); }); it("describes only the listed moment types", async () => { - await processSongMomentsRequest( + vi.mocked(describeMoments).mockResolvedValue(described("climax")); + + const result = await processSongMomentsRequest( { audio_url: audio, describe: ["climax"] }, { accountId: "acc_1" }, ); - expect(describeMoment).toHaveBeenCalledTimes(1); - expect(describeMoment).toHaveBeenCalledWith({ audioUrl: audio, start: 143.36, end: 158.1 }); + expect(describeMoments).toHaveBeenCalledWith({ audioUrl: audio, moments: [climax] }); + expect(result.moments).toEqual([hook, { ...climax, ...words }]); }); - it("returns a moment without words, and charges nothing for it, when its description fails", async () => { - vi.mocked(describeMoment) - .mockResolvedValueOnce({ description: words, elapsed_seconds: 2 }) - .mockRejectedValueOnce(new Error("Music analysis failed (status 503)")); + it("returns a moment without words, and charges nothing for it, when the model drops it", async () => { + vi.mocked(describeMoments).mockResolvedValue(described("vocal_hook")); const result = await processSongMomentsRequest( { audio_url: audio, describe: true }, @@ -111,7 +113,23 @@ describe("processSongMomentsRequest", () => { ); expect(result.moments[1]).toEqual(climax); - expect(chargeForFlamingoCall).toHaveBeenCalledTimes(1); + const [charge] = vi.mocked(chargeForBatchAnswers).mock.calls[0]; + expect(charge.answers).toEqual([...described("vocal_hook").values()]); + }); + + it("returns every moment without words, and charges no model time, when the batch fails", async () => { + vi.mocked(describeMoments).mockRejectedValue(new Error("Music analysis failed (status 503)")); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await processSongMomentsRequest( + { audio_url: audio, describe: true }, + { accountId: "acc_1" }, + ); + + expect(result.moments).toEqual([hook, climax]); + expect(chargeForSongMeasure).toHaveBeenCalledTimes(1); + expect(chargeForBatchAnswers).toHaveBeenCalledWith(expect.objectContaining({ answers: [] })); + errorSpy.mockRestore(); }); it("charges nothing when the measurement fails", async () => { @@ -123,6 +141,6 @@ describe("processSongMomentsRequest", () => { processSongMomentsRequest({ audio_url: audio, describe: true }, { accountId: "acc_1" }), ).rejects.toThrow("status 422"); expect(chargeForSongMeasure).not.toHaveBeenCalled(); - expect(describeMoment).not.toHaveBeenCalled(); + expect(describeMoments).not.toHaveBeenCalled(); }); }); diff --git a/lib/songMoments/describeMoment.ts b/lib/songMoments/describeMoment.ts deleted file mode 100644 index 2ac021e31..000000000 --- a/lib/songMoments/describeMoment.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { callFlamingoGenerate } from "@/lib/flamingo/callFlamingoGenerate"; -import { - parseMomentDescription, - type MomentDescription, -} from "@/lib/songMoments/parseMomentDescription"; - -const MOMENT_DESCRIPTION_PROMPT = `You are hearing a short clip cut from a song, for music licensing. Describe only this clip. Return ONLY a JSON object with double quotes: {"description": one sentence on what happens musically in the clip, "emotion": up to 3 mood words, "vocals": true or false, "best_for": up to 3 uses chosen from ["film trailer", "TV promo", "ad", "social video", "video game", "sports", "documentary", "end credits"]}. No other text.`; - -const MAX_NEW_TOKENS = 256; - -/** - * Has Music Flamingo listen to one moment, cut from the song on the model's - * side, and describe it. The times come from the audio measurement; the model - * only supplies words, never times. - * - * @param params - The song's audio URL and the moment's start and end in seconds. - * @returns The parsed description and the model's reported seconds, for billing. - * @throws Error when the model call fails (nothing to charge for). - */ -export async function describeMoment(params: { - audioUrl: string; - start: number; - end: number; -}): Promise<{ description: MomentDescription; elapsed_seconds: number }> { - const result = await callFlamingoGenerate( - { - prompt: MOMENT_DESCRIPTION_PROMPT, - audio_url: params.audioUrl, - max_new_tokens: MAX_NEW_TOKENS, - temperature: 1.0, - top_p: 1.0, - do_sample: false, - }, - { clip: { start: params.start, end: params.end } }, - ); - return { - description: parseMomentDescription(result.response), - elapsed_seconds: result.elapsed_seconds, - }; -} diff --git a/lib/songMoments/describeMoments.ts b/lib/songMoments/describeMoments.ts new file mode 100644 index 000000000..651ed6c42 --- /dev/null +++ b/lib/songMoments/describeMoments.ts @@ -0,0 +1,72 @@ +import { callFlamingoGenerateBatch } from "@/lib/flamingo/callFlamingoGenerateBatch"; +import type { MeasuredMoment } from "@/lib/songMoments/isMeasuredSong"; +import { + parseMomentDescription, + type MomentDescription, +} from "@/lib/songMoments/parseMomentDescription"; +import type { SongMomentType } from "@/lib/songMoments/songMomentsBodySchema"; + +const MOMENT_DESCRIPTION_PROMPT = `You are hearing a short clip cut from a song, for music licensing. Describe only this clip. Return ONLY a JSON object with double quotes: {"description": one sentence on what happens musically in the clip, "emotion": up to 3 mood words, "vocals": true or false, "best_for": up to 3 uses chosen from ["film trailer", "TV promo", "ad", "social video", "video game", "sports", "documentary", "end credits"]}. No other text.`; + +const MAX_NEW_TOKENS = 256; + +/** One moment's words, and the model seconds they took (for billing). */ +export interface DescribedMoment { + description: MomentDescription; + elapsed_seconds: number; +} + +/** + * Has Music Flamingo describe every given moment in ONE batch: the song is + * downloaded once and each prompt hears only its own moment's clip, cut on + * the model's side. The times come from the audio measurement; the model + * only supplies words, never times. + * + * An answer counts only when it echoes the clip it was asked about, so a + * description can never come from the whole song (for example from a model + * service that does not cut clips yet), and only when its reply parses into + * at least one field, so an unusable reply is neither shown nor billed. + * + * @param params - The song's audio URL and the moments to describe (at most + * one of each type, so the type is each prompt's id). + * @returns The described moments' words and seconds, keyed by moment type; + * a moment the model did not answer for is absent. + * @throws When the batch call fails (nothing to charge for). + */ +export async function describeMoments(params: { + audioUrl: string; + moments: Pick[]; +}): Promise> { + const result = await callFlamingoGenerateBatch({ + audio_url: params.audioUrl, + prompts: params.moments.map(moment => ({ + id: moment.type, + prompt: MOMENT_DESCRIPTION_PROMPT, + max_new_tokens: MAX_NEW_TOKENS, + clip: { start: moment.start, end: moment.end }, + })), + temperature: 1.0, + top_p: 1.0, + do_sample: false, + }); + + const described = new Map(); + for (const moment of params.moments) { + const answer = result.answers.find(a => a.id === moment.type); + if (!answer || answer.clip_start !== moment.start || answer.clip_end !== moment.end) continue; + const description = parseMomentDescription(answer.response); + if (!hasWords(description)) continue; + described.set(moment.type, { description, elapsed_seconds: answer.elapsed_seconds }); + } + return described; +} + +/** False for the empty shape `parseMomentDescription` returns when a reply has nothing usable. */ +function hasWords(description: MomentDescription): boolean { + return ( + description.description !== null || + description.emotion.length > 0 || + description.best_for.length > 0 || + description.vocals !== null + ); +} diff --git a/lib/songMoments/minimumCreditsForSongMomentsRequest.ts b/lib/songMoments/minimumCreditsForSongMomentsRequest.ts index 80db4e313..35547a7a5 100644 --- a/lib/songMoments/minimumCreditsForSongMomentsRequest.ts +++ b/lib/songMoments/minimumCreditsForSongMomentsRequest.ts @@ -1,11 +1,11 @@ import { FLAMINGO_BASE_CREDITS } from "@/lib/flamingo/creditsForFlamingoCall"; import { SONG_MEASURE_CREDITS } from "@/lib/songMoments/songMeasureCredits"; -import { SONG_MOMENT_TYPES, type SongMomentsBody } from "@/lib/songMoments/songMomentsBodySchema"; +import type { SongMomentsBody } from "@/lib/songMoments/songMomentsBodySchema"; /** * The pre-flight credit gate for a song-moments request: the measurement - * price plus one model base fee for every moment that might be described. - * A song has at most one moment of each type, so that bounds the calls. + * price plus one model base fee when any moment may be described, since + * every described moment rides in the same batch. * * @param body - The validated request; only `describe` matters. * @returns Credits the account must hold before anything runs. @@ -13,11 +13,7 @@ import { SONG_MOMENT_TYPES, type SongMomentsBody } from "@/lib/songMoments/songM export function minimumCreditsForSongMomentsRequest( body: Pick, ): number { - const calls = - body.describe === true - ? SONG_MOMENT_TYPES.length - : body.describe === false - ? 0 - : body.describe.length; - return SONG_MEASURE_CREDITS + calls * FLAMINGO_BASE_CREDITS; + const describes = + body.describe === true || (Array.isArray(body.describe) && body.describe.length > 0); + return SONG_MEASURE_CREDITS + (describes ? FLAMINGO_BASE_CREDITS : 0); } diff --git a/lib/songMoments/processSongMomentsRequest.ts b/lib/songMoments/processSongMomentsRequest.ts index b81369f7e..efefa7008 100644 --- a/lib/songMoments/processSongMomentsRequest.ts +++ b/lib/songMoments/processSongMomentsRequest.ts @@ -1,10 +1,10 @@ -import { chargeForFlamingoCall } from "@/lib/flamingo/chargeForFlamingoCall"; +import { chargeForBatchAnswers } from "@/lib/flamingo/chargeForBatchAnswers"; import { callMeasureSongMoments } from "@/lib/songMoments/callMeasureSongMoments"; import { chargeForSongMeasure } from "@/lib/songMoments/chargeForSongMeasure"; -import { describeMoment } from "@/lib/songMoments/describeMoment"; +import { describeMoments, type DescribedMoment } from "@/lib/songMoments/describeMoments"; import type { MeasuredMoment, MeasuredSong } from "@/lib/songMoments/isMeasuredSong"; import type { MomentDescription } from "@/lib/songMoments/parseMomentDescription"; -import type { SongMomentsBody } from "@/lib/songMoments/songMomentsBodySchema"; +import type { SongMomentsBody, SongMomentType } from "@/lib/songMoments/songMomentsBodySchema"; /** A moment with Music Flamingo's words added when it was described. */ export type SongMoment = MeasuredMoment & Partial; @@ -15,12 +15,14 @@ export interface SongMomentsResult extends Omit { /** * Measures a song's sections and moments, then has Music Flamingo describe - * the chosen moments, each from a clip of just that moment. + * the chosen moments in one batch, each prompt hearing a clip of just that + * moment. * * Charges `accountId` the flat measurement price once the measurement returns, - * and each model call on its own reported seconds, like /api/songs/analyze. - * A moment whose description fails is returned without words and is not - * charged. Callers verify `audio_url` and gate the balance first. + * and the batch once: one base price plus the seconds of the moments it + * described. A moment the model did not describe (or every moment, when the + * batch fails) is returned without words and is not charged. Callers verify + * `audio_url` and gate the balance first. * * @param body - Validated request body. * @param context - The account the work is charged to. @@ -39,31 +41,31 @@ export async function processSongMomentsRequest( }); await chargeForSongMeasure({ accountId: context.accountId, audioUrl: body.audio_url }); - const wanted = (moment: MeasuredMoment) => - body.describe === true || (Array.isArray(body.describe) && body.describe.includes(moment.type)); - - const moments = await Promise.all( - song.moments.map(async (moment): Promise => { - if (!wanted(moment)) return moment; - try { - const { description, elapsed_seconds } = await describeMoment({ - audioUrl: body.audio_url, - start: moment.start, - end: moment.end, - }); - await chargeForFlamingoCall({ - accountId: context.accountId, - elapsedSeconds: elapsed_seconds, - audioUrl: body.audio_url, - }); - return { ...moment, ...description }; - } catch (error) { - console.error(`[processSongMomentsRequest] describing ${moment.type} failed:`, error); - return moment; - } - }), + const wanted = song.moments.filter( + moment => + body.describe === true || + (Array.isArray(body.describe) && body.describe.includes(moment.type)), ); + let described = new Map(); + if (wanted.length > 0) { + try { + described = await describeMoments({ audioUrl: body.audio_url, moments: wanted }); + } catch (error) { + console.error("[processSongMomentsRequest] describing moments failed:", error); + } + } + await chargeForBatchAnswers({ + accountId: context.accountId, + answers: [...described.values()], + audioUrl: body.audio_url, + }); + + const moments = song.moments.map((moment): SongMoment => { + const words = described.get(moment.type); + return words ? { ...moment, ...words.description } : moment; + }); + return { ...song, moments, diff --git a/modal/README.md b/modal/README.md index e11e53d20..99667dcda 100644 --- a/modal/README.md +++ b/modal/README.md @@ -20,11 +20,28 @@ workspace settings. ## Capacity `MAX_CONTAINERS` is 4 (workspace GPU cap is 10). Recoup `full_report` -fans out 13 HTTP calls; they only run in parallel if this cap is >1. +sends its 13 sections as one `generate_batch` request, so one report holds +one container and the sections run one after another on it. + +## Batches + +`generate_batch` downloads (and transcodes) the audio once and answers up to +40 prompts against it. A prompt may carry `clip_start`/`clip_end` (seconds, +both or neither) to hear only that stretch, cut fresh from the one download +for each prompt, and its own `temperature`, `top_p` or `do_sample` to +override the batch's settings for that prompt alone. An answer from a clip +echoes `clip_start`/`clip_end`, so a caller can tell it was not answered from +the whole song. The validation and the batch loop are pure Python and tested +locally (ffmpeg and ffprobe must be on the PATH): + +```bash +uv run --no-project --with modal --with pytest \ + python -I -m pytest modal/test_serve_music_flamingo.py -q +``` ## Auth -Both web endpoints (`generate`, `health`) require Modal proxy auth +Every web endpoint (`generate`, `generate_batch`, `health`) requires Modal proxy auth (`requires_proxy_auth=True`): requests must carry `Modal-Key` and `Modal-Secret` headers from a proxy auth token minted in the `shared-78369` workspace settings. Recoup API sends them from the @@ -40,9 +57,10 @@ Modal URL. app behind `POST /api/songs/moments`. It is CPU only: `song_moments.py` measures the audio (beat grid, loudness, which passages repeat) to find a song's sections and moments, and names sections from an optional lyric-sheet -section order. Recoup API then has Music Flamingo describe each moment by -sending `clip_start`/`clip_end` to the `generate` endpoint above, which cuts -the clip before the model hears it. +section order. Recoup API then has Music Flamingo describe the moments in one +`generate_batch` request above, one prompt per moment with its +`clip_start`/`clip_end`, so the song downloads once and each prompt hears +only its clip. The same workflow deploys it, after `music-flamingo`. Its `measure` endpoint uses the same proxy auth as Music Flamingo; Recoup API calls it at diff --git a/modal/serve_music_flamingo.py b/modal/serve_music_flamingo.py index 7146322f5..207e921a3 100644 --- a/modal/serve_music_flamingo.py +++ b/modal/serve_music_flamingo.py @@ -84,6 +84,33 @@ def download_model_weights(): print(f"Model weights saved to {MODEL_CACHE_DIR}/model") +def extract_clip(path, start, end): + """ + Cut start..end seconds of `path` into a new 16 kHz mono WAV and return its + path. The source is kept, so a batch can cut several clips from one + download; the caller deletes the clip. + """ + import os + import subprocess + import tempfile + + if not 0 <= start < end: + raise ValueError("clip_start must be at least 0 and before clip_end") + clip = tempfile.NamedTemporaryFile(suffix="-clip.wav", delete=False) + clip.close() + try: + subprocess.run( + ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-ss", str(start), "-t", str(end - start), + "-i", path, "-ac", "1", "-ar", "16000", clip.name], + check=True, + timeout=60, + ) + except Exception: + os.unlink(clip.name) + raise + return clip.name + + def cut_clip(path, start, end): """ Replace the downloaded file with just start..end seconds of it, so the @@ -91,21 +118,136 @@ def cut_clip(path, start, end): path; the original is removed. """ import os - import subprocess - if not 0 <= start < end: - raise ValueError("clip_start must be at least 0 and before clip_end") - clip_path = os.path.splitext(path)[0] + "-clip.wav" - subprocess.run( - ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-ss", str(start), "-t", str(end - start), - "-i", path, "-ac", "1", "-ar", "16000", clip_path], - check=True, - timeout=60, - ) + clip_path = extract_clip(path, start, end) os.unlink(path) return clip_path +def is_number(value) -> bool: + """ + A finite JSON number: int or float, but not bool (bool is an int subclass + in Python), NaN, infinity, or an int too large to be a float. + """ + import math + + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(float(value)) + except OverflowError: + return False + + +def check_generation_settings(settings: dict, where: str = "") -> None: + """ + Raises ValueError unless `temperature` and `top_p` (when present) are + numbers and `do_sample` (when present) is a boolean. `where` prefixes the + message, e.g. "prompts[2]." for one prompt's own settings. + """ + for key in ("temperature", "top_p"): + if key in settings and not is_number(settings[key]): + raise ValueError(f"{where}{key} must be a number") + if "do_sample" in settings and not isinstance(settings["do_sample"], bool): + raise ValueError(f"{where}do_sample must be a boolean") + + +def parse_batch_prompts(prompts, defaults: dict) -> list: + """ + Validates /generate_batch's `prompts` and resolves each one into a job. + + A prompt may carry `clip_start` and `clip_end` (seconds, both or neither) + to be answered from just that stretch of the audio, and `temperature`, + `top_p` or `do_sample` to override the batch's `defaults` for that prompt + alone. Returns one job dict per prompt, in order; raises ValueError naming + the first invalid field. + """ + if not isinstance(prompts, list) or not prompts: + raise ValueError("prompts must be a non-empty list") + if len(prompts) > MAX_BATCH_PROMPTS: + raise ValueError(f"prompts cannot contain more than {MAX_BATCH_PROMPTS} items") + + jobs = [] + seen_ids = set() + for index, item in enumerate(prompts): + if not isinstance(item, dict): + raise ValueError(f"prompts[{index}] must be an object") + prompt_id = item.get("id") + prompt = item.get("prompt") + max_tokens = item.get("max_new_tokens", MAX_NEW_TOKENS) + if not isinstance(prompt_id, str) or not prompt_id: + raise ValueError(f"prompts[{index}].id must be a non-empty string") + if prompt_id in seen_ids: + raise ValueError(f"prompts[{index}].id '{prompt_id}' is repeated") + seen_ids.add(prompt_id) + if not isinstance(prompt, str) or not prompt: + raise ValueError(f"prompts[{index}].prompt must be a non-empty string") + # bool is an int subclass in Python; refuse it explicitly + if ( + isinstance(max_tokens, bool) + or not isinstance(max_tokens, int) + or not 1 <= max_tokens <= MAX_NEW_TOKENS_LIMIT + ): + raise ValueError(f"prompts[{index}].max_new_tokens must be 1-{MAX_NEW_TOKENS_LIMIT}") + check_generation_settings(item, f"prompts[{index}].") + + clip = None + clip_start, clip_end = item.get("clip_start"), item.get("clip_end") + if clip_start is not None or clip_end is not None: + if not (is_number(clip_start) and is_number(clip_end) and 0 <= clip_start < clip_end): + raise ValueError( + f"prompts[{index}] needs clip_start and clip_end in seconds, " + "with 0 <= clip_start < clip_end" + ) + clip = (float(clip_start), float(clip_end)) + + jobs.append({ + "id": prompt_id, + "prompt": prompt, + "max_new_tokens": max_tokens, + "clip": clip, + "temperature": item.get("temperature", defaults["temperature"]), + "top_p": item.get("top_p", defaults["top_p"]), + "do_sample": item.get("do_sample", defaults["do_sample"]), + }) + return jobs + + +def run_batch(jobs: list, audio_path: str, run_prompt) -> list: + """ + Answers each job (from parse_batch_prompts) against `audio_path`, or + against a clip cut from it when the job has one; every clip is deleted + after its prompt, whether or not the prompt succeeds. `run_prompt(prompt, + audio_path, max_new_tokens, temperature, top_p, do_sample)` returns the + answer text. Returns the answers in job order; an answer from a clip + echoes the window so the caller can tell it came from the clip. + """ + import os + import time + + answers = [] + for job in jobs: + started = time.time() + clip_path = extract_clip(audio_path, *job["clip"]) if job["clip"] else None + try: + response = run_prompt( + job["prompt"], clip_path or audio_path, job["max_new_tokens"], + job["temperature"], job["top_p"], job["do_sample"], + ) + finally: + if clip_path: + os.unlink(clip_path) + answer = { + "id": job["id"], + "response": response, + "elapsed_seconds": round(time.time() - started, 2), + } + if job["clip"]: + answer["clip_start"], answer["clip_end"] = job["clip"] + answers.append(answer) + return answers + + # --------------------------------------------------------------------------- # Modal Image — the "Docker image" that runs on the cloud GPU. # Key optimization: .run_function(download_model_weights) bakes the 16GB @@ -358,7 +500,11 @@ def generate_batch(self, request: dict): "audio_url": "https://example.com/s.mp3", (required) "prompts": [ (required, 1-40) {"id": "genre", "prompt": "What genre is this?", - "max_new_tokens": 64} (optional, 1-2048) + "max_new_tokens": 64, (optional, 1-2048) + "clip_start": 45.2, "clip_end": 60.4, (optional, seconds; both or + neither: the prompt hears + only that clip) + "temperature": 0.7, "top_p": 0.9, "do_sample": true} (optional, this prompt only) ], "temperature": 0.2, (optional, default 1.0) "top_p": 1.0, (optional, default 1.0) @@ -367,7 +513,8 @@ def generate_batch(self, request: dict): Returns: { - "answers": [{"id": "genre", "response": "...", "elapsed_seconds": 1.2}], + "answers": [{"id": "genre", "response": "...", "elapsed_seconds": 1.2, + "clip_start": 45.2, "clip_end": 60.4}], (clip echoed when one was cut) "elapsed_seconds": 9.87 (whole batch, download included) } @@ -388,63 +535,24 @@ def reject(detail: str): if not isinstance(audio_url, str) or not audio_url: reject("audio_url is required") - prompts = request.get("prompts") - if not isinstance(prompts, list) or not prompts: - reject("prompts must be a non-empty list") - if len(prompts) > MAX_BATCH_PROMPTS: - reject(f"prompts cannot contain more than {MAX_BATCH_PROMPTS} items") - - jobs = [] - seen_ids = set() - for index, item in enumerate(prompts): - if not isinstance(item, dict): - reject(f"prompts[{index}] must be an object") - prompt_id = item.get("id") - prompt = item.get("prompt") - max_tokens = item.get("max_new_tokens", MAX_NEW_TOKENS) - if not isinstance(prompt_id, str) or not prompt_id: - reject(f"prompts[{index}].id must be a non-empty string") - if prompt_id in seen_ids: - reject(f"prompts[{index}].id '{prompt_id}' is repeated") - seen_ids.add(prompt_id) - if not isinstance(prompt, str) or not prompt: - reject(f"prompts[{index}].prompt must be a non-empty string") - # bool is an int subclass in Python; refuse it explicitly - if ( - isinstance(max_tokens, bool) - or not isinstance(max_tokens, int) - or not 1 <= max_tokens <= MAX_NEW_TOKENS_LIMIT - ): - reject(f"prompts[{index}].max_new_tokens must be 1-{MAX_NEW_TOKENS_LIMIT}") - jobs.append((prompt_id, prompt, max_tokens)) - - # Generation parameters apply to every prompt in the batch - temperature = request.get("temperature", 1.0) - top_p = request.get("top_p", 1.0) - do_sample = request.get("do_sample", False) - if isinstance(temperature, bool) or not isinstance(temperature, (int, float)): - reject("temperature must be a number") - if isinstance(top_p, bool) or not isinstance(top_p, (int, float)): - reject("top_p must be a number") - if not isinstance(do_sample, bool): - reject("do_sample must be a boolean") + # Generation parameters apply to every prompt that does not set its own + defaults = { + "temperature": request.get("temperature", 1.0), + "top_p": request.get("top_p", 1.0), + "do_sample": request.get("do_sample", False), + } + try: + jobs = parse_batch_prompts(request.get("prompts"), defaults) + check_generation_settings(defaults) + except ValueError as err: + reject(str(err)) # ---- Download (and transcode) the audio ONCE ---- tmp_path = self.download_audio(audio_url) - # ---- Run every prompt against the same file ---- - answers = [] + # ---- Run every prompt against the same file (or a clip cut from it) ---- try: - for prompt_id, prompt, max_tokens in jobs: - prompt_start = time.time() - response = self.run_prompt( - prompt, tmp_path, max_tokens, temperature, top_p, do_sample - ) - answers.append({ - "id": prompt_id, - "response": response, - "elapsed_seconds": round(time.time() - prompt_start, 2), - }) + answers = run_batch(jobs, tmp_path, self.run_prompt) finally: os.unlink(tmp_path) diff --git a/modal/test_serve_music_flamingo.py b/modal/test_serve_music_flamingo.py new file mode 100644 index 000000000..2819433ac --- /dev/null +++ b/modal/test_serve_music_flamingo.py @@ -0,0 +1,220 @@ +""" +Tests for the pure helpers behind /generate_batch: prompt validation with +per-prompt clips and settings, and cutting clips from one download. + +Needs `modal` importable (the module defines the Modal app) and ffmpeg: + uv run --no-project --with modal --with pytest \ + python -I -m pytest modal/test_serve_music_flamingo.py -q +""" + +import os +import subprocess + +import pytest + +from serve_music_flamingo import ( + MAX_BATCH_PROMPTS, + check_generation_settings, + cut_clip, + extract_clip, + is_number, + parse_batch_prompts, + run_batch, +) + +DEFAULTS = {"temperature": 0.2, "top_p": 1.0, "do_sample": False} + + +def tone(path, seconds): + """Write a sine tone of `seconds` to `path` with ffmpeg.""" + subprocess.run( + ["ffmpeg", "-nostdin", "-loglevel", "error", "-y", "-f", "lavfi", + "-i", f"sine=frequency=440:duration={seconds}", path], + check=True, + ) + return path + + +def duration(path): + out = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path], + check=True, capture_output=True, text=True, + ) + return float(out.stdout.strip()) + + +def test_prompts_take_the_batch_settings_when_they_set_none(): + jobs = parse_batch_prompts([{"id": "genre", "prompt": "Genre?"}], DEFAULTS) + + assert jobs == [{ + "id": "genre", "prompt": "Genre?", "max_new_tokens": 512, "clip": None, + "temperature": 0.2, "top_p": 1.0, "do_sample": False, + }] + + +def test_a_prompt_overrides_the_batch_settings_for_itself_only(): + jobs = parse_batch_prompts([ + {"id": "pitch", "prompt": "Pitch it.", "temperature": 0.6, "do_sample": True}, + {"id": "key", "prompt": "Key?"}, + ], DEFAULTS) + + assert (jobs[0]["temperature"], jobs[0]["top_p"], jobs[0]["do_sample"]) == (0.6, 1.0, True) + assert (jobs[1]["temperature"], jobs[1]["do_sample"]) == (0.2, False) + + +def test_a_prompt_with_a_clip_window_carries_it_as_seconds(): + jobs = parse_batch_prompts( + [{"id": "climax", "prompt": "Describe.", "clip_start": 36, "clip_end": 50.9}], DEFAULTS + ) + + assert jobs[0]["clip"] == (36.0, 50.9) + + +@pytest.mark.parametrize("window", [ + {"clip_start": 10}, + {"clip_end": 10}, + {"clip_start": 10, "clip_end": 10}, + {"clip_start": 12, "clip_end": 10}, + {"clip_start": -1, "clip_end": 10}, + {"clip_start": True, "clip_end": 10}, + {"clip_start": "0", "clip_end": 10}, +]) +def test_a_clip_needs_both_ends_in_order(window): + with pytest.raises(ValueError, match=r"prompts\[0\] needs clip_start and clip_end"): + parse_batch_prompts([{"id": "a", "prompt": "Describe.", **window}], DEFAULTS) + + +@pytest.mark.parametrize("item, message", [ + ({"id": "a", "prompt": "x", "temperature": "hot"}, r"prompts\[0\]\.temperature must be a number"), + ({"id": "a", "prompt": "x", "top_p": True}, r"prompts\[0\]\.top_p must be a number"), + ({"id": "a", "prompt": "x", "do_sample": 1}, r"prompts\[0\]\.do_sample must be a boolean"), + ({"id": "a", "prompt": "x", "max_new_tokens": 0}, r"prompts\[0\]\.max_new_tokens must be 1-2048"), + ({"id": "", "prompt": "x"}, r"prompts\[0\]\.id must be a non-empty string"), + ({"id": "a", "prompt": ""}, r"prompts\[0\]\.prompt must be a non-empty string"), +]) +def test_invalid_prompt_fields_are_named(item, message): + with pytest.raises(ValueError, match=message): + parse_batch_prompts([item], DEFAULTS) + + +def test_repeated_ids_and_list_bounds_are_refused(): + with pytest.raises(ValueError, match="is repeated"): + parse_batch_prompts([{"id": "a", "prompt": "x"}, {"id": "a", "prompt": "y"}], DEFAULTS) + with pytest.raises(ValueError, match="non-empty list"): + parse_batch_prompts([], DEFAULTS) + too_many = [{"id": str(i), "prompt": "x"} for i in range(MAX_BATCH_PROMPTS + 1)] + with pytest.raises(ValueError, match="more than 40"): + parse_batch_prompts(too_many, DEFAULTS) + + +def test_batch_level_settings_are_checked_without_a_prefix(): + check_generation_settings(DEFAULTS) + with pytest.raises(ValueError, match="^temperature must be a number$"): + check_generation_settings({**DEFAULTS, "temperature": None}) + + +def test_extract_clip_cuts_the_window_and_keeps_the_source(tmp_path): + source = tone(str(tmp_path / "song.wav"), 4) + + first = extract_clip(source, 0.5, 1.5) + second = extract_clip(source, 2.0, 3.0) + try: + assert os.path.exists(source) + assert first != second + assert duration(first) == pytest.approx(1.0, abs=0.05) + assert duration(second) == pytest.approx(1.0, abs=0.05) + finally: + os.unlink(first) + os.unlink(second) + + +def test_cut_clip_replaces_the_source_with_the_clip(tmp_path): + source = tone(str(tmp_path / "song.wav"), 4) + + clip = cut_clip(source, 1.0, 3.0) + try: + assert not os.path.exists(source) + assert duration(clip) == pytest.approx(2.0, abs=0.05) + finally: + os.unlink(clip) + + +def test_extract_clip_refuses_a_backwards_window(tmp_path): + source = tone(str(tmp_path / "song.wav"), 2) + + with pytest.raises(ValueError, match="before clip_end"): + extract_clip(source, 1.5, 1.0) + + +@pytest.mark.parametrize("value, ok", [ + (3, True), (2.5, True), (0, True), + (True, False), ("1", False), (None, False), + (float("inf"), False), (float("nan"), False), (10 ** 400, False), +]) +def test_is_number_takes_only_finite_numbers(value, ok): + assert is_number(value) is ok + + +def test_a_clip_window_at_infinity_or_too_large_for_a_float_is_refused(): + for end in (float("inf"), 10 ** 400): + with pytest.raises(ValueError, match="needs clip_start and clip_end"): + parse_batch_prompts([{"id": "a", "prompt": "x", "clip_start": 0, "clip_end": end}], DEFAULTS) + with pytest.raises(ValueError, match="temperature must be a number"): + parse_batch_prompts([{"id": "a", "prompt": "x", "temperature": float("nan")}], DEFAULTS) + + +class FakeModel: + """Stands in for the GPU: records each call and what audio it was given.""" + + def __init__(self, fail_on=None): + self.calls = [] + self.fail_on = fail_on + + def __call__(self, prompt, audio_path, max_new_tokens, temperature, top_p, do_sample): + self.calls.append({ + "prompt": prompt, "audio_path": audio_path, "existed": os.path.exists(audio_path), + "seconds": duration(audio_path), "max_new_tokens": max_new_tokens, + "temperature": temperature, "top_p": top_p, "do_sample": do_sample, + }) + if prompt == self.fail_on: + raise RuntimeError("model failed") + return f"answer to {prompt}" + + +def test_run_batch_answers_in_order_with_each_prompts_own_settings_and_clip(tmp_path): + source = tone(str(tmp_path / "song.wav"), 4) + jobs = parse_batch_prompts([ + {"id": "whole", "prompt": "Genre?"}, + {"id": "hook", "prompt": "Describe.", "clip_start": 1, "clip_end": 2.5, + "temperature": 0.7, "do_sample": True}, + ], DEFAULTS) + model = FakeModel() + + answers = run_batch(jobs, source, model) + + assert [a["id"] for a in answers] == ["whole", "hook"] + assert answers[0]["response"] == "answer to Genre?" + assert "clip_start" not in answers[0] + assert (answers[1]["clip_start"], answers[1]["clip_end"]) == (1.0, 2.5) + whole, hook = model.calls + assert whole["audio_path"] == source and whole["seconds"] == pytest.approx(4, abs=0.05) + assert (whole["temperature"], whole["do_sample"]) == (0.2, False) + assert hook["audio_path"] != source and hook["existed"] + assert hook["seconds"] == pytest.approx(1.5, abs=0.05) + assert (hook["temperature"], hook["top_p"], hook["do_sample"]) == (0.7, 1.0, True) + assert not os.path.exists(hook["audio_path"]) + assert os.path.exists(source) + + +def test_run_batch_deletes_the_clip_when_the_model_fails(tmp_path): + source = tone(str(tmp_path / "song.wav"), 3) + jobs = parse_batch_prompts( + [{"id": "hook", "prompt": "Describe.", "clip_start": 0.5, "clip_end": 1.5}], DEFAULTS + ) + model = FakeModel(fail_on="Describe.") + + with pytest.raises(RuntimeError, match="model failed"): + run_batch(jobs, source, model) + + assert not os.path.exists(model.calls[0]["audio_path"]) + assert os.path.exists(source)