diff --git a/README.md b/README.md index a33bcfd..0a02736 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ The 159 tools are organized around the OBS protocol rather than a smaller opinio - transitions, transition overrides, the transition cursor, and the T-Bar - canvases, screenshots, profiles, scene collections, and persistent data - a recording preflight (`obs-preflight`) that catches the conditions under which OBS silently refuses or botches a recording -- `obs-record-clip`, which preflights, records up to 50 seconds with chapter markers, stops, and checks the file's length and audio with ffprobe/ffmpeg when they are installed +- `obs-record-clip`, which preflights, records up to 50 seconds with chapter markers, stops, and checks the file's length and audio with ffprobe/ffmpeg when they are installed. While it records, it watches the take (see [Watching a take](#watching-a-take)) - `obs-capture-window` (macOS), which points a `screen_capture` input at a window or app by name, silences it, and fits it to the canvas - input and transform changes that return the resulting settings, size, and an optional screenshot, warning when a source renders at 0×0 - protocol description, the guarded generic request fallback, and `obs-batch`, which sends several requests in one message, in order or one per rendered frame (so a change to two sources lands on the same frame), with `Sleep` between them. OBS's parallel mode is not offered: in obs-websocket 5.7, concurrent parallel batches deadlock its WebSocket server until OBS restarts @@ -157,6 +157,7 @@ Resources give clients that support them a live view without tool calls. Each on | `obs://scenes` | `GetSceneList` | scenes added, removed, renamed, or switched | | `obs://scene/{sceneName}/items` | `GetSceneItemList` | items added, removed, reordered, shown, hidden, locked | | `obs://scene/{sceneName}/screenshot` | 960px JPEG of the scene | (read on demand) | +| `obs://take/current` | the running take: audio levels, skipped frames, black or unchanging picture, chapters, warnings | a warning or chapter | Clients on the 2025 protocol receive `notifications/resources/updated` for URIs they subscribe to; 2026-07-28 clients choose them on their `subscriptions/listen` stream. @@ -186,6 +187,16 @@ Pay particular attention to approvals for tools that: `obs-call-request` and `obs-batch` accept only request types in the bundled OBS protocol, but their payloads are intentionally generic. With `OBS_MCP_CONFIRM_LIVE=true`, they ask before sending `StopStream`, `ToggleStream`, `StopRecord`, or `ToggleRecord`. Both are always advertised as destructive and open-world so a client does not silently treat an unfamiliar operation as safe. +### Watching a take + +While a take records, the server watches OBS instead of waiting for the file: + +- **Audio**: it subscribes to OBS's `InputVolumeMeters` events for the length of the take and records each input's loudest level after volume and mute. An input on a recorded track that rises above −60 dBFS is logged, and with `expectSilent` it is a problem. Muted inputs and inputs on unrecorded tracks are not. +- **Frames**: it polls `GetStats` every 2 seconds and warns when OBS misses frames rendering or the encoder skips them. +- **Picture**: once a second it takes a 32-pixel-wide PPM screenshot of the program scene. Two black samples in a row raise a warning (a capture that lost its window or permission looks like this). Stretches of 3 seconds or more with no change are recorded as stills, which mark dead time to trim. + +The result lists the findings, and when OBS records to this machine the server writes them next to the recording as `.take.json`, with chapter times and OBS events (scene switches, mutes, output state) in seconds from the start. `obs://take/current` shows the same data while the take runs. + ## Troubleshooting **Tools appear, but OBS calls fail:** make sure OBS is open and its WebSocket server is enabled. The MCP process stays available while it retries the connection. diff --git a/src/client.test.ts b/src/client.test.ts index 859f352..840b595 100644 --- a/src/client.test.ts +++ b/src/client.test.ts @@ -306,6 +306,48 @@ describe("OBSWebSocketClient request batches", () => { }); }); +describe("OBSWebSocketClient high-volume events", () => { + it("adds a subscription while anyone holds it and restores it after a reconnect", async () => { + const server = await createServer(); + const client = createClient(server); + await client.connect(); + expect(server.eventSubscriptions()).toBe(EventSubscription.All); + + const first = client.subscribeHighVolume(EventSubscription.InputVolumeMeters); + const second = client.subscribeHighVolume(EventSubscription.InputVolumeMeters); + const withMeters = EventSubscription.All | EventSubscription.InputVolumeMeters; + await vi.waitFor(() => expect(server.eventSubscriptions()).toBe(withMeters)); + const reidentifies = () => server.history().filter(({ frame }) => frame.op === OBS_OP.Reidentify).length; + expect(reidentifies()).toBe(1); + + first(); + first(); + expect(reidentifies()).toBe(1); + + server.disconnect(); + await vi.waitFor(() => expect(client.isConnected()).toBe(false)); + await client.connect(); + expect(server.eventSubscriptions(2)).toBe(withMeters); + + second(); + await vi.waitFor(() => expect(server.eventSubscriptions(2)).toBe(EventSubscription.All)); + }); + + it("keeps unapplied output settings flagged when OBS acknowledges a Reidentify", async () => { + const server = await createServer(); + const client = createClient(server); + await client.connect(); + client.markOutputSettingsPending(); + + const release = client.subscribeHighVolume(EventSubscription.InputVolumeMeters); + await server.waitForFrame(({ frame }) => frame.op === OBS_OP.Reidentify); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(client.hasPendingOutputSettings()).toBe(true); + release(); + }); +}); + describe("OBSWebSocketClient events and socket ownership", () => { it("emits both generic and event-specific OBS events", async () => { const server = await createServer(); diff --git a/src/client.ts b/src/client.ts index c611b2a..b27a8b9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -36,6 +36,11 @@ export enum EventSubscription { Ui = 1 << 10, Canvases = 1 << 11, All = (1 << 12) - 1, + // High-volume events are left out of All; subscribe with subscribeHighVolume. + InputVolumeMeters = 1 << 16, + InputActiveStateChanged = 1 << 17, + InputShowStateChanged = 1 << 18, + SceneItemTransformChanged = 1 << 19, } /** @@ -244,6 +249,8 @@ export class OBSWebSocketClient extends EventEmitter { private availableRequests: Set | null = null; private versionInfo: VersionResponse | null = null; private readonly pendingRequests = new Map(); + // How many callers want each high-volume subscription bit. + private readonly highVolumeCounts = new Map(); constructor(url = "ws://localhost:4455", password: string | null = null) { super(); @@ -283,6 +290,45 @@ export class OBSWebSocketClient extends EventEmitter { }; } + /** The event subscriptions this client identifies with. */ + public eventSubscriptions(): number { + let mask: number = EventSubscription.All; + for (const [bit, count] of this.highVolumeCounts) if (count > 0) mask |= bit; + return mask; + } + + /** + * Subscribes to a high-volume event such as InputVolumeMeters until the + * returned function is called. Callers share one subscription per event; + * the last release turns it off. Survives reconnects. + */ + public subscribeHighVolume(event: EventSubscription): () => void { + const before = this.eventSubscriptions(); + this.highVolumeCounts.set(event, (this.highVolumeCounts.get(event) ?? 0) + 1); + this.reidentifyIfChanged(before); + let released = false; + return () => { + if (released) return; + released = true; + const previous = this.eventSubscriptions(); + const count = (this.highVolumeCounts.get(event) ?? 1) - 1; + if (count > 0) this.highVolumeCounts.set(event, count); + else this.highVolumeCounts.delete(event); + this.reidentifyIfChanged(previous); + }; + } + + private reidentifyIfChanged(before: number): void { + const eventSubscriptions = this.eventSubscriptions(); + const socket = this.ws; + if (eventSubscriptions === before || !this.isConnected() || !socket || socket.readyState !== WebSocket.OPEN) return; + try { + socket.send(JSON.stringify({ op: OpCode.Reidentify, d: { eventSubscriptions } })); + } catch (error) { + logger.error(`Unable to change OBS event subscriptions: ${asError(error).message}`); + } + } + public markOutputSettingsPending(): void { this.outputSettingsPending = true; } @@ -547,8 +593,10 @@ export class OBSWebSocketClient extends EventEmitter { logger.error("Ignoring malformed OBS Identified message"); return; } + // OBS answers Reidentify with Identified too; only a new session means + // OBS may have restarted and rebuilt its outputs. + if (!this.identified) this.outputSettingsPending = false; this.identified = true; - this.outputSettingsPending = false; this.emit("identified", socket); break; case OpCode.RequestResponse: { @@ -631,7 +679,7 @@ export class OBSWebSocketClient extends EventEmitter { op: OpCode.Identify, d: { rpcVersion: hello.rpcVersion, - eventSubscriptions: EventSubscription.All, + eventSubscriptions: this.eventSubscriptions(), ...(authentication ? { authentication } : {}), }, }; diff --git a/src/tools/frame-sample.test.ts b/src/tools/frame-sample.test.ts new file mode 100644 index 0000000..812df41 --- /dev/null +++ b/src/tools/frame-sample.test.ts @@ -0,0 +1,43 @@ +/* + * Modified for the independent OBS MCP project in August 2026. + * See NOTICE.md and Git history for authorship and change dates. + * SPDX-License-Identifier: GPL-2.0-only + */ +import { describe, expect, it } from "vitest"; +import { decodePpm, encodePpm, isBlank, isSameFrame } from "./frame-sample.js"; + +const dataUri = (buffer: Buffer) => `data:image/ppm;base64,${buffer.toString("base64")}`; + +describe("frame samples", () => { + it("decodes a binary PPM from a data URI, including header comments", () => { + const pixels = Buffer.from([255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255]); + const ppm = Buffer.concat([Buffer.from("P6\n# made by a test\n2 2\n255\n"), pixels]); + + const sample = decodePpm(dataUri(ppm)); + + expect(sample).toMatchObject({ width: 2, height: 2 }); + expect([...sample.luma]).toEqual([54, 182, 18, 255]); + }); + + it("decodes 16-bit PPMs", () => { + const ppm = Buffer.concat([Buffer.from("P6 1 1 65535\n"), Buffer.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff])]); + + expect([...decodePpm(ppm).luma]).toEqual([255]); + }); + + it("rejects other formats and truncated data", () => { + expect(() => decodePpm(Buffer.from("\x89PNG...."))).toThrow("Not a binary PPM"); + expect(() => decodePpm(Buffer.from("P6\n4 4\n255\n\x00\x00"))).toThrow("Truncated"); + }); + + it("recognizes black frames and unchanged frames", () => { + const black = decodePpm(encodePpm(4, 4, () => [8, 8, 8])); + const page = decodePpm(encodePpm(4, 4, (pixel) => (pixel % 2 ? [240, 240, 240] : [20, 20, 20]))); + const scrolled = decodePpm(encodePpm(4, 4, (pixel) => (pixel % 2 ? [20, 20, 20] : [240, 240, 240]))); + + expect(isBlank(black)).toBe(true); + expect(isBlank(page)).toBe(false); + expect(isSameFrame(page, decodePpm(encodePpm(4, 4, (pixel) => (pixel % 2 ? [240, 240, 240] : [20, 20, 20]))))).toBe(true); + expect(isSameFrame(page, scrolled)).toBe(false); + }); +}); diff --git a/src/tools/frame-sample.ts b/src/tools/frame-sample.ts new file mode 100644 index 0000000..fabb507 --- /dev/null +++ b/src/tools/frame-sample.ts @@ -0,0 +1,81 @@ +/* + * Modified for the independent OBS MCP project in August 2026. + * See NOTICE.md and Git history for authorship and change dates. + * SPDX-License-Identifier: GPL-2.0-only + */ + +/** + * A tiny grayscale copy of a frame. OBS can return screenshots as binary PPM + * (Qt's "ppm" format), which decodes without an image library. + */ +export type FrameSample = { width: number; height: number; luma: Uint8Array }; + +/** Brightest pixel below this means the frame is black. */ +const BLANK_MAX_LUMA = 16; +/** Mean per-pixel change below this means the frame did not change. */ +const STILL_MEAN_DELTA = 1; + +/** Decodes a binary (P6) PPM, as a buffer or a base64 data URI, into luma. */ +export function decodePpm(input: Buffer | string): FrameSample { + const buffer = typeof input === "string" + ? Buffer.from(input.slice(input.indexOf(",") + 1), "base64") + : input; + + // Header: "P6", width, height, maxval, separated by whitespace and comments, + // then exactly one whitespace byte before the pixels. + const fields: number[] = []; + let offset = 0; + const magic = buffer.subarray(0, 2).toString("ascii"); + if (magic !== "P6") throw new Error(`Not a binary PPM (starts with ${JSON.stringify(magic)})`); + offset = 2; + while (fields.length < 3) { + const byte = buffer[offset]; + if (byte === undefined) throw new Error("Truncated PPM header"); + if (byte === 0x23) { // "#": comment to end of line + while (buffer[offset] !== undefined && buffer[offset] !== 0x0a) offset++; + } else if (byte >= 0x30 && byte <= 0x39) { + let end = offset; + while ((buffer[end] ?? 0) >= 0x30 && (buffer[end] ?? 0) <= 0x39) end++; + fields.push(Number(buffer.subarray(offset, end).toString("ascii"))); + offset = end; + continue; + } + offset++; + } + offset++; // the single whitespace byte after maxval + + const [width = 0, height = 0, maxValue = 255] = fields; + const bytesPerSample = maxValue > 255 ? 2 : 1; + const pixels = width * height; + if (buffer.length < offset + pixels * 3 * bytesPerSample) throw new Error("Truncated PPM pixel data"); + + const luma = new Uint8Array(pixels); + const read = bytesPerSample === 1 + ? (index: number) => buffer[offset + index] ?? 0 + : (index: number) => buffer.readUInt16BE(offset + index * 2); + const scale = 255 / maxValue; + for (let pixel = 0; pixel < pixels; pixel++) { + const r = read(pixel * 3), g = read(pixel * 3 + 1), b = read(pixel * 3 + 2); + luma[pixel] = Math.round((0.2126 * r + 0.7152 * g + 0.0722 * b) * scale); + } + return { width, height, luma }; +} + +export function isBlank(sample: FrameSample): boolean { + return sample.luma.every((value) => value < BLANK_MAX_LUMA); +} + +export function isSameFrame(a: FrameSample, b: FrameSample): boolean { + if (a.width !== b.width || a.height !== b.height || a.luma.length === 0) return false; + let total = 0; + for (let index = 0; index < a.luma.length; index++) total += Math.abs((a.luma[index] ?? 0) - (b.luma[index] ?? 0)); + return total / a.luma.length < STILL_MEAN_DELTA; +} + +/** Builds a binary PPM filled with one color, or with a color per pixel. For tests and fakes. */ +export function encodePpm(width: number, height: number, color: (pixel: number) => [number, number, number]): Buffer { + const header = Buffer.from(`P6\n${width} ${height}\n255\n`, "ascii"); + const pixels = Buffer.alloc(width * height * 3); + for (let pixel = 0; pixel < width * height; pixel++) pixels.set(color(pixel), pixel * 3); + return Buffer.concat([header, pixels]); +} diff --git a/src/tools/preflight.ts b/src/tools/preflight.ts index ca675a6..215ecf1 100644 --- a/src/tools/preflight.ts +++ b/src/tools/preflight.ts @@ -157,7 +157,7 @@ async function checkEncoder(client: OBSWebSocketClient): Promise { +export async function recordedTracks(client: OBSWebSocketClient): Promise { if (await profileParameter(client, "Output", "Mode") !== "Advanced") return [1]; const mask = Number(await profileParameter(client, "AdvOut", "RecTracks") ?? "1"); const tracks = [1, 2, 3, 4, 5, 6].filter((track) => (mask & (1 << (track - 1))) !== 0); diff --git a/src/tools/resources.test.ts b/src/tools/resources.test.ts index 68817c8..9aaa87f 100644 --- a/src/tools/resources.test.ts +++ b/src/tools/resources.test.ts @@ -5,6 +5,7 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { startMcpHarness, type McpHarness } from "../../test/support/mcp-harness.js"; +import { TakeMonitor, takeUpdates } from "./take-monitor.js"; const JPEG = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AN//Z"; @@ -47,6 +48,7 @@ describe("resources", () => { expect(resources.map(({ uri }) => uri)).toEqual(expect.arrayContaining([ "obs://status", "obs://scenes", + "obs://take/current", "obs://scene/Demo/items", "obs://scene/Be%20Right%20Back/items", ])); @@ -97,6 +99,28 @@ describe("resources", () => { expect(updated).toHaveLength(2); }); + it("serves the running take and notifies subscribers when it changes", async () => { + const updated: string[] = []; + harness.mcpClient.setNotificationHandler("notifications/resources/updated", async (notification) => { + updated.push(notification.params.uri); + }); + const read = async () => JSON.parse(String( + ((await harness.mcpClient.readResource({ uri: "obs://take/current" })).contents[0] as { text: string }).text, + )) as Record; + expect(await read()).toEqual({ running: false }); + + await harness.obsClient.connect(); + const take = new TakeMonitor(harness.obsClient, { sampleIntervalMs: 60_000, statsIntervalMs: 60_000 }); + await take.start(); + await harness.mcpClient.subscribeResource({ uri: "obs://take/current" }); + take.mark("Intro"); + + await vi.waitFor(() => expect(updated).toContain("obs://take/current")); + expect(await read()).toMatchObject({ running: true, chapters: [{ name: "Intro" }] }); + await take.stop(); + expect(await read()).toEqual({ running: false }); + }); + it("removes its OBS event listeners when the session closes", async () => { expect(harness.obsClient.listenerCount("RecordStateChanged")).toBeGreaterThan(0); const { obsClient } = harness; @@ -105,6 +129,7 @@ describe("resources", () => { closed = true; expect(obsClient.listenerCount("RecordStateChanged")).toBe(0); + expect(takeUpdates.listenerCount("update")).toBe(0); }); }); diff --git a/src/tools/resources.ts b/src/tools/resources.ts index cf4c4d1..b6c59da 100644 --- a/src/tools/resources.ts +++ b/src/tools/resources.ts @@ -6,11 +6,13 @@ import { ResourceTemplate, type McpServer } from "@modelcontextprotocol/server"; import type { OBSWebSocketClient } from "../client.js"; import { screenshotResult } from "./screenshot.js"; +import { activeTake, takeUpdates } from "./take-monitor.js"; type JsonObject = Record; export const STATUS_URI = "obs://status"; export const SCENES_URI = "obs://scenes"; +export const TAKE_URI = "obs://take/current"; export const sceneItemsUri = (sceneName: string) => `obs://scene/${encodeURIComponent(sceneName)}/items`; const SCREENSHOT_WIDTH = 960; @@ -47,7 +49,9 @@ function isObject(value: unknown): value is JsonObject { } function json(uri: string, value: unknown) { - return { contents: [{ uri, mimeType: "application/json", text: JSON.stringify(value, null, 2) }] }; + // Silent audio levels are -Infinity, which JSON cannot carry. + const text = JSON.stringify(value, (_key, item: unknown) => (item === -Infinity ? null : item), 2); + return { contents: [{ uri, mimeType: "application/json", text }] }; } async function sceneNames(client: OBSWebSocketClient): Promise { @@ -85,6 +89,18 @@ export function initialize(server: McpServer, client: OBSWebSocketClient): void }, ); + server.registerResource( + "take", + TAKE_URI, + { + title: "Current take", + description: "While obs-record-clip or a recording session runs: audio levels on recorded tracks, skipped " + + "frames, black or unchanging picture, chapters, and warnings; updates when a warning is raised", + mimeType: "application/json", + }, + async (uri) => json(uri.href, activeTake(client)?.summary() ?? { running: false }), + ); + server.registerResource( "scenes", SCENES_URI, @@ -180,11 +196,14 @@ function forwardEvents(server: McpServer, client: OBSWebSocketClient): void { ]), ]; for (const [event, listener] of listeners) client.on(event, listener); + const onTakeUpdate = () => notify(TAKE_URI); + takeUpdates.on("update", onTakeUpdate); // serveStdio builds a server per session; drop this session's listeners with it. const previousOnClose = lowLevel.onclose; lowLevel.onclose = () => { for (const [event, listener] of listeners) client.off(event, listener); + takeUpdates.off("update", onTakeUpdate); previousOnClose?.(); }; } diff --git a/src/tools/take-monitor.test.ts b/src/tools/take-monitor.test.ts new file mode 100644 index 0000000..fe3f6f2 --- /dev/null +++ b/src/tools/take-monitor.test.ts @@ -0,0 +1,213 @@ +/* + * Modified for the independent OBS MCP project in August 2026. + * See NOTICE.md and Git history for authorship and change dates. + * SPDX-License-Identifier: GPL-2.0-only + */ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EventSubscription, OBSWebSocketClient } from "../client.js"; +import { FakeOBSServer } from "../../test/support/fake-obs-server.js"; +import { healthyObsState, servePreflightState, type FakeObsState } from "../../test/support/fake-obs-state.js"; +import { encodePpm } from "./frame-sample.js"; +import { activeTake, takeLogPath, TakeMonitor, writeTakeLog, type TakeMonitorOptions } from "./take-monitor.js"; + +const SAMPLE_MS = 20; + +let fakeObs: FakeOBSServer; +let client: OBSWebSocketClient; +let state: FakeObsState; +let directory: string; +let stats: { renderSkippedFrames: number; renderTotalFrames: number; outputSkippedFrames: number; outputTotalFrames: number }; +let picture: () => Buffer; +let monitor: TakeMonitor | undefined; + +const page = () => encodePpm(32, 18, (pixel) => (pixel % 3 ? [230, 230, 230] : [30, 30, 30])); +const black = () => encodePpm(32, 18, () => [0, 0, 0]); +let frameCounter = 0; +const moving = () => encodePpm(32, 18, (pixel) => ((pixel + frameCounter++) % 5 ? [230, 230, 230] : [20, 20, 20])); + +async function startMonitor(options: TakeMonitorOptions = {}): Promise { + monitor = new TakeMonitor(client, { + expectSilent: true, + sampleIntervalMs: SAMPLE_MS, + statsIntervalMs: SAMPLE_MS, + stillAfterSeconds: 0.1, + ...options, + }); + await monitor.start(); + return monitor; +} + +function meters(inputs: { inputName: string; peak: number; inputPeak?: number }[]): void { + fakeObs.sendEvent("InputVolumeMeters", { + inputs: inputs.map(({ inputName, peak, inputPeak = peak }) => ({ + inputName, + inputLevelsMul: [[peak / 2, peak, inputPeak], [0, 0, 0]], + })), + }); +} + +beforeEach(async () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + directory = mkdtempSync(join(tmpdir(), "obs-mcp-take-")); + state = healthyObsState(directory); + state.inputs = [ + { inputName: "Chrome", inputKind: "screen_capture", muted: false, tracks: { 1: true } }, + { inputName: "Muted Chrome", inputKind: "screen_capture", muted: true, tracks: { 1: true } }, + { inputName: "Off-track Mic", inputKind: "coreaudio_input_capture", muted: false, tracks: { 1: false, 2: true } }, + ]; + stats = { renderSkippedFrames: 0, renderTotalFrames: 0, outputSkippedFrames: 0, outputTotalFrames: 0 }; + picture = moving; + fakeObs = await FakeOBSServer.start({ supportedImageFormats: ["png", "ppm"] }); + servePreflightState(fakeObs, () => state); + fakeObs.respondWith("GetStats", () => ({ ...stats, availableDiskSpace: 50_000 })); + fakeObs.respondWith("GetSourceScreenshot", ({ imageFormat }) => { + expect(imageFormat).toBe("ppm"); + return { imageData: `data:image/ppm;base64,${picture().toString("base64")}` }; + }); + client = new OBSWebSocketClient(fakeObs.url); + await client.connect(); +}); + +afterEach(async () => { + await monitor?.stop(); + monitor = undefined; + await client.disconnect(); + await fakeObs.close(); + rmSync(directory, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe("TakeMonitor", () => { + it("subscribes to volume meters only while it runs, and is the active take", async () => { + const take = await startMonitor(); + + await vi.waitFor(() => expect(fakeObs.eventSubscriptions()).toBe(EventSubscription.All | EventSubscription.InputVolumeMeters)); + expect(activeTake(client)).toBe(take); + + await take.stop(); + + await vi.waitFor(() => expect(fakeObs.eventSubscriptions()).toBe(EventSubscription.All)); + expect(activeTake(client)).toBeUndefined(); + }); + + it("warns once when a recorded input becomes audible, ignoring muted and off-track inputs", async () => { + const take = await startMonitor(); + + meters([ + { inputName: "Chrome", peak: 0.5 }, + { inputName: "Muted Chrome", peak: 0, inputPeak: 0.8 }, + { inputName: "Off-track Mic", peak: 0.9 }, + ]); + meters([{ inputName: "Chrome", peak: 0.4 }]); + await vi.waitFor(() => expect(take.summary().warnings).toHaveLength(1)); + meters([{ inputName: "Chrome", peak: 0 }]); + await vi.waitFor(() => expect(take.summary().events.map(({ type }) => type)).toContain("audio-end")); + + const summary = await take.stop(); + expect(summary.warnings[0]).toMatchObject({ kind: "audio", message: expect.stringContaining("Chrome is audible in the recording at -6 dB") }); + expect(summary.audio).toEqual([ + { inputName: "Chrome", peakDb: -6, inputPeakDb: -6, recorded: true }, + { inputName: "Muted Chrome", peakDb: -Infinity, inputPeakDb: -1.9, recorded: true }, + { inputName: "Off-track Mic", peakDb: -0.9, inputPeakDb: -0.9, recorded: false }, + ].sort((a, b) => b.peakDb - a.peakDb)); + }); + + it("records audible inputs without warning when silence is not expected", async () => { + const take = await startMonitor({ expectSilent: false }); + + meters([{ inputName: "Chrome", peak: 0.5 }]); + await vi.waitFor(() => expect(take.summary().events.map(({ type }) => type)).toContain("audio-start")); + + expect((await take.stop()).warnings).toEqual([]); + }); + + it("follows track changes made during the take", async () => { + const take = await startMonitor(); + + fakeObs.sendEvent("InputAudioTracksChanged", { inputName: "Chrome", inputAudioTracks: { 1: false } }); + await new Promise((resolve) => setTimeout(resolve, 20)); + meters([{ inputName: "Chrome", peak: 0.5 }]); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect((await take.stop()).warnings).toEqual([]); + }); + + it("warns about skipped frames and totals them", async () => { + const take = await startMonitor(); + + stats = { renderSkippedFrames: 3, renderTotalFrames: 120, outputSkippedFrames: 0, outputTotalFrames: 118 }; + await vi.waitFor(() => expect(take.summary().warnings.map(({ kind }) => kind)).toContain("frames")); + + const summary = await take.stop(); + expect(summary.frames).toMatchObject({ renderSkipped: 3, renderTotal: 120, outputSkipped: 0 }); + expect(summary.warnings.filter(({ kind }) => kind === "frames")).toHaveLength(1); + }); + + it("reports black stretches as warnings and unchanged stretches as stills", async () => { + const take = await startMonitor(); + await vi.waitFor(() => expect(take.summary().framesSampled).toBeGreaterThan(2)); + + picture = page; + await new Promise((resolve) => setTimeout(resolve, 200)); + picture = black; + await vi.waitFor(() => expect(take.summary().warnings.map(({ kind }) => kind)).toContain("blank")); + picture = moving; + await vi.waitFor(() => expect(take.summary().blanks.length).toBe(1)); + + const summary = await take.stop(); + expect(summary.pictureChecks).toBe("on"); + expect(summary.warnings.filter(({ kind }) => kind === "blank")).toHaveLength(1); + expect(summary.stills.length).toBeGreaterThanOrEqual(1); + const still = summary.stills[0]!; + expect(still.endSeconds - still.startSeconds).toBeGreaterThanOrEqual(0.1); + }); + + it("skips picture checks when OBS cannot return PPM", async () => { + await client.disconnect(); + await fakeObs.close(); + fakeObs = await FakeOBSServer.start(); + servePreflightState(fakeObs, () => state); + client = new OBSWebSocketClient(fakeObs.url); + await client.connect(); + + const take = await startMonitor(); + await new Promise((resolve) => setTimeout(resolve, 60)); + + const summary = await take.stop(); + expect(summary.pictureChecks).toBe("off"); + expect(fakeObs.history().some(({ frame }) => frame.d.requestType === "GetSourceScreenshot")).toBe(false); + }); + + it("warns when the recording stops before it was asked to, but not after expectStop", async () => { + const take = await startMonitor(); + fakeObs.sendEvent("RecordStateChanged", { outputActive: false, outputState: "OBS_WEBSOCKET_OUTPUT_STOPPED" }); + await vi.waitFor(() => expect(take.summary().warnings.map(({ kind }) => kind)).toContain("output")); + await take.stop(); + + const second = await startMonitor(); + second.expectStop(); + fakeObs.sendEvent("RecordStateChanged", { outputActive: false, outputState: "OBS_WEBSOCKET_OUTPUT_STOPPED" }); + await vi.waitFor(() => expect(second.summary().events.map(({ type }) => type)).toContain("RecordStateChanged")); + expect(second.summary().warnings).toEqual([]); + }); + + it("writes a take log next to the recording with chapters and without -Infinity", async () => { + const take = await startMonitor(); + take.mark("Intro"); + meters([{ inputName: "Muted Chrome", peak: 0, inputPeak: 0.8 }]); + await new Promise((resolve) => setTimeout(resolve, 20)); + const summary = await take.stop(); + + const recording = join(directory, "2026-09-23 10-00-00.mp4"); + const path = await writeTakeLog(recording, summary); + + expect(path).toBe(join(directory, "2026-09-23 10-00-00.take.json")); + const log = JSON.parse(readFileSync(path!, "utf8")) as Record; + expect(log).toMatchObject({ recording, running: false, chapters: [{ name: "Intro" }] }); + expect(log.audio).toEqual([expect.objectContaining({ inputName: "Muted Chrome", peakDb: null })]); + expect(takeLogPath("/a.b/clip")).toBe("/a.b/clip.take.json"); + }); +}); diff --git a/src/tools/take-monitor.ts b/src/tools/take-monitor.ts new file mode 100644 index 0000000..f4bdb8e --- /dev/null +++ b/src/tools/take-monitor.ts @@ -0,0 +1,447 @@ +/* + * Modified for the independent OBS MCP project in August 2026. + * See NOTICE.md and Git history for authorship and change dates. + * SPDX-License-Identifier: GPL-2.0-only + */ +import { EventEmitter } from "node:events"; +import { writeFile } from "node:fs/promises"; +import { EventSubscription, type OBSWebSocketClient } from "../client.js"; +import { logger } from "../logger.js"; +import { decodePpm, isBlank, isSameFrame, type FrameSample } from "./frame-sample.js"; +import { recordedTracks } from "./preflight.js"; + +type JsonObject = Record; + +export type TakeWarningKind = "audio" | "blank" | "frames" | "output" | "connection"; +export type TakeWarning = { atSeconds: number; kind: TakeWarningKind; message: string }; +export type TakeEvent = { atSeconds: number; type: string; detail?: string }; +export type TakeSpan = { startSeconds: number; endSeconds: number }; +export type InputLevel = { + inputName: string; + /** Loudest level after volume and mute, in dBFS. */ + peakDb: number; + /** Loudest level before volume and mute, in dBFS. */ + inputPeakDb: number; + /** Whether the input feeds a track the recording writes. */ + recorded: boolean; +}; + +export type TakeSummary = { + running: boolean; + startedAt: string; + durationSeconds: number; + expectSilent: boolean; + silenceThresholdDb: number; + warnings: TakeWarning[]; + events: TakeEvent[]; + chapters: { atSeconds: number; name: string }[]; + audio: InputLevel[]; + frames: { renderSkipped: number; renderTotal: number; outputSkipped: number; outputTotal: number }; + /** Stretches where the program output did not change, for trimming dead time. */ + stills: TakeSpan[]; + /** Stretches where the program output was black. */ + blanks: TakeSpan[]; + framesSampled: number; + pictureChecks: "on" | "off"; +}; + +export type TakeMonitorOptions = { + expectSilent?: boolean; + silenceThresholdDb?: number; + sampleIntervalMs?: number; + statsIntervalMs?: number; + /** Shortest unchanged stretch reported in stills. */ + stillAfterSeconds?: number; +}; + +const SAMPLE_WIDTH = 32; +const MAX_EVENTS = 500; +const MAX_WARNINGS_PER_KIND = 20; +/** Two black samples in a row, so a fade or cut through black is not flagged. */ +const BLANK_SAMPLES_TO_WARN = 2; + +/** Events worth a line in the take log. */ +const LOGGED_EVENTS: Record string | undefined> = { + RecordStateChanged: (data) => String(data.outputState ?? ""), + RecordFileChanged: (data) => String(data.newOutputPath ?? ""), + StreamStateChanged: (data) => String(data.outputState ?? ""), + CurrentProgramSceneChanged: (data) => String(data.sceneName ?? ""), + SceneItemEnableStateChanged: (data) => `${String(data.sceneName)} #${String(data.sceneItemId)} ${data.sceneItemEnabled ? "shown" : "hidden"}`, + InputMuteStateChanged: (data) => `${String(data.inputName)} ${data.inputMuted ? "muted" : "unmuted"}`, + InputAudioTracksChanged: (data) => String(data.inputName ?? ""), +}; + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function toDb(multiplier: number): number { + return multiplier > 0 ? Math.round(20 * Math.log10(multiplier) * 10) / 10 : -Infinity; +} + +function round(seconds: number): number { + return Math.round(seconds * 100) / 100; +} + +/** Tells the obs://take/current resource that the running take changed. */ +export const takeUpdates = new EventEmitter().setMaxListeners(0); // one listener per MCP session +const activeTakes = new WeakMap(); + +export function activeTake(client: OBSWebSocketClient): TakeMonitor | undefined { + return activeTakes.get(client); +} + +/** + * Watches OBS while a take records: audio levels on recorded tracks from + * InputVolumeMeters, skipped frames from GetStats, and a 32px sample of the + * program output for black or unchanging picture. Start it once the + * recording is confirmed, so its clock matches the file's. + */ +export class TakeMonitor { + private readonly client: OBSWebSocketClient; + private readonly options: Required; + private startedAtMs = 0; + private stoppedAtMs: number | null = null; + private stopExpected = false; + private readonly warnings: TakeWarning[] = []; + private readonly events: TakeEvent[] = []; + private readonly chapters: { atSeconds: number; name: string }[] = []; + private readonly levels = new Map(); + private readonly audibleNow = new Set(); + private readonly inputTracks = new Map(); + private recordTracks: number[] = [1]; + private readonly frames = { renderSkipped: 0, renderTotal: 0, outputSkipped: 0, outputTotal: 0 }; + private lastStats: JsonObject | null = null; + private lastFrameWarningMs = new Map(); + private readonly stills: TakeSpan[] = []; + private readonly blanks: TakeSpan[] = []; + private stillSince: number | null = null; + private blankSince: number | null = null; + private blankRun = 0; + private previousSample: FrameSample | null = null; + private framesSampled = 0; + private pictureChecks: "on" | "off" = "off"; + private programScene: string | null = null; + private readonly timers: NodeJS.Timeout[] = []; + private readonly busy = new Set(); + private releaseMeters: (() => void) | null = null; + private readonly listeners: [string, (...args: unknown[]) => void][] = []; + + constructor(client: OBSWebSocketClient, options: TakeMonitorOptions = {}) { + this.client = client; + this.options = { + expectSilent: options.expectSilent ?? false, + silenceThresholdDb: options.silenceThresholdDb ?? -60, + sampleIntervalMs: options.sampleIntervalMs ?? 1_000, + statsIntervalMs: options.statsIntervalMs ?? 2_000, + stillAfterSeconds: options.stillAfterSeconds ?? 3, + }; + } + + /** Seconds since the monitor started. */ + now(): number { + return round(((this.stoppedAtMs ?? Date.now()) - this.startedAtMs) / 1000); + } + + async start(): Promise { + this.startedAtMs = Date.now(); + activeTakes.set(this.client, this); + + this.listen("InputVolumeMeters", (data) => this.onMeters(data)); + this.listen("InputAudioTracksChanged", (data) => { + if (typeof data.inputName === "string" && isObject(data.inputAudioTracks)) { + this.inputTracks.set(data.inputName, data.inputAudioTracks); + } + }); + this.listen("CurrentProgramSceneChanged", (data) => { + if (typeof data.sceneName === "string") this.programScene = data.sceneName; + this.previousSample = null; + this.closeStill(this.now()); + }); + this.listen("RecordStateChanged", (data) => { + if (data.outputActive === false && !this.stopExpected && data.outputState === "OBS_WEBSOCKET_OUTPUT_STOPPED") { + this.warn("output", "The recording stopped before it was asked to"); + } + }); + for (const [event, describe] of Object.entries(LOGGED_EVENTS)) { + this.listen(event, (data) => this.log(event, describe(data))); + } + const onDisconnect = () => this.warn("connection", "Lost the connection to OBS; checks paused until it reconnects"); + this.client.on("disconnected", onDisconnect); + this.listeners.push(["disconnected", onDisconnect]); + + await Promise.all([this.loadTracks(), this.loadProgramScene(), this.sampleStats().catch(() => undefined)]); + this.releaseMeters = this.client.subscribeHighVolume(EventSubscription.InputVolumeMeters); + + const formats = this.client.getConnectionStatus().versionInfo?.supportedImageFormats; + this.pictureChecks = Array.isArray(formats) && formats.includes("ppm") ? "on" : "off"; + if (this.pictureChecks === "on") this.every(this.options.sampleIntervalMs, "picture", () => this.samplePicture()); + else this.log("picture-checks-off", "OBS cannot return PPM screenshots"); + this.every(this.options.statsIntervalMs, "stats", () => this.sampleStats()); + this.changed(); + } + + mark(name: string): void { + this.chapters.push({ atSeconds: this.now(), name }); + this.changed(); + } + + /** Call before stopping the recording, so the stop is not reported as a problem. */ + expectStop(): void { + this.stopExpected = true; + } + + async stop(): Promise { + if (this.stoppedAtMs === null) { + this.stopExpected = true; + for (const timer of this.timers) clearInterval(timer); + this.timers.length = 0; + this.releaseMeters?.(); + this.releaseMeters = null; + for (const [event, listener] of this.listeners) this.client.off(event, listener); + this.listeners.length = 0; + await this.sampleStats().catch(() => undefined); + this.stoppedAtMs = Date.now(); + const end = this.now(); + this.closeStill(end); + this.closeBlank(end); + if (activeTakes.get(this.client) === this) activeTakes.delete(this.client); + this.changed(); + } + return this.summary(); + } + + summary(): TakeSummary { + return { + running: this.stoppedAtMs === null, + startedAt: new Date(this.startedAtMs).toISOString(), + durationSeconds: this.now(), + expectSilent: this.options.expectSilent, + silenceThresholdDb: this.options.silenceThresholdDb, + warnings: [...this.warnings], + events: [...this.events], + chapters: [...this.chapters], + audio: [...this.levels.values()].sort((a, b) => b.peakDb - a.peakDb), + frames: { ...this.frames }, + stills: [...this.stills, ...this.openSpan(this.stillSince)], + blanks: [...this.blanks, ...this.openSpan(this.blankSince)], + framesSampled: this.framesSampled, + pictureChecks: this.pictureChecks, + }; + } + + private openSpan(since: number | null): TakeSpan[] { + return since === null || this.stoppedAtMs !== null ? [] : [{ startSeconds: since, endSeconds: this.now() }]; + } + + private listen(event: string, handler: (data: JsonObject) => void): void { + const listener = (data: unknown) => handler(isObject(data) ? data : {}); + this.client.on(event, listener); + this.listeners.push([event, listener as (...args: unknown[]) => void]); + } + + private every(intervalMs: number, name: string, task: () => Promise): void { + const timer = setInterval(() => { + if (this.busy.has(name)) return; + this.busy.add(name); + task().catch((error: unknown) => { + logger.debug(`Take monitor ${name} check failed: ${error instanceof Error ? error.message : String(error)}`); + }).finally(() => this.busy.delete(name)); + }, intervalMs); + timer.unref(); + this.timers.push(timer); + } + + private changed(): void { + takeUpdates.emit("update"); + } + + private log(type: string, detail?: string): void { + if (this.events.length >= MAX_EVENTS) return; + this.events.push({ atSeconds: this.now(), type, ...(detail ? { detail } : {}) }); + } + + private warn(kind: TakeWarningKind, message: string): void { + if (this.warnings.filter((warning) => warning.kind === kind).length >= MAX_WARNINGS_PER_KIND) return; + this.warnings.push({ atSeconds: this.now(), kind, message }); + this.changed(); + } + + private async loadTracks(): Promise { + try { + this.recordTracks = await recordedTracks(this.client); + const { inputs } = await this.client.sendRequest("GetInputList") as { inputs?: unknown[] }; + const names = (inputs ?? []).filter(isObject).map(({ inputName }) => inputName).filter((name): name is string => typeof name === "string"); + const results = await this.client.sendBatch( + names.map((inputName) => ({ requestType: "GetInputAudioTracks", requestData: { inputName } })), + ); + names.forEach((name, index) => { + const result = results[index]; + if (result?.ok && isObject(result.responseData) && isObject(result.responseData.inputAudioTracks)) { + this.inputTracks.set(name, result.responseData.inputAudioTracks); + } + }); + } catch (error) { + // Unknown tracks count as recorded, which errs toward warning. + this.log("tracks-unknown", error instanceof Error ? error.message : String(error)); + } + } + + private async loadProgramScene(): Promise { + const program = await this.client.sendRequest("GetCurrentProgramScene").catch(() => ({})) as JsonObject; + const name = program.currentProgramSceneName ?? program.sceneName; + this.programScene = typeof name === "string" ? name : null; + } + + private isRecorded(inputName: string): boolean { + const tracks = this.inputTracks.get(inputName); + return !tracks || this.recordTracks.some((track) => tracks[String(track)] === true); + } + + private onMeters(data: JsonObject): void { + if (this.stoppedAtMs !== null || !Array.isArray(data.inputs)) return; + for (const input of data.inputs) { + if (!isObject(input) || typeof input.inputName !== "string" || !Array.isArray(input.inputLevelsMul)) continue; + // Each channel is [magnitude, peak, input peak]; peak is after volume and mute. + let peak = 0; + let inputPeak = 0; + for (const channel of input.inputLevelsMul) { + if (!Array.isArray(channel)) continue; + peak = Math.max(peak, Number(channel[1]) || 0); + inputPeak = Math.max(inputPeak, Number(channel[2]) || 0); + } + const name = input.inputName; + const recorded = this.isRecorded(name); + const peakDb = toDb(peak); + const level = this.levels.get(name); + this.levels.set(name, { + inputName: name, + peakDb: Math.max(level?.peakDb ?? -Infinity, peakDb), + inputPeakDb: Math.max(level?.inputPeakDb ?? -Infinity, toDb(inputPeak)), + recorded, + }); + + const audible = recorded && peakDb > this.options.silenceThresholdDb; + if (audible && !this.audibleNow.has(name)) { + this.audibleNow.add(name); + this.log("audio-start", `${name} at ${peakDb} dB`); + if (this.options.expectSilent) { + this.warn("audio", `${name} is audible in the recording at ${peakDb} dB (expected silence). ` + + "Mute it or clear its recorded tracks"); + } + } else if (!audible && this.audibleNow.has(name)) { + this.audibleNow.delete(name); + this.log("audio-end", name); + } + } + } + + private async sampleStats(): Promise { + const stats = await this.client.sendRequest("GetStats") as JsonObject; + const previous = this.lastStats; + this.lastStats = stats; + if (!previous) return; + + const delta = (field: string) => Math.max(0, Number(stats[field] ?? 0) - Number(previous[field] ?? 0)); + const renderSkipped = delta("renderSkippedFrames"); + const outputSkipped = delta("outputSkippedFrames"); + this.frames.renderSkipped += renderSkipped; + this.frames.renderTotal += delta("renderTotalFrames"); + this.frames.outputSkipped += outputSkipped; + this.frames.outputTotal += delta("outputTotalFrames"); + + const nowMs = Date.now(); + const report = (key: string, skipped: number, message: string) => { + if (skipped === 0 || nowMs - (this.lastFrameWarningMs.get(key) ?? -Infinity) < 10_000) return; + this.lastFrameWarningMs.set(key, nowMs); + this.warn("frames", message); + }; + report("render", renderSkipped, `OBS missed ${renderSkipped} frame(s) while rendering; the GPU or CPU is overloaded`); + report("output", outputSkipped, `The encoder skipped ${outputSkipped} frame(s); lower the encoder preset or resolution`); + } + + private async samplePicture(): Promise { + if (!this.programScene) return; + const response = await this.client.sendRequest("GetSourceScreenshot", { + sourceName: this.programScene, + imageFormat: "ppm", + imageWidth: SAMPLE_WIDTH, + }) as JsonObject; + if (typeof response.imageData !== "string" || this.stoppedAtMs !== null) return; + const sample = decodePpm(response.imageData); + const at = this.now(); + this.framesSampled += 1; + + if (isBlank(sample)) { + this.blankRun += 1; + if (this.blankRun === BLANK_SAMPLES_TO_WARN) { + this.blankSince = round(at - this.options.sampleIntervalMs / 1000); + this.warn("blank", `The program output (${this.programScene}) is black. A capture may have lost its window ` + + "or screen recording permission"); + } + } else { + this.blankRun = 0; + this.closeBlank(at); + } + + if (this.previousSample && isSameFrame(this.previousSample, sample)) { + this.stillSince ??= round(at - this.options.sampleIntervalMs / 1000); + } else { + this.closeStill(at); + } + this.previousSample = sample; + } + + private closeStill(at: number): void { + if (this.stillSince !== null && at - this.stillSince >= this.options.stillAfterSeconds) { + this.stills.push({ startSeconds: this.stillSince, endSeconds: at }); + } + this.stillSince = null; + } + + private closeBlank(at: number): void { + if (this.blankSince !== null) this.blanks.push({ startSeconds: this.blankSince, endSeconds: at }); + this.blankSince = null; + } +} + +/** "take.mp4" -> "take.take.json", next to the recording. */ +export function takeLogPath(recordingPath: string): string { + const dot = recordingPath.lastIndexOf("."); + const slash = Math.max(recordingPath.lastIndexOf("/"), recordingPath.lastIndexOf("\\")); + return `${dot > slash ? recordingPath.slice(0, dot) : recordingPath}.take.json`; +} + +/** Writes the take log next to a local recording; returns its path, or null if it could not. */ +export async function writeTakeLog(recordingPath: string, summary: TakeSummary & JsonObject): Promise { + const path = takeLogPath(recordingPath); + try { + await writeFile(path, `${JSON.stringify({ recording: recordingPath, ...summary }, (_key, value: unknown) => ( + value === -Infinity ? null : value + ), 2)}\n`); + return path; + } catch (error) { + logger.error(`Could not write the take log ${path}: ${error instanceof Error ? error.message : String(error)}`); + return null; + } +} + +/** One-line summary of a finished take for tool results. */ +export function describeTake(summary: TakeSummary): string[] { + const lines: string[] = []; + const loudest = summary.audio.filter(({ recorded }) => recorded)[0]; + lines.push(loudest && loudest.peakDb > summary.silenceThresholdDb + ? `Monitor: loudest recorded input ${loudest.inputName} at ${loudest.peakDb} dB` + : "Monitor: no recorded input rose above " + + `${summary.silenceThresholdDb} dB`); + if (summary.frames.renderSkipped + summary.frames.outputSkipped > 0) { + lines.push(`Monitor: ${summary.frames.renderSkipped} frame(s) missed rendering, ${summary.frames.outputSkipped} skipped encoding`); + } + if (summary.pictureChecks === "on") { + const still = summary.stills.reduce((total, span) => total + span.endSeconds - span.startSeconds, 0); + lines.push(`Monitor: ${summary.framesSampled} picture sample(s), ${summary.blanks.length} black stretch(es), ` + + `${round(still)}s unchanged`); + } + for (const warning of summary.warnings) lines.push(`Warning at ${warning.atSeconds}s: ${warning.message}`); + return lines; +} diff --git a/src/tools/workflows.test.ts b/src/tools/workflows.test.ts index 1625e80..35f8e43 100644 --- a/src/tools/workflows.test.ts +++ b/src/tools/workflows.test.ts @@ -4,11 +4,11 @@ * SPDX-License-Identifier: GPL-2.0-only */ import { execFileSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { FakeOBSRequestError } from "../../test/support/fake-obs-server.js"; +import { FakeOBSRequestError, OBS_OP } from "../../test/support/fake-obs-server.js"; import { healthyObsState, servePreflightState, type FakeObsState } from "../../test/support/fake-obs-state.js"; import { resultText, startMcpHarness, type McpHarness } from "../../test/support/mcp-harness.js"; @@ -163,6 +163,34 @@ describe("obs-record-clip", () => { expect(resultText(result)).toContain("shorter than the requested 4s"); }, 10_000); + it("flags an input that becomes audible during the take, and writes a take log", async () => { + await startHarness(); + // Muted at the start, so preflight passes; unmuted by someone mid-take. + state.inputs[0] = { inputName: "Chrome", inputKind: "screen_capture", muted: true, tracks: { 1: true } }; + + const pending = harness.call("obs-record-clip", { + durationSeconds: 0.5, + expectSilent: true, + chapters: [{ atSeconds: 0, name: "Start" }], + }); + await harness.fakeObs.waitForFrame(({ frame }) => frame.op === OBS_OP.Reidentify, { timeoutMs: 2_000 }); + harness.fakeObs.sendEvent("InputVolumeMeters", { + inputs: [{ inputName: "Chrome", inputLevelsMul: [[0.2, 0.5, 0.5]] }], + }); + const result = await pending; + + expect(result.isError).toBe(true); + expect(resultText(result)).toMatch(/Problem: At [\d.]+s: Chrome is audible in the recording at -6 dB/); + const takeLog = join(recordDirectory, "clip.take.json"); + expect(resultText(result)).toContain(`Take log: ${takeLog}`); + expect(existsSync(takeLog)).toBe(true); + expect(JSON.parse(readFileSync(takeLog, "utf8"))).toMatchObject({ + recording: clipPath, + chapters: [{ name: "Start" }], + warnings: [{ kind: "audio" }], + }); + }); + it("rejects clips longer than a client timeout allows", async () => { await startHarness(); diff --git a/src/tools/workflows.ts b/src/tools/workflows.ts index 09c6e1a..84c9478 100644 --- a/src/tools/workflows.ts +++ b/src/tools/workflows.ts @@ -11,6 +11,7 @@ import type { OBSWebSocketClient } from "../client.js"; import { appendInputScreenshot, inputStateResult, readInputState } from "./after-change.js"; import { RECORD_OUTPUT, startOutputAndConfirm } from "./output-start.js"; import { runPreflight } from "./preflight.js"; +import { describeTake, TakeMonitor, writeTakeLog, type TakeSummary } from "./take-monitor.js"; type JsonObject = Record; @@ -34,6 +35,9 @@ function errorResult(text: string): CallToolResult { return { content: [{ type: "text", text }], isError: true }; } +/** JSON has no -Infinity; silence reports as null. */ +const finite = (value: number): number | null => (Number.isFinite(value) ? value : null); + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export type ClipCheck = { @@ -95,12 +99,16 @@ async function recordClip( if (started.isError) return started; const startedAt = Date.now(); + const monitor = new TakeMonitor(client, { expectSilent: args.expectSilent }); const chapterNotes: string[] = []; let outputPath: string | undefined; + let take: TakeSummary; try { + await monitor.start(); const chapters = [...args.chapters].sort((a, b) => a.atSeconds - b.atSeconds); for (const chapter of chapters) { await sleep(Math.max(0, startedAt + chapter.atSeconds * 1000 - Date.now())); + monitor.mark(chapter.name); try { await client.sendRequest("CreateRecordChapter", { chapterName: chapter.name }); } catch (error) { @@ -110,8 +118,10 @@ async function recordClip( } await sleep(Math.max(0, startedAt + args.durationSeconds * 1000 - Date.now())); } finally { + monitor.expectStop(); const stopped: unknown = await client.sendRequest("StopRecord").catch(() => undefined); if (isObject(stopped) && typeof stopped.outputPath === "string") outputPath = stopped.outputPath; + take = await monitor.stop(); } if (!outputPath) { @@ -123,7 +133,11 @@ async function recordClip( ? await inspectRecording(outputPath, args.expectSilent) : { skipped: "OBS is remote, so the file was not inspected" }; + const takeLog = local ? await writeTakeLog(outputPath, take) : null; const problems: string[] = []; + for (const warning of take.warnings.filter(({ kind }) => kind === "blank" || kind === "output" || (kind === "audio" && args.expectSilent))) { + problems.push(`At ${warning.atSeconds}s: ${warning.message}`); + } if (inspection.durationSeconds !== undefined && inspection.durationSeconds < args.durationSeconds - 2) { problems.push(`The file is ${inspection.durationSeconds.toFixed(1)}s, shorter than the requested ${args.durationSeconds}s`); } @@ -137,6 +151,8 @@ async function recordClip( ? `Not verified: ${inspection.skipped}` : `Verified: ${inspection.durationSeconds?.toFixed(1)}s, ${inspection.audioStreams} audio stream(s)` + (inspection.maxVolumeDb !== undefined ? `, peak ${inspection.maxVolumeDb} dB` : ""), + ...describeTake(take).filter((line) => !line.startsWith("Warning")), + ...(takeLog ? [`Take log: ${takeLog}`] : []), ...chapterNotes, ...problems.map((problem) => `Problem: ${problem}`), ]; @@ -148,6 +164,8 @@ async function recordClip( ...(inspection.maxVolumeDb === -Infinity ? { maxVolumeDb: null } : {}), chapterNotes, problems, + takeLog, + take: { ...take, audio: take.audio.map((level) => ({ ...level, peakDb: finite(level.peakDb), inputPeakDb: finite(level.inputPeakDb) })) }, }, ...(problems.length > 0 ? { isError: true } : {}), }; diff --git a/test/support/fake-obs-server.ts b/test/support/fake-obs-server.ts index f50bb33..e283eff 100644 --- a/test/support/fake-obs-server.ts +++ b/test/support/fake-obs-server.ts @@ -12,6 +12,7 @@ export const OBS_OP = { Hello: 0, Identify: 1, Identified: 2, + Reidentify: 3, Event: 5, Request: 6, RequestResponse: 7, @@ -53,6 +54,7 @@ export type FakeOBSOptions = { obsStudioVersion?: string; obsWebSocketVersion?: string; platform?: string; + supportedImageFormats?: readonly string[]; }; type ResponseStatus = { @@ -122,6 +124,7 @@ export class FakeOBSServer { private readonly events = new EventEmitter(); private readonly frames: InternalRecordedFrame[] = []; private readonly sockets = new Map(); + private readonly subscriptions = new Map(); private readonly scripts = new Map(); private readonly responders = new Map(); private readonly timers = new Set(); @@ -130,6 +133,7 @@ export class FakeOBSServer { private readonly obsStudioVersion: string; private readonly obsWebSocketVersion: string; private readonly platform: string; + private readonly supportedImageFormats: readonly string[]; private nextConnectionId = 1; private nextCursor = 1; private closed = false; @@ -141,6 +145,7 @@ export class FakeOBSServer { this.obsStudioVersion = options.obsStudioVersion ?? "32.2.2"; this.obsWebSocketVersion = options.obsWebSocketVersion ?? "5.7.0"; this.platform = options.platform ?? "macos"; + this.supportedImageFormats = options.supportedImageFormats ?? ["png", "jpeg", "webp"]; this.availableRequests = Array.from(new Set(["GetVersion", "Sleep", ...(options.availableRequests ?? [])])); const address = server.address() as AddressInfo; @@ -162,6 +167,11 @@ export class FakeOBSServer { return this.sockets.size; } + /** The event subscriptions a connection last identified with. */ + eventSubscriptions(connectionId = this.nextConnectionId - 1): number | undefined { + return this.subscriptions.get(connectionId); + } + cursor(): number { return this.nextCursor - 1; } @@ -333,7 +343,17 @@ export class FakeOBSServer { private handleFrame(record: InternalRecordedFrame): void { const { frame, socket } = record; + if (frame.op === OBS_OP.Reidentify) { + if (typeof frame.d.eventSubscriptions === "number") { + this.subscriptions.set(record.connectionId, frame.d.eventSubscriptions); + } + socket.send(JSON.stringify({ op: OBS_OP.Identified, d: { negotiatedRpcVersion: 1 } })); + return; + } if (frame.op === OBS_OP.Identify) { + if (typeof frame.d.eventSubscriptions === "number") { + this.subscriptions.set(record.connectionId, frame.d.eventSubscriptions); + } if (this.password) { const expected = expectedAuthentication(this.password, this.salt, this.challenge); if (frame.d.authentication !== expected) { @@ -512,7 +532,7 @@ export class FakeOBSServer { obsWebSocketVersion: this.obsWebSocketVersion, rpcVersion: 1, availableRequests: this.availableRequests, - supportedImageFormats: ["png", "jpeg", "webp"], + supportedImageFormats: this.supportedImageFormats, platform: this.platform, platformDescription: "Fake OBS Studio", };