From 5a3c7b3c45aefed38fbfccba5acb5e39f8bce93f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:08:53 -0800 Subject: [PATCH 1/9] chore(quest): claim quest/m1/av-clock.md Quest-Claim: d2c7ec3fb5811503 From c9aef75361f6a6ed695fc33fcf0b3b0b5b2f1d1b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:20:34 -0800 Subject: [PATCH 2/9] feat(watch): the audio playhead drives Sync while audio plays Co-Authored-By: Claude Opus 5.5 --- js/msf/src/catalog.ts | 3 +- js/watch/src/audio/buffer.test.ts | 56 ++- js/watch/src/audio/buffer.ts | 32 ++ js/watch/src/audio/decoder.test.ts | 33 +- js/watch/src/audio/decoder.ts | 29 +- js/watch/src/audio/render-worklet.test.ts | 32 +- js/watch/src/audio/render-worklet.ts | 2 + js/watch/src/audio/render.ts | 4 + js/watch/src/audio/replay.ts | 5 +- js/watch/src/index.ts | 2 +- js/watch/src/msf.ts | 7 +- js/watch/src/retention.test.ts | 10 +- js/watch/src/sync.test.ts | 159 +++--- js/watch/src/sync.ts | 580 +++++++++++++++------- js/watch/src/text/renderer.ts | 14 +- js/watch/src/video/decoder.test.ts | 51 +- js/watch/src/video/decoder.ts | 28 +- 17 files changed, 719 insertions(+), 328 deletions(-) diff --git a/js/msf/src/catalog.ts b/js/msf/src/catalog.ts index bb9eb2ff6cb..37be0bb6f0e 100644 --- a/js/msf/src/catalog.ts +++ b/js/msf/src/catalog.ts @@ -56,7 +56,8 @@ const trackShape = { // Non-standard: how far this rendition trails the broadcast's earliest, in milliseconds. // Mirrors hang `delay`. A player holds `delay + jitter` and does not subtract across renditions. - delay: z.optional(z.number()), + // A rendition cannot trail the earliest by a negative amount, so that is malformed. + delay: z.optional(z.number().check(z.nonnegative())), }; /** Zod schema describing a single track entry in an MSF catalog. */ diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index aacabd46e21..e1d339eb678 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -5,6 +5,9 @@ import type { Init, Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; const HOUR = Milli(60 * 60 * 1000); + +// A context whose output has not started, so no state report maps to a rolling playhead. +const silent = { getOutputTimestamp: () => ({ contextTime: 0, performanceTime: 0 }) } as unknown as AudioContext; const ms = (value: number) => Micro.fromMilli(Milli(value)); // A buffered buffer over a real port, with the worklet's end of it left to the test. @@ -16,6 +19,7 @@ function open(): { } { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + context: silent, channels: 1, rate: 48_000, delay: Milli(100), @@ -29,7 +33,9 @@ function open(): { const msg: State = { type: "state", timestamp: (timestamp ?? 0) as Micro, + time: 0, stalled, + playing: true, anchored: timestamp !== undefined, underruns: 0, epoch, @@ -52,6 +58,7 @@ describe("AudioBuffer", () => { it("transfers PCM ownership and orders control messages after the data they change", async () => { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + context: silent, channels: 2, rate: 48_000, delay: Milli(100), @@ -419,20 +426,21 @@ describe("Backpressure", () => { }); // A worklet whose port records what the buffer posts and delivers state reports synchronously. -function fake(delay: Milli, buffered = false) { +function fake(delay: Milli, buffered = false, context = silent) { const sent: Message[] = []; const port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => sent.push(msg), start() {}, }); const buffer = new AudioBuffer({ port } as unknown as AudioWorkletNode, { + context, channels: 1, rate: 1000, delay, buffered, }); - const state = (timestamp: Micro, stalled: boolean, epoch: number) => { - const data: State = { type: "state", timestamp, stalled, anchored: true, underruns: 0, epoch }; + const state = (timestamp: Micro, stalled: boolean, epoch: number, time = 0, playing = true) => { + const data: State = { type: "state", timestamp, time, stalled, playing, anchored: true, underruns: 0, epoch }; port.dispatchEvent(new MessageEvent("message", { data })); }; return { buffer, sent, state }; @@ -448,6 +456,7 @@ describe("AudioBuffer past its decoded floor", () => { start() {}, }); new AudioBuffer({ port } as unknown as AudioWorkletNode, { + context: silent, channels: 2, rate: 48_000, delay, @@ -538,3 +547,44 @@ describe("AudioBuffer past its decoded floor", () => { } }); }); + +describe("AudioBuffer playhead", () => { + it("maps each report to the sample reaching the output, on the current epoch only", () => { + // Context time 1 s reaches the output at local time 5000 ms. + const output = { contextTime: 0, performanceTime: 0 }; + const context = { getOutputTimestamp: () => output } as unknown as AudioContext; + const { buffer, state } = fake(Milli(100), false, context); + try { + // Nothing has reached the output yet, so there is no mapping to report. + state(ms(2000), false, 0, 1.03); + expect(buffer.playhead.peek()).toBeUndefined(); + + // The next sample, 2 s into the media, renders at context time 1.03 s: 30 ms of output + // latency behind it, the output is at 1.97 s. + Object.assign(output, { contextTime: 1, performanceTime: 5000 }); + state(ms(2000), false, 0, 1.03); + const rolling = buffer.playhead.peek(); + expect(rolling?.rolling).toBe(true); + expect(rolling?.timestamp).toBeCloseTo(1970, 6); + expect(rolling?.rolling && rolling.wall).toBe(Milli(5000)); + + // A stall holds where the ring stopped reading. + state(ms(2010), true, 0, 1.05); + expect(buffer.playhead.peek()).toEqual({ rolling: false, timestamp: Milli(2010) }); + + // A ring held for buffered startup drives nothing. + state(ms(2010), false, 0, 1.06, false); + expect(buffer.playhead.peek()).toBeUndefined(); + + // A reset flushes the timeline, and a report the worklet sent before seeing it is ignored. + state(ms(2020), false, 0, 1.07); + expect(buffer.playhead.peek()?.rolling).toBe(true); + buffer.reset(); + expect(buffer.playhead.peek()).toBeUndefined(); + state(ms(2030), false, 0, 1.08); + expect(buffer.playhead.peek()).toBeUndefined(); + } finally { + buffer.close(); + } + }); +}); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 1bd9e2641bf..7207827b015 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,5 +1,6 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; +import type { Playhead } from "../sync"; import { ringHold, ringSamples } from "./latency"; import type { Data, Init, Latency, Playing, Reset, State, Truncate } from "./render"; @@ -270,6 +271,12 @@ export class AudioBuffer { /** How many times the ring has run dry mid-playback, cumulative. */ readonly underruns: Getter = this.#underruns; + readonly #playhead = new Signal(undefined); + /** Where the output stands on the current timeline, undefined until the ring anchors and is released. */ + readonly playhead: Getter = this.#playhead; + + readonly #context: AudioContext; + // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; // Bumped per reset or park. A state message posted before the worklet saw it reports the ring @@ -282,6 +289,8 @@ export class AudioBuffer { constructor( worklet: AudioWorkletNode, props: { + /** The context the worklet renders in, which maps its output time to the local clock. */ + context: AudioContext; channels: number; rate: number; /** The target delay. The ring holds up to a fixed floor of it decoded; the rest stays encoded. */ @@ -294,6 +303,7 @@ export class AudioBuffer { ) { const { channels, rate, buffered, playing } = props; this.#playing = props.playing; + this.#context = props.context; this.#worklet = worklet; this.channels = channels; this.rate = rate; @@ -324,6 +334,7 @@ export class AudioBuffer { this.#underruns.set(data.underruns); if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); + this.#observe(data); } } }); @@ -331,6 +342,26 @@ export class AudioBuffer { worklet.port.start(); } + // Map a state report to the playhead at the output. The worklet reports the context time its next + // sample renders at, and the context reports which context time is reaching the output at which + // local time, so the output latency is accounted for. + #observe(data: State): void { + if (!data.anchored || !data.playing) { + this.#playhead.set(undefined); + return; + } + const timestamp = Milli.fromMicro(data.timestamp); + if (data.stalled) { + this.#playhead.set({ rolling: false, timestamp }); + return; + } + const output = this.#context.getOutputTimestamp(); + // Nothing has reached the output yet, so there is no mapping to report. + if (!output.performanceTime || output.contextTime === undefined) return; + const behind = Milli.fromSecond((data.time - output.contextTime) as Second); + this.#playhead.set({ rolling: true, timestamp: Milli(timestamp - behind), wall: Milli(output.performanceTime) }); + } + // The floor as the worklet takes it. #latency(): Milli { return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); @@ -399,6 +430,7 @@ export class AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void { this.#epoch++; + this.#playhead.set(undefined); const msg: Reset = { type: "reset", epoch: this.#epoch, diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 1c6060d407f..cdf8a45bc4c 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -33,6 +33,10 @@ class FakeContext extends EventTarget { readonly sampleRate: number; readonly audioWorklet = { addModule: () => moduleLoaded }; readonly calls = { suspend: 0, resume: 0, close: 0 }; + // Which context time reaches the output at which local time. Nothing has until a test says so, + // so the audio playhead drives the clock only in the tests about that. + output: AudioTimestamp = { contextTime: 0, performanceTime: 0 }; + getOutputTimestamp = () => this.output; constructor(options: AudioContextOptions) { super(); this.sampleRate = options.sampleRate ?? 48_000; @@ -63,6 +67,8 @@ let worklets: FakeWorklet[] = []; // every quantum read reports the ring's state back, the way `render-worklet.ts` does. class FakeWorklet { ring?: AudioRingBuffer; + // The context time each reported playhead renders at. + time = 0; #epoch = 0; #playing = true; readonly port = Object.assign(new EventTarget(), { @@ -103,7 +109,9 @@ class FakeWorklet { const state: State = { type: "state", timestamp: ring.timestamp, + time: this.time, stalled: ring.stalled, + playing: this.#playing, anchored: ring.anchored, underruns: ring.underruns, epoch: this.#epoch, @@ -1289,7 +1297,7 @@ it("keeps audio's opening buffered while an enabled video track fills", async () const installed = clock.install(); const playback = await play(Milli(100), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(100)), video); + const remove = playback.sync.track("video", { target: new Signal(Milli(100)), startup: video }); try { await playback.play(8); await clock.advance(Nano.zero); @@ -1304,7 +1312,7 @@ it("keeps audio's opening buffered while an enabled video track fills", async () expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); expect(ring.ring?.timestamp).toBe(Micro((opening ?? 0) + 20_000)); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); installed.close(); @@ -1316,7 +1324,7 @@ it("starts above the decoded audio floor without expanding PCM storage", async ( const installed = clock.install(); const playback = await play(Milli(1000), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(1000)), video); + const remove = playback.sync.track("video", { target: new Signal(Milli(1000)), startup: video }); try { await playback.play(80); await clock.advance(Nano.zero); @@ -1333,7 +1341,7 @@ it("starts above the decoded audio floor without expanding PCM storage", async ( expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); expect(ring.ring?.timestamp).toBe(Micro((opening ?? 0) + 20_000)); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); installed.close(); @@ -1384,6 +1392,7 @@ for (const cmaf of [false, true]) { const installed = clock.install(); const playback = await play(Milli(100), true, cmaf); const reset = spyOn(playback.sync, "reset"); + const video = playback.sync.track("video", { target: new Signal(undefined) }); const prime = spyOn(AudioDecoder.prototype, "reset"); try { await playback.play(7); @@ -1395,7 +1404,7 @@ for (const cmaf of [false, true]) { const held = ring.length; const reference = playback.sync.out.reference.peek(); const resets = playback.resets(); - const parked = playback.sync.wait(Milli(1000)); + const parked = video.wait(Milli(1000)); expect(held).toBeGreaterThan(0); playback.push({ kind: "gap", timestamp: Micro(160_000), duration: Micro(20_000) }); await microtasks(); @@ -1425,7 +1434,7 @@ it.each([100, 1000])( async (delay) => { const playback = await play(Milli(delay), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(delay)), video); + const remove = playback.sync.track("video", { target: new Signal(Milli(delay)), startup: video }); const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); try { @@ -1444,7 +1453,7 @@ it.each([100, 1000])( await microtasks(); expect(playback.inserted()).toHaveLength(decoded); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); disconnect.mockRestore(); @@ -1455,7 +1464,7 @@ it.each([100, 1000])( it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const remove = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + const remove = sync.track("video", { target: new Signal(Milli(100)), startup: new Signal({ start: Milli(0), end: Milli(100) }) }); let reject!: (error: Error) => void; const failure = new Error("audio support probe failed"); const logged = spyOn(console, "warn").mockImplementation(() => {}); @@ -1485,7 +1494,7 @@ it("a rejected audio support probe does not leave buffered video waiting on a pe const decoder = new Decoder({ source, sync }); try { await microtasks(); - sync.received(Milli(0)); + remove.received(Milli(0)); await microtasks(); expect(sync.playing.peek()).toBe(false); reject(failure); @@ -1497,7 +1506,7 @@ it("a rejected audio support probe does not leave buffered video waiting on a pe } finally { decoder.close(); source.close(); - remove(); + remove.close(); sync.close(); logged.mockRestore(); } @@ -1508,7 +1517,7 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C const installed = clock.install(); const playback = await play(Milli(1500), true, cmaf); const video = new Signal<{ start: Milli; end: Milli } | null>({ start: Milli(20), end: Milli(1520) }); - const remove = playback.sync.register(new Signal(Milli(1500)), video); + const remove = playback.sync.track("video", { target: new Signal(Milli(1500)), startup: video }); try { await playback.play(25); // The audio after the hole counts, the hole itself does not. @@ -1534,7 +1543,7 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C expect(playback.sync.playing.peek()).toBe(true); expect(ring.read([new Float32Array(960), new Float32Array(960)])).toBe(960); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); installed.close(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 4dad4c81cb9..318ef1b23df 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -20,7 +20,7 @@ import { base64ToBytes } from "../base64"; import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { supportCacheKey } from "../support-probe"; -import type { Sync } from "../sync"; +import type { AudioTrack, Sync } from "../sync"; import { AudioBuffer } from "./buffer"; import { type DecoderConfig, @@ -127,6 +127,8 @@ export class Decoder { readonly source: Source; /** The clock that schedules playback. */ readonly sync: Sync; + // This track's hold on the clock, which its playhead drives while it plays. + readonly #track: AudioTrack; readonly #out: DecoderOutput = { error: new Signal(undefined), @@ -232,7 +234,8 @@ export class Decoder { const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); return { start: span.start, end: Milli(span.end + Math.min(queued, hold)) }; }); - this.#signals.cleanup(this.sync.register(playout, startup)); + this.#track = this.sync.track("audio", { target: playout, startup }); + this.#signals.cleanup(() => this.#track.close()); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -380,6 +383,7 @@ export class Decoder { effect.cleanup(() => worklet.disconnect()); const ring = new AudioBuffer(worklet, { + context, channels: channelCount, rate: sampleRate, delay: this.sync.out.delay.peek(), @@ -398,6 +402,18 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); + // Drive the clock while this rendition plays. Muted, disabled, or gone, it hands the clock + // back to the wall where it stands, instead of holding video on a ring that only drains. + effect.run((inner) => { + const config = inner.get(this.source.out.config); + const live = + inner.get(this.in.enabled) && + !inner.get(this.sync.out.instant) && + config !== undefined && + config.enabled !== false; + this.#track.playhead(live ? inner.get(ring.playhead) : undefined); + }); + effect.cleanup(() => this.#track.playhead(undefined)); // Each ring counts from zero, so add only what this one counted to the cumulative stats. let counted = 0; effect.run((inner) => { @@ -614,7 +630,7 @@ export class Decoder { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Milli.fromMicro(frame.timestamp as Micro); - const late = this.sync.received(timestamp, frame.arrival); + const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); const duration = packetDuration(config.codec, frame); @@ -700,7 +716,7 @@ export class Decoder { const { frame } = next; const timestamp = Milli.fromMicro(frame.timestamp); - const late = this.sync.received(timestamp, frame.arrival); + const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); const duration = packetDuration(config.codec, frame); @@ -837,6 +853,8 @@ export class Decoder { #retire(): void { this.#consumer = undefined; this.#measured.set(undefined); + // Its arrivals stop holding the clock's reference. + this.#track.reset(); } // The ring, once the worklet has loaded. Decoding waits for it, so the frames that arrive first @@ -985,8 +1003,9 @@ export class Decoder { #onRead(read: Container.Read): boolean { if (!this.#terminal.update(read)) return false; // A hole restarts codec prediction on the same timeline, keeping decoded PCM and parked video. + // A frame after a declared end starts another audio timeline, which leaves video's alone. if (read.kind === "frame") { - this.sync.reset(); + this.#track.reset(); this.reset(); } return true; diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index b6afeadb67e..7276d65f1db 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { Micro, Milli } from "@moq/clock"; import type { Message, State } from "./render"; @@ -37,6 +37,14 @@ try { } } +// The worklet scope's context time, which each quantum starts at; held at zero. +const clock = Object.getOwnPropertyDescriptor(globalThis, "currentTime"); +beforeEach(() => Object.defineProperty(globalThis, "currentTime", { configurable: true, value: 0 })); +afterEach(() => { + if (clock) Object.defineProperty(globalThis, "currentTime", clock); + else Reflect.deleteProperty(globalThis, "currentTime"); +}); + function renderer() { const render = new Render(); const send = (data: Message) => render.port.onmessage?.({ data } as MessageEvent); @@ -78,7 +86,16 @@ describe("render dispatch", () => { send({ type: "data", timestamp: Micro(500_000), data: [new Float32Array(100).fill(2)] }); for (let i = 0; i < 3; i++) read(20); expect(sent).toEqual([ - { type: "state", epoch: 7, timestamp: Micro(560_000), stalled: false, anchored: true, underruns: 0 }, + { + type: "state", + epoch: 7, + timestamp: Micro(560_000), + time: 0.02, + stalled: false, + playing: true, + anchored: true, + underruns: 0, + }, ]); }); @@ -98,7 +115,16 @@ describe("render dispatch", () => { expect(read(20)[19]).toBe(1); read(20); expect(sent).toEqual([ - { type: "state", epoch: 3, timestamp: Micro(60_000), stalled: false, anchored: true, underruns: 0 }, + { + type: "state", + epoch: 3, + timestamp: Micro(60_000), + time: 0.02, + stalled: false, + playing: true, + anchored: true, + underruns: 0, + }, ]); }); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 6cafe639a57..87ffb904f36 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -85,7 +85,9 @@ class Render extends AudioWorkletProcessor { const state: State = { type: "state", timestamp: ring.timestamp, + time: currentTime + output[0].length / ring.rate, stalled: ring.stalled, + playing: this.#playing, anchored: ring.anchored, underruns: ring.underruns, epoch: this.#epoch, diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 13689ab7f76..1000dec17b7 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -49,7 +49,11 @@ export interface Latency { export interface State { type: "state"; timestamp: Micro; + // The context time, in seconds, at which the sample at `timestamp` is rendered. + time: number; stalled: boolean; + // Whether consumption is released; false holds the ring for buffered startup. + playing: boolean; // Whether `timestamp` is on the current timeline: it reads 0 until a frame anchors the ring. anchored: boolean; // How many times the ring has run dry mid-playback, cumulative. diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 9e8f366ab34..f6b2d9ac7a2 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -150,7 +150,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato delay: config.delay !== undefined ? Milli(config.delay) : undefined, }), ); - signals.cleanup(sync.register(playout)); + const audio = sync.track("audio", { target: playout }); + signals.cleanup(() => audio.close()); await settle(); let delay = sync.out.delay.peek(); @@ -190,7 +191,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const { timestamp, arrival } = next.frame; read.set(timestamp); const media = Milli.fromMicro(timestamp); - const late = sync.received(media, arrival); + const late = audio.received(media, arrival); arrivals.push({ kind: "frame", timestamp: media, arrival, late }); const entry = recorded.get(timestamp); if (!entry) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); diff --git a/js/watch/src/index.ts b/js/watch/src/index.ts index c3b1679de15..4ee7d5cc566 100644 --- a/js/watch/src/index.ts +++ b/js/watch/src/index.ts @@ -22,7 +22,7 @@ export { assets } from "./assets"; export * as Audio from "./audio"; export * from "./broadcast"; export * from "./player"; -export * from "./sync"; +export { type Delay, Sync, type SyncInput, type SyncOutput } from "./sync"; /** Caption and subtitle playback. */ export * as Text from "./text"; /** Video rendition selection, decoding, and rendering. */ diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index 9d7219cdc4c..d87c0611099 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -49,10 +49,11 @@ function toContainer(track: Msf.Track): ContainerInfo | undefined { } // Hang stores delay as whole milliseconds rounded up, and zero means the rendition is not behind. -// MSF writes a fractional millisecond, which u53 rejects. +// MSF writes a fractional millisecond, which u53 rejects. A negative value goes to u53 unrounded, so +// it is rejected like a negative jitter rather than read as absent. function delayMillis(value: number): ReturnType | undefined { - if (value <= 0) return undefined; - return u53(Math.ceil(value)); + if (value === 0) return undefined; + return u53(value < 0 ? value : Math.ceil(value)); } // Hang stores jitter as whole milliseconds; round MSF's fractional value up. A negative value goes diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 1a0b823030e..a9879a83bfe 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -59,14 +59,15 @@ async function pendingListeners(run: () => Promise): Promise { // piles it up for the life of the player. test("waits on a stable clock leave nothing behind", async () => { const sync = new Sync({ delay: Milli(10) }); + const video = sync.track("video", { target: new Signal(undefined) }); // Let the delay effect run before anchoring. await yieldNow(); - sync.received(Nano.toMilli(Nano(now()))); + video.received(Nano.toMilli(Nano(now()))); const wait = async () => { for (let round = 0; round < 10; round++) { const at = Nano.toMilli(Nano(now())); - await Promise.all(Array.from({ length: 100 }, () => sync.wait(at))); + await Promise.all(Array.from({ length: 100 }, () => video.wait(at))); } }; expect(await pendingReactions(wait)).toBe(0); @@ -87,6 +88,7 @@ test("a long subscription through the player path leaves nothing behind", async const producer = new Container.Legacy.Producer(track, format); const sync = new Sync({ delay: Milli(10) }); + const audio = sync.track("audio", { target: new Signal(undefined) }); const effect = new Effect(); const sub = subscribeMedia(effect, { broadcast: broadcast.consume(), @@ -111,8 +113,8 @@ test("a long subscription through the player path leaves nothing behind", async if (!next) throw new Error("the track ended"); const timestamp = Milli.fromMicro(next.frame.timestamp); - sync.received(timestamp); - const presented = effect.race(sync.wait(timestamp)); + audio.received(timestamp); + const presented = effect.race(audio.wait(timestamp)); presenting.add(presented); void presented.finally(() => presenting.delete(presented)); } diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 5dc3869b326..4dd2368c4b0 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -77,8 +77,8 @@ describe("auto delay", () => { const audio = new Signal(120 as Milli); const video = new Signal(40 as Milli); const sync = new Sync(); - const unregisterAudio = sync.register(audio); - sync.register(video); + const unregisterAudio = sync.track("audio", { target: audio }); + sync.track("video", { target: video }); await flush(); expect(sync.out.delay.peek()).toBe(120 as Milli); @@ -87,7 +87,7 @@ describe("auto delay", () => { await flush(); expect(sync.out.delay.peek()).toBe(270 as Milli); - unregisterAudio(); + unregisterAudio.close(); await flush(); expect(sync.out.delay.peek()).toBe(40 as Milli); sync.close(); @@ -96,14 +96,14 @@ describe("auto delay", () => { it("unregisters duplicate targets independently", async () => { const media = new Signal(20 as Milli); const sync = new Sync(); - const unregisterFirst = sync.register(media); - const unregisterSecond = sync.register(media); + const unregisterFirst = sync.track("video", { target: media }); + const unregisterSecond = sync.track("video", { target: media }); - unregisterFirst(); + unregisterFirst.close(); await flush(); expect(sync.out.delay.peek()).toBe(20 as Milli); - unregisterSecond(); + unregisterSecond.close(); await flush(); expect(sync.out.delay.peek()).toBe(0 as Milli); sync.close(); @@ -112,7 +112,7 @@ describe("auto delay", () => { it("takes a fixed delay literally, ignoring the measured targets", async () => { const media = new Signal(250 as Milli); const sync = new Sync({ delay: 100 as Milli }); - sync.register(media); + sync.track("video", { target: media }); await flush(); expect(sync.out.delay.peek()).toBe(100 as Milli); sync.close(); @@ -133,24 +133,27 @@ describe("wait", () => { it("presents at once when constructed instant, before effects flush", async () => { const sync = new Sync({ delay: "instant" }); - sync.received(nowMs()); - expect(await sync.wait(Milli(nowMs() + 10_000))).toBe(true); + const video = sync.track("video", { target: new Signal(undefined) }); + video.received(nowMs()); + expect(await video.wait(Milli(nowMs() + 10_000))).toBe(true); sync.close(); }); it("drops an unanchored frame", async () => { const sync = new Sync({ delay: "instant" }); - expect(await sync.wait(nowMs())).toBe(false); + const video = sync.track("video", { target: new Signal(undefined) }); + expect(await video.wait(nowMs())).toBe(false); sync.close(); }); it("presents a frame once its deadline comes due", async () => { const sync = new Sync({ delay: 100 as Milli }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); - sync.received(timestamp); + video.received(timestamp); - const waiting = sync.wait(timestamp); + const waiting = video.wait(timestamp); await flush(); expect(Bun.peek.status(waiting)).toBe("pending"); @@ -162,10 +165,11 @@ describe("wait", () => { it("wakes a sleeping wait to present when the delay switches to instant", async () => { const delay = new Signal(10_000 as Milli); const sync = new Sync({ delay }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); - sync.received(nowMs()); + video.received(nowMs()); - const waiting = sync.wait(nowMs()); + const waiting = video.wait(nowMs()); await flush(); expect(Bun.peek.status(waiting)).toBe("pending"); @@ -176,45 +180,48 @@ describe("wait", () => { it("drops a frame a skip passed over, before and after it parks", async () => { const sync = new Sync({ delay: 100 as Milli }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); - sync.received(timestamp); + video.received(timestamp); - const parked = sync.wait(timestamp); + const parked = video.wait(timestamp); await flush(); expect(Bun.peek.status(parked)).toBe("pending"); // A frame 10s newer, with no buffer, skips the playhead past the parked one. - sync.received(Milli(timestamp + 10_000)); + video.received(Milli(timestamp + 10_000)); expect(await parked).toBe(false); - expect(await sync.wait(timestamp)).toBe(false); + expect(await video.wait(timestamp)).toBe(false); sync.close(); }); it("drops a sleeping wait at a reset, even when the next anchor restores the reference", async () => { const sync = new Sync({ delay: 10_000 as Milli }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); - sync.received(timestamp); + video.received(timestamp); - const waiting = sync.wait(timestamp); + const waiting = video.wait(timestamp); await flush(); // In one turn, so the reference never visibly changes. sync.reset(); - sync.received(timestamp); + video.received(timestamp); expect(await waiting).toBe(false); sync.close(); }); it("drops a sleeping wait when its signal aborts", async () => { const sync = new Sync({ delay: 10_000 as Milli }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); - sync.received(timestamp); + video.received(timestamp); const controller = new AbortController(); - const waiting = sync.wait(timestamp, controller.signal); + const waiting = video.wait(timestamp, controller.signal); await flush(); controller.abort(); expect(await waiting).toBe(false); @@ -223,11 +230,12 @@ describe("wait", () => { it("drops a sleeping wait when closed", async () => { const sync = new Sync({ delay: 10_000 as Milli }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); - sync.received(timestamp); + video.received(timestamp); - const waiting = sync.wait(timestamp); + const waiting = video.wait(timestamp); await flush(); sync.close(); expect(await waiting).toBe(false); @@ -240,12 +248,13 @@ describe("recorded ingress", () => { clock.jump(Nano.fromMilli(Milli(10_000))); const installed = clock.install(); const sync = new Sync({ delay: Milli(100) }); + const video = sync.track("video", { target: new Signal(undefined) }); try { for (let i = 0; i < 20; i++) await Promise.resolve(); - expect(sync.received(Milli(0), Milli(1000))).toBe(false); + expect(video.received(Milli(0), Milli(1000))).toBe(false); expect(sync.out.reference.peek()).toBe(1000); - expect(sync.received(Milli(20), Milli(1020))).toBe(false); - expect(sync.received(Milli(40), Milli(1200))).toBe(true); + expect(video.received(Milli(20), Milli(1020))).toBe(false); + expect(video.received(Milli(40), Milli(1200))).toBe(true); } finally { installed.close(); sync.close(); @@ -266,63 +275,67 @@ describe("skip floor", () => { it("a skip passes over only frames behind the playhead it jumped to", async () => { const sync = new Sync({ delay: Milli(100) }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); - sync.received(Milli(1000), Milli(5000)); - expect(sync.presentable(Milli(0))).toBe(true); + video.received(Milli(1000), Milli(5000)); + expect(video.presentable(Milli(0))).toBe(true); // A startup burst with no buffer: audio 50ms newer skips the playhead to 950ms. - sync.received(Milli(1050), Milli(5000)); - expect(sync.presentable(Milli(1000))).toBe(true); - expect(sync.presentable(Milli(949))).toBe(false); + video.received(Milli(1050), Milli(5000)); + expect(video.presentable(Milli(1000))).toBe(true); + expect(video.presentable(Milli(949))).toBe(false); // A large skip passes over the earlier frame. - sync.received(Milli(10_000), Milli(5000)); - expect(sync.presentable(Milli(1000))).toBe(false); - expect(sync.presentable(Milli(9900))).toBe(true); + video.received(Milli(10_000), Milli(5000)); + expect(video.presentable(Milli(1000))).toBe(false); + expect(video.presentable(Milli(9900))).toBe(true); sync.close(); }); it("a skip never lowers the floor and a reset clears it", async () => { const delay = new Signal(Milli(100)); const sync = new Sync({ delay }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); - sync.received(Milli(0), Milli(5000)); - sync.received(Milli(10_000), Milli(5000)); - expect(sync.presentable(Milli(9000))).toBe(false); + video.received(Milli(0), Milli(5000)); + video.received(Milli(10_000), Milli(5000)); + expect(video.presentable(Milli(9000))).toBe(false); // A larger cap re-anchors to a lower playhead, but cannot revive what was passed over. delay.set(Milli(5000)); await flush(); - sync.received(Milli(11_000), Milli(5000)); - expect(sync.presentable(Milli(9000))).toBe(false); + video.received(Milli(11_000), Milli(5000)); + expect(video.presentable(Milli(9000))).toBe(false); // A new timeline may start its timestamps over, once it is anchored. sync.reset(); - expect(sync.presentable(Milli(9000))).toBe(false); - sync.received(Milli(0), Milli(5000)); - expect(sync.presentable(Milli(9000))).toBe(true); + expect(video.presentable(Milli(9000))).toBe(false); + video.received(Milli(0), Milli(5000)); + expect(video.presentable(Milli(9000))).toBe(true); sync.close(); }); it("a skip counts how long the skipping frame waited after ingress", async () => { const sync = new Sync({ delay: Milli(100) }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); - sync.received(Milli(0), Milli(5000)); + video.received(Milli(0), Milli(5000)); // Reordering held this audio for 200ms, so the new reference puts the playhead at 10.1s now. clock.jump(Nano.fromMilli(Milli(200))); - sync.received(Milli(10_000), Milli(5000)); - expect(sync.presentable(Milli(10_050))).toBe(false); - expect(sync.presentable(Milli(10_100))).toBe(true); + video.received(Milli(10_000), Milli(5000)); + expect(video.presentable(Milli(10_050))).toBe(false); + expect(video.presentable(Milli(10_100))).toBe(true); sync.close(); }); it("instant skips pass over nothing", async () => { const sync = new Sync({ delay: "instant" }); + const video = sync.track("video", { target: new Signal(undefined) }); await flush(); - sync.received(Milli(0), Milli(5000)); - sync.received(Milli(10_000), Milli(5000)); - expect(sync.presentable(Milli(0))).toBe(true); + video.received(Milli(0), Milli(5000)); + video.received(Milli(10_000), Milli(5000)); + expect(video.presentable(Milli(0))).toBe(true); sync.close(); }); }); @@ -337,14 +350,15 @@ describe("buffered startup", () => { audio: new Signal<{ start: Milli; end: Milli } | null>(null), video: new Signal<{ start: Milli; end: Milli } | null>(null), }; - const dispose = Object.values(spans).map((span) => sync.register(new Signal(Milli(100)), span)); + const tracks = Object.values(spans).map((startup) => sync.track("video", { target: new Signal(Milli(100)), startup })); + const [video] = tracks; try { await clock.advance(Nano.zero); - sync.received(Milli(500)); + video.received(Milli(500)); spans[first].set({ start: Milli(500), end: Milli(600) }); await clock.advance(Nano.zero); let delivered = false; - const waiting = sync.wait(Milli(500)).then((ok) => { + const waiting = video.wait(Milli(500)).then((ok) => { delivered = ok; }); await clock.advance(Nano.fromMilli(Milli(1000))); @@ -359,7 +373,7 @@ describe("buffered startup", () => { expect(delivered).toBe(true); expect(sync.now()).toBe(Milli(500)); } finally { - for (const fn of dispose) fn(); + for (const track of tracks) track.close(); sync.close(); installed.close(); } @@ -369,24 +383,24 @@ describe("buffered startup", () => { const clock = new Manual(); const installed = clock.install(); const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const dispose = sync.register(new Signal(Milli(100)), new Signal(null)); + const video = sync.track("video", { target: new Signal(Milli(100)), startup: new Signal(null) }); try { await clock.advance(Nano.zero); - sync.received(Milli(0)); + video.received(Milli(0)); await clock.advance(Nano.zero); const abort = new AbortController(); - const cancelled = sync.wait(Milli(0), abort.signal); + const cancelled = video.wait(Milli(0), abort.signal); abort.abort(); expect(await cancelled).toBe(false); - const reset = sync.wait(Milli(0)); + const reset = video.wait(Milli(0)); sync.reset(); - sync.received(Milli(0)); + video.received(Milli(0)); expect(await reset).toBe(false); - const closed = sync.wait(Milli(0)); + const closed = video.wait(Milli(0)); sync.close(); expect(await closed).toBe(false); } finally { - dispose(); + video.close(); sync.close(); installed.close(); } @@ -397,27 +411,30 @@ describe("buffered startup", () => { const buffer = new Signal(Milli(1000)); const sync = new Sync({ delay: Milli(100), buffer }); const absent = new Signal<{ start: Milli; end: Milli } | null | undefined>(null); - const remove = sync.register(new Signal(Milli(100)), absent); - const video = sync.register(new Signal(Milli(100)), new Signal({ start: Milli(0), end: Milli(100) })); + const remove = sync.track("audio", { target: new Signal(Milli(100)), startup: absent }); + const video = sync.track("video", { + target: new Signal(Milli(100)), + startup: new Signal({ start: Milli(0), end: Milli(100) }), + }); try { await clock.advance(Nano.zero); - sync.received(Milli(0)); + video.received(Milli(0)); await clock.advance(Nano.zero); expect(sync.playing.peek()).toBe(false); absent.set(undefined); await clock.advance(Nano.zero); - expect(await sync.wait(Milli(0))).toBe(true); + expect(await video.wait(Milli(0))).toBe(true); buffer.set(Milli.zero); sync.reset(); absent.set(null); await clock.advance(Nano.zero); - sync.received(Milli(0)); - const waiting = sync.wait(Milli(0)); + video.received(Milli(0)); + const waiting = video.wait(Milli(0)); await clock.advance(Nano.fromMilli(Milli(100))); expect(await waiting).toBe(true); } finally { - remove(); - video(); + remove.close(); + video.close(); sync.close(); installed.close(); } diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 7abc4f18fbe..2a6fdf49f3f 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -1,14 +1,5 @@ import { Milli, Nano, now, sleep } from "@moq/clock"; -import { - type Dispose, - Effect, - type Getter, - getter, - type Inputs, - type Readonlys, - readonlys, - Signal, -} from "@moq/signals"; +import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; /** * How far playback trails the live edge. @@ -48,9 +39,11 @@ export type SyncOutput = { // that only care about the mode don't rerun when the number changes. instant: Signal; - // The earliest time we've received a frame, relative to its timestamp: arrival minus timestamp - // in milliseconds, an offset that is negative when the timestamps run ahead of the local clock. - // This will keep being updated as we catch up to the live playhead then will be relatively static. + // Local time minus media time, less the delay: `now()` is the local time minus this minus the + // delay. While audio plays it follows the audio playhead, re-synced on every report from the + // render worklet. Otherwise it is the earliest a frame has arrived relative to its timestamp, + // negative when timestamps run ahead of the local clock, and it settles once playback catches up + // to the live edge. reference: Signal; // The resolved delay from the live edge to the playhead: the configured number, or in "auto" the @@ -70,12 +63,43 @@ export type SyncOutput = { maxDelay: Signal; }; -/** The playback clock: decides when each frame plays so audio and video stay together at the configured delay. */ -export class Sync { - /** The wired inputs, read-only. */ - readonly in: Readonlys; +/** The media kinds that pace against the clock. @internal */ +export type TrackKind = "audio" | "video" | "text"; + +/** A decoded span from the opening of a track's timeline. @internal */ +export type Span = { start: Milli; end: Milli }; + +/** What a track holds in the shared clock. @internal */ +export type TrackProps = { + /** The track's "auto" playout target, unset until it has one. */ + target: Getter; + /** For buffered startup: undefined is inactive, null is selected but has no playable span yet. */ + startup?: Getter; +}; - readonly #out: SyncOutput = { +/** + * Where audio output stands, as the render worklet last reported it: rolling, the sample at + * `timestamp` reaches the output at local time `wall` (on the `@moq/clock` timeline); stalled + * mid-playback, the output holds at `timestamp`. + * @internal + */ +export type Playhead = { rolling: true; timestamp: Milli; wall: Milli } | { rolling: false; timestamp: Milli }; + +/** What one track holds in the clock. @internal */ +export type Part = { + props: TrackProps; + // Bumped whenever this track's timeline ends, so its parked frames drop. + epoch: Signal; + // The reference this track's own arrivals hold on its current timeline. + anchor?: number; +}; + +const wall = (): Milli => Nano.toMilli(Nano(now())); + +/** The state a {@link Sync} and its track handles share. @internal */ +export class Clock { + readonly in: Readonlys; + readonly out: SyncOutput = { instant: new Signal(false), reference: new Signal(undefined), delay: new Signal(Milli.zero), @@ -83,251 +107,222 @@ export class Sync { buffered: new Signal(false), maxDelay: new Signal(Milli.zero), }; - /** The derived outputs, read-only. */ - readonly out = readonlys(this.#out); - - #media = new Signal< - { - target: Getter; - // undefined is inactive, null is selected but has no playable first span yet. - startup?: Getter<{ start: Milli; end: Milli } | null | undefined>; - }[] - >([]); - #started = false; - #playing = new Signal(true); - /** Whether buffered startup has released the media consumers. @internal */ - readonly playing: Getter = this.#playing; + + readonly parts = new Signal([]); + readonly playing = new Signal(true); + started = false; // Bumped by every `reset()`. A frame that waited across a bump belongs to the previous timeline, // even when the next frame restores the same reference value. - #epoch = new Signal(0); + readonly epoch = new Signal(0); // The furthest playhead a skip has jumped to on this timeline. A frame below it was passed over. - #floor: number | undefined; + floor: number | undefined; // Releases parked waits on `close()`. - #closed = new Signal(false); + readonly closed = new Signal(false); - #signals = new Effect(); + // The track whose arrivals set the reference, which releases it when its timeline ends. + owner?: Part; + + // The audio track, and whether its playhead drives the reference. + audio?: Part; + driving = false; + + // The audio playhead a re-stall holds the clock at. + readonly paused = new Signal(undefined); constructor(props?: Inputs) { this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), buffer: getter(props?.buffer ?? Milli.zero), }; - - this.#signals.run(this.#runInstant.bind(this)); - this.#signals.run(this.#runDelay.bind(this)); - this.#signals.run(this.#runMaxDelay.bind(this)); - this.#signals.run(this.#runStartup.bind(this)); - } - - /** - * Hold a decoder's "auto" playout target in the shared playback clock until disposed. - * @internal - */ - register( - target: Getter, - startup?: Getter<{ start: Milli; end: Milli } | null | undefined>, - ): Dispose { - const registered = { target, startup }; - this.#media.update((media) => [...media, registered]); - return () => this.#media.update((media) => media.filter((candidate) => candidate !== registered)); } - // Startup is data-driven: time passing cannot fill a missing track. The first picture still - // travels through the ordinary renderer while the consumers wait for this shared gate. - #runStartup(effect: Effect): void { - effect.get(this.#epoch); - const buffered = effect.get(this.#out.buffered); - const delay = effect.get(this.#out.delay); - const reference = effect.get(this.#out.reference); - const spans = effect.get(this.#media).flatMap(({ startup }) => { - const span = startup && effect.get(startup); - return span === undefined ? [] : [span]; - }); - if (!buffered || spans.length === 0) { - this.#playing.set(true); - return; - } - if (this.#started) return; - const ready = spans.filter((span) => span !== null); - if ( - reference === undefined || - ready.length !== spans.length || - ready.some((span) => span.end - span.start < delay) - ) { - this.#playing.set(false); - return; - } - const first = Math.min(...ready.map((span) => span.start)); - this.#started = true; - // Anchor at the retained opening, not at its arrival: startup may have waited on another track. - this.#out.reference.set(Nano.toMilli(Nano(now())) - first - delay); - this.#playing.set(true); + // Fold an arrival into a reference: keep it while the frame adds lookahead within `maxDelay`, and + // lower it (skip ahead) only as far as that budget requires. + fold(current: number | undefined, ref: number): number { + if (current === undefined || ref >= current) return current ?? ref; + const sleep = current - ref + this.out.delay.peek(); + const cap = this.out.maxDelay.peek(); + if (sleep <= cap) return current; + return ref + cap - this.out.delay.peek(); } - #runInstant(effect: Effect): void { - this.#out.instant.set(effect.get(this.in.delay) === "instant"); - } - - // Derive `buffered` / `maxDelay` from the resolved delay and the configured lookahead. - #runMaxDelay(effect: Effect): void { - const delay = effect.get(this.#out.delay); - // "instant" holds nothing, so a configured lookahead doesn't apply. - const buffer = effect.get(this.#out.instant) ? Milli.zero : effect.get(this.in.buffer); - - this.#out.buffered.set(buffer > 0); - this.#out.maxDelay.set(Milli(delay + buffer)); - } - - // A fixed delay is taken literally and "auto" holds the deepest track's target, so every track - // plays from one clock. See doc/concept/audio-jitter.md for how a target is measured. - #runDelay(effect: Effect): void { - const mode = effect.get(this.in.delay); - - let delay = Milli.zero; - if (typeof mode === "number") { - delay = mode; - } else if (mode === "auto") { - for (const registered of effect.get(this.#media)) { - delay = Milli(Math.max(delay, effect.get(registered.target) ?? Milli.zero)); - } - } - - this.#out.delay.set(delay); - } - - /** - * Anchor playback from ingress time and report whether the frame arrived late. - * @internal - */ - // Fold a newly received frame into the reference. The reference anchors playback to the - // wall clock; we lower it (skip ahead) only when keeping it would push the lookahead past `maxDelay`. - received(timestamp: Milli, arrival = Nano.toMilli(Nano(now()))): boolean { - this.#out.timestamp.update((current) => (current === undefined || timestamp > current ? timestamp : current)); + received(part: Part, timestamp: Milli, arrival: Milli): boolean { + this.out.timestamp.update((current) => (current === undefined || timestamp > current ? timestamp : current)); const ref = arrival - timestamp; - const currentRef = this.#out.reference.peek(); + const current = this.out.reference.peek(); + part.anchor = this.fold(part.anchor, ref); // First frame anchors the reference. - if (currentRef === undefined) { - this.#out.reference.set(ref); + if (current === undefined) { + this.out.reference.set(ref); + this.owner = part; return false; } // Retain the opening while another track fills; the ordinary max-delay bounds still own // storage, but a later arrival must not advance the shared playhead before startup. - if (!this.#playing.peek()) return false; + if (!this.playing.peek()) return false; // Check if `wait()` would not sleep at all. // NOTE: We check here instead of in `wait()` so we can identify when frames are received late. // Otherwise, chained `wait()` calls would cause a false-positive during CPU starvation. - const delay = this.#out.delay.peek(); - const sleep = currentRef - ref + delay; - const late = sleep < 0; - - // Frame isn't earlier than the anchor: it can't add lookahead, so keep the reference. - if (ref >= currentRef) return late; + const delay = this.out.delay.peek(); + const late = current - ref + delay < 0; - // Frame is earlier (more lookahead). `sleep` is how far ahead of the playhead keeping the - // anchor would put it. - const cap = this.#out.maxDelay.peek(); - if (sleep <= cap) return late; // within budget: let the buffer grow instead of skipping ahead + // The audio playhead owns the reference while it plays: the ring skips and refills on its own, + // and another track's arrivals must not move its timeline. + if (this.driving) return late; - // Over the cap: re-anchor down so the resulting lookahead is exactly the cap. - const reference = ref + cap - delay; - this.#out.reference.set(reference); + const reference = this.fold(current, ref); + if (reference === current) return late; + this.out.reference.set(reference); + this.owner = part; // The playhead now sits at `timestamp - cap`, plus however long this frame waited between // ingress and here. "instant" has no cap, so every newer frame would pass over whatever is // still decoding; it presents frames as they exist instead. if (this.in.delay.peek() !== "instant") { - const floor = Nano.toMilli(Nano(now())) - reference - delay; - if (this.#floor === undefined || floor > this.#floor) this.#floor = floor; + const floor = wall() - reference - delay; + if (this.floor === undefined || floor > this.floor) this.floor = floor; } return late; } - /** - * Re-anchor playback to the next frame received. Call this at an utterance boundary - * in buffered mode (typically alongside flushing the audio buffer) so the new content - * plays from its own first frame instead of inheriting the previous reference. - */ + // Re-sync the reference to the audio playhead, hold it at a re-stall, or hand it back to arrivals. + clock(part: Part, playhead: Playhead | undefined): void { + if (playhead?.rolling) { + this.driving = true; + this.owner = undefined; + this.paused.set(undefined); + this.out.reference.set(playhead.wall - playhead.timestamp - this.out.delay.peek()); + return; + } + if (this.audio !== part || !this.driving) return; + // A ring that runs dry while it still plays holds the clock with it, so video waits for the + // audio it accompanies. Buffered playback drains between bursts by design, so its ring + // stalling means the audio ended, not that it is late. + if (playhead && !this.out.buffered.peek()) { + this.paused.set(playhead.timestamp); + return; + } + this.release(); + } + + // Hand the clock back to the wall at the last audio-derived value, so video does not jump. The + // reference passes to another track still receiving, whose own timeline ending releases it. + release(): void { + const held = this.paused.peek(); + if (held !== undefined) this.out.reference.set(wall() - held - this.out.delay.peek()); + this.paused.set(undefined); + this.driving = false; + this.owner = this.parts.peek().find((part) => part !== this.audio && part.anchor !== undefined); + } + + // End one track's timeline: its parked frames drop and it releases its part of the reference. + end(part: Part): void { + part.epoch.update((epoch) => epoch + 1); + part.anchor = undefined; + if (this.driving) { + // The audio playhead holds the reference, which another track's timeline leaves alone. + if (this.audio !== part) return; + this.release(); + } + // A reference nobody holds, set before any arrival, is released by whichever track ends. + if (this.owner !== undefined && this.owner !== part) return; + + // The earliest remaining track holds the reference, else the next frame anchors a new one. + let earliest: Part | undefined; + for (const other of this.parts.peek()) { + if (other.anchor === undefined) continue; + if (earliest?.anchor === undefined || other.anchor < earliest.anchor) earliest = other; + } + this.owner = earliest; + if (earliest) this.out.reference.set(earliest.anchor); + else this.unanchor(); + } + + // Clear the reference so the next frame anchors a new timeline. Every parked frame belongs to + // the old one. + unanchor(): void { + this.epoch.update((epoch) => epoch + 1); + this.floor = undefined; + this.started = false; + this.owner = undefined; + this.playing.set(!this.out.buffered.peek()); + this.out.reference.set(undefined); + } + reset(): void { - this.#epoch.update((epoch) => epoch + 1); - this.#floor = undefined; - this.#started = false; - this.#playing.set(!this.#out.buffered.peek()); - this.#out.reference.set(undefined); + for (const part of this.parts.peek()) part.anchor = undefined; + this.driving = false; + this.paused.set(undefined); + this.unanchor(); } - /** - * Whether a frame at `timestamp` is anchored on the current timeline and ahead of any skip, so - * it may still present. Advisory, for showing a frame early; {@link Sync.wait} decides on its own. - * @internal - */ presentable(timestamp: Milli): boolean { - return this.#out.reference.peek() !== undefined && !this.#skipped(timestamp); + return this.out.reference.peek() !== undefined && !this.skipped(timestamp); } // Whether a playhead skip on this timeline passed over `timestamp`. - #skipped(timestamp: Milli): boolean { - return this.#floor !== undefined && timestamp < this.#floor; + skipped(timestamp: Milli): boolean { + return this.floor !== undefined && timestamp < this.floor; } - /** - * The PTS that should be rendering right now, derived from the reference + delay; negative - * before the first frame's turn. Undefined before ingress or while buffered startup is filling. - */ now(): Milli | undefined { - if (!this.#playing.peek()) return undefined; - const reference = this.#out.reference.peek(); + if (!this.playing.peek()) return undefined; + const reference = this.out.reference.peek(); if (reference === undefined) return undefined; - return Milli(Nano.toMilli(Nano(now())) - reference - this.#out.delay.peek()); + return this.paused.peek() ?? Milli(wall() - reference - this.out.delay.peek()); } - /** - * Sleep until it's time to render this frame, then resolve whether to present it. - * - * `true` presents the frame now. `false` drops it: it was unanchored, a skip passed over it, a - * `reset()` ended its timeline, or `close()` or `signal` released the wait. - * @internal - */ - async wait(timestamp: Milli, signal?: AbortSignal): Promise { - const epoch = this.#epoch.peek(); + async wait(part: Part | undefined, timestamp: Milli, signal?: AbortSignal): Promise { + const epoch = this.epoch.peek(); + const own = part?.epoch.peek(); for (;;) { // Closing or aborting wakes the sleep below, so frames parked here leave. - if (this.#closed.peek() || signal?.aborted) return false; + if (this.closed.peek() || signal?.aborted) return false; // A reset clears the reference, so an unanchored frame predates the current timeline. - const reference = this.#out.reference.peek(); - if (reference === undefined || this.#epoch.peek() !== epoch || this.#skipped(timestamp)) return false; + const reference = this.out.reference.peek(); + if (reference === undefined || this.epoch.peek() !== epoch || this.skipped(timestamp)) return false; + if (part && part.epoch.peek() !== own) return false; // A zero delay still sleeps: the sleep comes from the reference, which holds an early frame // until its timestamp comes up. "instant" is the only thing that skips the wait itself. // Peek the input, not `out.instant`, which lags a same-turn change until effects flush. if (this.in.delay.peek() === "instant") return true; - if (!this.#playing.peek()) { - await this.#sleep(undefined, signal); + if (!this.playing.peek()) { + await this.sleep(part, undefined, signal); + continue; + } + + // A stalled audio ring holds the playhead until it plays again. + const paused = this.paused.peek(); + if (paused !== undefined) { + if (timestamp <= paused) return true; + await this.sleep(part, undefined, signal); continue; } // NOTE: This function runs in parallel for each frame. - const sleep = reference + timestamp - Nano.toMilli(Nano(now())) + this.#out.delay.peek(); + const sleep = reference + timestamp - wall() + this.out.delay.peek(); // Skip the timer for small sleeps; the host's timer resolution (~4ms) would overshoot. if (sleep < 5) return true; - await this.#sleep(sleep, signal); + await this.sleep(part, sleep, signal); } } // Sleeps for `ms`, or wakes early once anything the sleep was computed from changes or `signal` // aborts. Releases every listener either way: frames sleep once each, so a listener left on a // signal that never changes would pile up for the life of the player. - #sleep(ms: number | undefined, signal?: AbortSignal): Promise { + sleep(part: Part | undefined, ms: number | undefined, signal?: AbortSignal): Promise { return new Promise((resolve) => { const timer = new AbortController(); const wake = () => { @@ -338,22 +333,227 @@ export class Sync { if (ms !== undefined) void sleep(Nano.fromMilli(Milli(ms)), { signal: timer.signal }).then(wake, () => {}); const disposes = [ this.in.delay.changed(wake), - this.#out.delay.changed(wake), - this.#out.reference.changed(wake), - this.#epoch.changed(wake), - this.#closed.changed(wake), - this.#playing.changed(wake), + this.out.delay.changed(wake), + this.out.reference.changed(wake), + this.epoch.changed(wake), + this.closed.changed(wake), + this.playing.changed(wake), + this.paused.changed(wake), ]; + if (part) disposes.push(part.epoch.changed(wake)); if (signal) { signal.addEventListener("abort", wake, { once: true }); disposes.push(() => signal.removeEventListener("abort", wake)); } }); } +} + +/** + * One track's hold on the shared clock: its playout target and startup span, its arrivals, and the + * waits that pace its frames. Close it when the track goes away. + * @internal + */ +export class Track { + protected readonly clock: Clock; + protected readonly part: Part; + + /** @internal */ + constructor(clock: Clock, props: TrackProps) { + this.clock = clock; + this.part = { props, epoch: new Signal(0) }; + clock.parts.update((parts) => [...parts, this.part]); + } + + /** Fold a received frame into the clock and report whether it arrived too late for its turn. */ + received(timestamp: Milli, arrival = wall()): boolean { + return this.clock.received(this.part, timestamp, arrival); + } + + /** + * Whether a frame at `timestamp` is anchored on the current timeline and ahead of any skip, so it + * may still present. Advisory, for showing a frame early; {@link Track.wait} decides on its own. + */ + presentable(timestamp: Milli): boolean { + return this.clock.presentable(timestamp); + } + + /** + * Sleep until it's time to render this frame, then resolve whether to present it. + * + * `true` presents the frame now. `false` drops it: it was unanchored, a skip passed over it, a + * reset ended its timeline, or `close()` or `signal` released the wait. + */ + wait(timestamp: Milli, signal?: AbortSignal): Promise { + return this.clock.wait(this.part, timestamp, signal); + } + + /** + * End this track's timeline, at a discontinuity or when it stops receiving. Its parked frames + * drop, and its arrivals stop holding the reference; the other tracks' timelines are untouched. + */ + reset(): void { + this.clock.end(this.part); + } + + /** Release this track's hold on the clock. */ + close(): void { + this.clock.end(this.part); + this.clock.parts.update((parts) => parts.filter((part) => part !== this.part)); + if (this.clock.audio === this.part) this.clock.audio = undefined; + } +} + +/** The audio track, whose playhead drives the clock while it plays. @internal */ +export class AudioTrack extends Track { + /** @internal */ + constructor(clock: Clock, props: TrackProps) { + if (clock.audio) throw new Error("the clock already has an audio track"); + super(clock, props); + clock.audio = this.part; + } + + /** + * Report the audio playhead. A rolling one drives the clock, a stalled one holds it, and undefined + * (muted, disabled, or flushed) hands it back to the wall clock where it stands. + */ + playhead(playhead: Playhead | undefined): void { + this.clock.clock(this.part, playhead); + } +} + +/** The playback clock: decides when each frame plays so audio and video stay together at the configured delay. */ +export class Sync { + /** The wired inputs, read-only. */ + readonly in: Readonlys; + /** The derived outputs, read-only. */ + readonly out: Readonlys; + + readonly #clock: Clock; + /** Whether buffered startup has released the media consumers. @internal */ + readonly playing: Getter; + + #signals = new Effect(); + + constructor(props?: Inputs) { + this.#clock = new Clock(props); + this.in = this.#clock.in; + this.out = readonlys(this.#clock.out); + this.playing = this.#clock.playing; + + this.#signals.run(this.#runInstant.bind(this)); + this.#signals.run(this.#runDelay.bind(this)); + this.#signals.run(this.#runMaxDelay.bind(this)); + this.#signals.run(this.#runStartup.bind(this)); + } + + /** + * Open a track's hold on the clock. The audio track's playhead drives the clock while it plays; + * only one may be open at a time. + * @internal + */ + track(kind: "audio", props: TrackProps): AudioTrack; + track(kind: "video" | "text", props: TrackProps): Track; + track(kind: TrackKind, props: TrackProps): Track { + return kind === "audio" ? new AudioTrack(this.#clock, props) : new Track(this.#clock, props); + } + + // Startup is data-driven: time passing cannot fill a missing track. The first picture still + // travels through the ordinary renderer while the consumers wait for this shared gate. + #runStartup(effect: Effect): void { + const clock = this.#clock; + effect.get(clock.epoch); + const buffered = effect.get(clock.out.buffered); + const delay = effect.get(clock.out.delay); + const reference = effect.get(clock.out.reference); + const spans = effect.get(clock.parts).flatMap((part) => { + const span = part.props.startup && effect.get(part.props.startup); + return span === undefined ? [] : [{ part, span }]; + }); + if (!buffered || spans.length === 0) { + clock.playing.set(true); + return; + } + if (clock.started) return; + const ready = spans.flatMap(({ part, span }) => (span ? [{ part, span }] : [])); + if ( + reference === undefined || + ready.length !== spans.length || + ready.some(({ span }) => span.end - span.start < delay) + ) { + clock.playing.set(false); + return; + } + // The track with the earliest opening holds the reference it anchors. + const earliest = ready.reduce((a, b) => (b.span.start < a.span.start ? b : a)); + const first = earliest.span.start; + clock.started = true; + clock.owner = earliest.part; + // Anchor at the retained opening, not at its arrival: startup may have waited on another track. + clock.out.reference.set(wall() - first - delay); + clock.playing.set(true); + } + + #runInstant(effect: Effect): void { + this.#clock.out.instant.set(effect.get(this.in.delay) === "instant"); + } + + // Derive `buffered` / `maxDelay` from the resolved delay and the configured lookahead. + #runMaxDelay(effect: Effect): void { + const out = this.#clock.out; + const delay = effect.get(out.delay); + // "instant" holds nothing, so a configured lookahead doesn't apply. + const buffer = effect.get(out.instant) ? Milli.zero : effect.get(this.in.buffer); + + out.buffered.set(buffer > 0); + out.maxDelay.set(Milli(delay + buffer)); + } + + // A fixed delay is taken literally and "auto" holds the deepest track's target, so every track + // plays from one clock. See doc/concept/audio-jitter.md for how a target is measured. + #runDelay(effect: Effect): void { + const mode = effect.get(this.in.delay); + + let delay = Milli.zero; + if (typeof mode === "number") { + delay = mode; + } else if (mode === "auto") { + for (const { props } of effect.get(this.#clock.parts)) { + delay = Milli(Math.max(delay, effect.get(props.target) ?? Milli.zero)); + } + } + + // The audio playhead does not move with the delay: the ring parks or sheds to reach it, and + // reports where it went. Keep `now()` on the playhead until it does. + const clock = this.#clock; + const reference = clock.out.reference.peek(); + if (clock.driving && reference !== undefined) { + clock.out.reference.set(reference - (delay - clock.out.delay.peek())); + } + clock.out.delay.set(delay); + } + + /** + * Re-anchor playback to the next frame received. Call this at an utterance boundary + * in buffered mode (typically alongside flushing the audio buffer) so the new content + * plays from its own first frame instead of inheriting the previous reference. + */ + reset(): void { + this.#clock.reset(); + } + + /** + * The PTS that should be rendering right now: the audio playhead while audio plays, else derived + * from the reference and delay; negative before the first frame's turn. Undefined before ingress + * or while buffered startup is filling. + */ + now(): Milli | undefined { + return this.#clock.now(); + } /** Stop and release everything this holds. */ close() { - this.#closed.set(true); + this.#clock.closed.set(true); this.#signals.close(); } } diff --git a/js/watch/src/text/renderer.ts b/js/watch/src/text/renderer.ts index 049caca2b23..c9a8ee1a814 100644 --- a/js/watch/src/text/renderer.ts +++ b/js/watch/src/text/renderer.ts @@ -5,7 +5,7 @@ import { Error as NetError } from "@moq/net"; import { Derived, Effect, type Getter, getter, type Inputs, type Readonlys } from "@moq/signals"; import type { VTTCue, VTTRegion } from "media-captions"; import { subscribeMedia } from "../media"; -import type { Sync } from "../sync"; +import type { Sync, Track } from "../sync"; import type { Source } from "./source"; // Drop cues whose end is more than this far behind the playhead, to bound memory on a long stream. @@ -227,6 +227,10 @@ export class Renderer { #signals = new Effect(); + // The captions' hold on the clock: the selected rendition's catalog delay, so cues from a + // rendition that trails the others still line up with them. + readonly #clock: Track; + // Whether the clock-skew warning has already fired for the current track, so a misconfigured // publisher costs one line instead of one per cue. #skewWarned = false; @@ -239,6 +243,14 @@ export class Renderer { enabled: getter(props?.enabled ?? true), }; + const target = this.#signals.computed((effect) => { + if (!effect.get(this.in.enabled)) return undefined; + const delay = effect.get(this.source.out.config)?.delay; + return delay !== undefined ? Milli(delay) : undefined; + }); + this.#clock = this.sync.track("text", { target }); + this.#signals.cleanup(() => this.#clock.close()); + this.#signals.run(this.#run.bind(this)); } diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index fa400a5cacf..dbea56d638b 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -8,7 +8,7 @@ import { Effect, Signal } from "@moq/signals"; import { ringSamples } from "../audio/latency"; import { AudioRingBuffer } from "../audio/ring-buffer"; import type { Broadcast } from "../broadcast"; -import { type Delay, Sync } from "../sync"; +import { type Delay, Sync, Track } from "../sync"; import { Decoder, insertRecoveryPoint } from "./decoder"; import { Source } from "./source"; @@ -380,9 +380,12 @@ async function guardedPlayback( ? mediaProducer.createTrack("video", { timescale: Moq.Time.Timescale.MILLI, maxAge: Container.MAX_AGE }) : undefined; const decoder = new Decoder({ source, sync, enabled }); + // Another track on the same clock, whose arrivals the tests drive by hand. + const audio = sync.track("audio", { target: new Signal(undefined) }); await microtasks(); return { decoder, + audio, delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => @@ -644,7 +647,7 @@ for (const kind of ["legacy", "cmaf"] as const) { expect(playback.outputs).toEqual(stage === "queued" ? [] : [picture]); const before = playback.decoder.out.frame.peek(); - playback.sync.received(Milli(skipTo), Milli(5000)); + playback.audio.received(Milli(skipTo), Milli(5000)); playback.release(); await microtasks(); const due = playback.decoder.out.frame.peek(); @@ -680,7 +683,7 @@ for (const kind of ["legacy", "cmaf"] as const) { expect(playback.outputs).toEqual([]); // "instant" has no cap, so every newer frame skips the playhead without passing over anything. - playback.sync.received(Milli(10_000), Nano.toMilli(Nano(now()))); + playback.audio.received(Milli(10_000), Nano.toMilli(Nano(now()))); playback.release(); await microtasks(); @@ -932,7 +935,7 @@ it("a shared audio clock reset cannot present a parked video frame", async () => await microtasks(); expect(playback.decoder.out.timestamp.peek()).toBe(Milli(0)); // The next utterance anchors the clock past the parked frame's deadline. - playback.sync.received(Milli(2000)); + playback.audio.received(Milli(2000)); await microtasks(); expect(playback.decoder.out.timestamp.peek()).toBe(Milli(0)); } finally { @@ -947,9 +950,9 @@ it("closing the track releases a parked video wait", async () => { const reads: Read[] = []; const playback = await guardedPlayback("legacy", reads); const waits: Promise[] = []; - const original = playback.sync.wait.bind(playback.sync); - const wait = spyOn(playback.sync, "wait").mockImplementation((timestamp, signal) => { - const waiting = original(timestamp, signal); + const original = Track.prototype.wait; + const wait = spyOn(Track.prototype, "wait").mockImplementation(function (this: Track, timestamp, signal) { + const waiting = original.call(this, timestamp, signal); waits.push(waiting); return waiting; }); @@ -988,7 +991,7 @@ it("a reset drops a parked video frame even when the next anchor restores the re expect(playback.submitted).toEqual([0, 100_000]); // In one turn, so the reference never visibly changes and the parked sleep keeps its timer. playback.sync.reset(); - playback.sync.received(Milli(0)); + playback.audio.received(Milli(0)); await microtasks(); // The parked sleep comes due; its frame belongs to the timeline the reset ended. await clock.advance(1000); @@ -1210,7 +1213,7 @@ for (const kind of ["legacy", "cmaf"] as const) { const reads: Read[] = []; const playback = await guardedPlayback(kind, reads, [], false, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(100)), audio); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); const reset = spyOn(playback.sync, "reset"); try { playback.enabled.set(false); @@ -1238,7 +1241,7 @@ for (const kind of ["legacy", "cmaf"] as const) { expect(playback.decoder.out.timestamp.peek()).toBe(Milli(940)); expect(reset).not.toHaveBeenCalled(); } finally { - remove(); + remove.close(); reset.mockRestore(); playback.close(); await microtasks(); @@ -1258,7 +1261,7 @@ for (const kind of ["legacy", "cmaf"] as const) { reads.push(sample(group * 2n, Number(group) * 40_000)); } const playback = await guardedPlayback(kind, reads, [], false, 1000); - const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: new Signal(null) }); try { await microtasks(); await clock.advance(0); @@ -1267,7 +1270,7 @@ for (const kind of ["legacy", "cmaf"] as const) { expect(playback.submitted.at(-1)).toBe(1_120_000); expect(playback.sync.playing.peek()).toBe(false); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); clock.installed.close(); @@ -1278,7 +1281,7 @@ for (const kind of ["legacy", "cmaf"] as const) { it("bounds startup decoding while another track is missing and releases it on close", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); - const remove = playback.sync.register(new Signal(Milli(100)), new Signal(null)); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: new Signal(null) }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1291,7 +1294,7 @@ it("bounds startup decoding while another track is missing and releases it on cl await microtasks(); expect(playback.pictures.every((frame) => frame.closed)).toBe(true); } finally { - remove(); + remove.close(); group.finish(); playback.close(); await microtasks(); @@ -1304,7 +1307,7 @@ it("resumes decoding the moment shared startup releases", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(100)), audio); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1317,7 +1320,7 @@ it("resumes decoding the moment shared startup releases", async () => { await clock.advance(0); expect(playback.submitted).toHaveLength(8); } finally { - remove(); + remove.close(); group.finish(); playback.close(); await microtasks(); @@ -1352,7 +1355,7 @@ it("continues filling when the shared floor rises while video waits on audio", a const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(100)), audio); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1366,7 +1369,7 @@ it("continues filling when the shared floor rises while video waits on audio", a await clock.advance(240); expect(playback.decoder.out.frame.peek()?.timestamp).toBeGreaterThanOrEqual(200_000); } finally { - remove(); + remove.close(); group.finish(); playback.close(); await microtasks(); @@ -1397,7 +1400,7 @@ for (const kind of ["legacy", "cmaf"] as const) { const clock = manualAt(0); const playback = await guardedPlayback(kind, [sample(0n, 0), sample(0n, 100_000)], [0, 100_000], false, 1000); const other = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.register(new Signal(Milli(100)), other); + const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: other }); try { playback.release(); await microtasks(); @@ -1419,7 +1422,7 @@ for (const kind of ["legacy", "cmaf"] as const) { await microtasks(); expect(playback.pictures.filter((frame) => !frame.closed)).toEqual([]); } finally { - remove(); + remove.close(); playback.close(); await microtasks(); clock.installed.close(); @@ -1577,16 +1580,16 @@ describe("Decoder with a warmup rendition", () => { it("moves the shared clock only from the min-delay decoder", async () => { const run = await refresh(); - const received = spyOn(run.playback.sync, "received"); - const reset = spyOn(run.playback.sync, "reset"); + const received = spyOn(Track.prototype, "received"); + const reset = spyOn(Track.prototype, "reset"); try { await run.push(MIN, at(10n, 1000, true), at(10n, 1033)); await run.push(MAX, at(9n, 700, true), at(9n, 733), at(10n, 1000, true), at(10n, 1033)); expect(received.mock.calls.map(([timestamp]) => timestamp)).toEqual([Milli(1000), Milli(1033)]); expect(run.current()).toEqual([1033, MAX]); - // The publisher restarts its timeline. The min-delay decoder resets the clock and shows the - // new timeline; the max-delay picture still on the old one never wins. + // The publisher restarts its timeline. The min-delay decoder restarts video's timeline on the + // clock and shows the new one; the max-delay picture still on the old one never wins. await run.push(MIN, marker(10n, 1_066_000), at(11n, 0, true)); expect(reset).toHaveBeenCalledTimes(1); expect(run.current()).toEqual([0, MIN]); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 38503bb77bc..340c1e9d257 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -20,7 +20,7 @@ import { base64ToBytes } from "../base64"; import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { supportCacheKey } from "../support-probe"; -import type { Sync } from "../sync"; +import type { Sync, Track } from "../sync"; import { type DecoderConfig, decoderConfigKey, type PlaybackIdentity, playbackIdentity } from "./config"; import { caughtUp, renditionJitter, switchJitter } from "./playhead"; import { Recovery } from "./recovery"; @@ -111,6 +111,8 @@ export class Decoder { readonly source: Source; /** The clock that schedules playback. */ readonly sync: Sync; + // Video's hold on the clock: its target, startup span, arrivals, and paced frames. + readonly #clock: Track; readonly #out: DecoderOutput = { error: new Signal(undefined), @@ -188,7 +190,8 @@ export class Decoder { const ranges = effect.get(active.buffered); return startupSpan(ranges, timestamp); }); - this.#signals.cleanup(this.sync.register(target, startup)); + this.#clock = this.sync.track("video", { target, startup }); + this.#signals.cleanup(() => this.#clock.close()); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -256,6 +259,8 @@ export class Decoder { // Close the active track when disabled (e.g. paused or not visible). // The pending cleanup won't do this because it was already promoted to #active. this.#active.set(undefined); + // Nothing arrives until it resumes, so its arrivals stop holding the clock. + this.#clock.reset(); // Pause and visibility keep the picture while the broadcast is active. Disconnecting // the broadcast or removing every rendition releases it. if (!broadcast || !effect.get(broadcast.out.active) || (enabled && !track)) { @@ -268,6 +273,7 @@ export class Decoder { if (!active || !key) { // Going offline should clear the last rendered frame. this.#active.set(undefined); + this.#clock.reset(); this.#clearCurrentFrame(); this.#out.buffered.set([]); return; @@ -293,6 +299,7 @@ export class Decoder { // Start a new pending effect. let pending: Rendition | undefined = new Rendition({ sync: this.sync, + clock: this.#clock, broadcast: active, track, name: Catalog.track(track, identity), @@ -474,6 +481,8 @@ export class Decoder { interface DecoderTrackProps { sync: Sync; + // Video's hold on the clock, shared by every rendition: their frames are one timeline. + clock: Track; // Set for the max-delay decoder of a rendition with a catalog `warmup`: its start and its // lateness budget reach this much further, and it leaves the shared clock and stats alone. warmup?: Getter; @@ -504,6 +513,7 @@ interface DecoderTrackProps { class DecoderTrack { error = new Signal(undefined); sync: Sync; + clock: Track; broadcast: Moq.Broadcast.Consumer; track: string; name: string; @@ -574,6 +584,7 @@ class DecoderTrack { constructor(props: DecoderTrackProps) { this.sync = props.sync; + this.clock = props.clock; this.broadcast = props.broadcast; this.track = props.track; this.name = props.name; @@ -608,7 +619,7 @@ class DecoderTrack { #receive(frame: Container.ReceivedFrame, accepted: boolean): void { if (!this.#drives) return; const timestamp = Milli.fromMicro(frame.timestamp as Micro); - const late = accepted && this.sync.received(timestamp, frame.arrival); + const late = accepted && this.clock.received(timestamp, frame.arrival); const bytes = frame.payload.byteLength; const arrived = arrivedAt(frame.arrival); this.tally?.arrive(this.key, timestamp, arrived); @@ -676,14 +687,14 @@ class DecoderTrack { // Render something while we wait for the sync to catch up, unless the frame is // already off the timeline: WebCodecs can hold a picture past a reset or a skip. // Only the min-delay decoder posts one: the max-delay picture shows on the clock. - if (this.#drives && this.frame.peek() === undefined && this.sync.presentable(timestamp)) { + if (this.#drives && this.frame.peek() === undefined && this.clock.presentable(timestamp)) { this.group = group; this.frame.set(frame.clone()); } // Resolves at once when the delay is "instant". Closing the track releases it. // Check the captured signal again in case cancellation followed the resolved wait. - if (!(await this.sync.wait(timestamp, signal)) || signal.aborted) return; + if (!(await this.clock.wait(timestamp, signal)) || signal.aborted) return; // The max-delay decoder restarts its timeline after the clock did, so the clock's own // epoch cannot tell its leftover pictures apart. if (this.#generation !== generation) return; @@ -903,7 +914,7 @@ class DecoderTrack { try { // Wait on the first picture, which is due the moment startup releases, so decoding // resumes with the whole floor still ahead instead of once it has played out. - await effect.race(refill, this.sync.wait(first, waiting.abort)); + await effect.race(refill, this.clock.wait(first, waiting.abort)); } finally { waiting.close(); } @@ -930,8 +941,9 @@ class DecoderTrack { this.#buffered.set([]); this.#generation++; this.start.set(group); - // The min-delay decoder restarts the shared clock. The max-delay one reads the same end later. - if (this.#drives) this.sync.reset(); + // The min-delay decoder restarts video's timeline, leaving audio's. The max-delay one reads the + // same end later. + if (this.#drives) this.clock.reset(); this.#resetPending = true; } From 115d99a55ae393072af62632fa52703bbea03005 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 23:49:07 -0800 Subject: [PATCH 3/9] feat(watch): track handles, audio clock tests, and finish the A/V clock quest Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 7 + js/msf/src/catalog.test.ts | 8 + js/watch/src/audio/buffer.ts | 2 +- js/watch/src/audio/decoder.test.ts | 47 +++++- js/watch/src/msf.test.ts | 10 +- js/watch/src/sync.test.ts | 216 ++++++++++++++++++++++++- js/watch/src/sync.ts | 27 ++-- js/watch/src/text/renderer.test.ts | 35 ++++ quest/m1/README.md | 1 - quest/m1/api-audit/README.md | 2 +- quest/m1/audio-jitter-target/README.md | 4 +- quest/m1/audio-jitter-target/watch.md | 4 - quest/m1/av-clock.md | 72 --------- quest/m2/audio-bufferbloat-av.md | 1 - quest/m2/moqsrc-av-sync.md | 1 - quest/m2/plan-watch-worker.md | 1 - quest/m2/watch-data-sync.md | 1 - quest/m2/watch-decode-gate.md | 4 - quest/m2/watch-worker.md | 1 - quest/m2/webcodecs-priming.md | 4 +- test/video/src/captions.ts | 5 +- 21 files changed, 347 insertions(+), 106 deletions(-) delete mode 100644 quest/m1/av-clock.md diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 12b7250b591..b753815114e 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -49,6 +49,13 @@ suspends the element until corrected or removed. The The overlay adds play/pause, volume, fullscreen, a quality selector, a buffering indicator, and a stats panel. +While audio plays, it is the clock: video and captions present against the +sample reaching the speakers, output latency included, so a re-buffering or +skipping audio ring cannot drift away from the picture. When the audio runs dry +mid-stream, the picture holds with it. Muted, paused, or without audio, playback +paces on the local clock, continuing from where audio left it. A video +discontinuity restarts only video's timeline. + `el.audio.out.error` and `el.video.out.error` report a fatal decode error. The failing track closes its subscription and stops decoding. Select another source or rendition to clear the failure; the player does not resubscribe to diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index 426192a9f52..bbffe319a7a 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -239,6 +239,14 @@ test("preserves delay through decode and encode", () => { expect(wireTracks[0].delay).toBe(200); }); +test("rejects a negative delay", () => { + const raw = encodeJson({ + version: "draft-01", + tracks: [{ name: "video0", packaging: "loc", isLive: true, role: "video", codec: "avc1.640028", delay: -1 }], + }); + expect(() => decode(raw)).toThrow(); +}); + test("carries extension root members through decode and encode", () => { // An extension section (here `mpegts`) rides the catalog root untouched, and a // section this build has never heard of survives the same way. diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 7207827b015..0005e5f6c9c 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -359,7 +359,7 @@ export class AudioBuffer { // Nothing has reached the output yet, so there is no mapping to report. if (!output.performanceTime || output.contextTime === undefined) return; const behind = Milli.fromSecond((data.time - output.contextTime) as Second); - this.#playhead.set({ rolling: true, timestamp: Milli(timestamp - behind), wall: Milli(output.performanceTime) }); + this.#playhead.set({ rolling: true, timestamp: timestamp - behind, wall: Milli(output.performanceTime) }); } // The floor as the worklet takes it. diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index cdf8a45bc4c..b4b477879ab 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1292,6 +1292,48 @@ describe("Decoder across a rename", () => { }); }); +it("drives the clock from the output while it plays, holds it at a re-stall, and hands it back when muted", async () => { + const clock = new Manual(); + clock.jump(Nano.fromMilli(Milli(5000))); + const installed = clock.install(); + const playback = await play(Milli(100)); + const quantum = () => [new Float32Array(960), new Float32Array(960)]; + try { + await playback.play(10); + const ring = playback.rings()[0]; + const context = playback.context.peek() as unknown as FakeContext; + if (!ring?.ring || !context) throw new Error("missing audio graph"); + + // Context time 1 s reaches the output now, and the next sample renders 30 ms after it. + context.output = { contextTime: 1, performanceTime: 5000 }; + ring.time = 1.03; + expect(ring.read(quantum())).toBe(960); + await microtasks(); + const heard = Milli.fromMicro(ring.ring.timestamp) - 30; + expect(playback.sync.now()).toBeCloseTo(heard, 6); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(playback.sync.now()).toBeCloseTo(heard + 10, 6); + + // The ring runs dry: the clock holds where it stopped, however long the refill takes. + while (ring.read(quantum()) > 0); + await microtasks(); + const held = playback.sync.now(); + expect(held).toBe(Milli.fromMicro(ring.ring.timestamp)); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(playback.sync.now()).toBe(held); + + // Muted, the ring only drains, so the clock runs on from where audio left it. + playback.decoding.set(false); + await microtasks(); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(playback.sync.now()).toBe(Milli((held ?? 0) + 10)); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } +}); + it("keeps audio's opening buffered while an enabled video track fills", async () => { const clock = new Manual(); const installed = clock.install(); @@ -1464,7 +1506,10 @@ it.each([100, 1000])( it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const remove = sync.track("video", { target: new Signal(Milli(100)), startup: new Signal({ start: Milli(0), end: Milli(100) }) }); + const remove = sync.track("video", { + target: new Signal(Milli(100)), + startup: new Signal({ start: Milli(0), end: Milli(100) }), + }); let reject!: (error: Error) => void; const failure = new Error("audio support probe failed"); const logged = spyOn(console, "warn").mockImplementation(() => {}); diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 3303c7564d1..97c797be4c5 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -50,7 +50,7 @@ test.each(["mp4a.40.2", "flac", "future-audio"])("keeps MSF decoder warmup absen expect(audio?.warmup).toBeUndefined(); }); -test("rounds a fractional MSF delay up, and drops zero", () => { +test("rounds a fractional MSF delay up, drops zero, and rejects a negative one", () => { const catalog: Msf.Catalog = { tracks: [ { @@ -81,6 +81,14 @@ test("rounds a fractional MSF delay up, and drops zero", () => { expect(hang.video?.renditions.video?.delay).toBe(u53(201)); expect(hang.audio?.renditions.audio?.delay).toBe(u53(1)); expect(hang.audio?.renditions.early?.delay).toBeUndefined(); + + // A rendition cannot trail the earliest by a negative amount, even between -1 and 0. + for (const delay of [-1, -0.5]) { + const negative: Msf.Catalog = { + tracks: [{ name: "audio", packaging: "loc", role: "audio", codec: "opus", delay }], + }; + expect(() => toHang(negative)).toThrow(); + } }); test("rounds a fractional MSF jitter up, keeps zero, and rejects a negative one", () => { diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 4dd2368c4b0..c822efab63f 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -350,7 +350,9 @@ describe("buffered startup", () => { audio: new Signal<{ start: Milli; end: Milli } | null>(null), video: new Signal<{ start: Milli; end: Milli } | null>(null), }; - const tracks = Object.values(spans).map((startup) => sync.track("video", { target: new Signal(Milli(100)), startup })); + const tracks = Object.values(spans).map((startup) => + sync.track("video", { target: new Signal(Milli(100)), startup }), + ); const [video] = tracks; try { await clock.advance(Nano.zero); @@ -440,3 +442,215 @@ describe("buffered startup", () => { } }); }); + +describe("audio clock", () => { + // Local time is pinned, so every playhead and deadline below is exact. + let clock: Manual; + let installed: Installed; + beforeEach(() => { + clock = new Manual(); + clock.jump(Nano.fromMilli(Milli(5000))); + installed = clock.install(); + }); + afterEach(() => installed.close()); + + const target = () => new Signal(undefined); + const local = () => Nano.toMilli(Nano(now())); + + it("follows the audio playhead while it rolls, not the earliest arrival", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + video.received(Milli(1000), Milli(5000)); + expect(sync.now()).toBe(Milli(900)); + + // The ring re-buffered and plays 40ms behind where the wall clock would be. + audio.playhead({ rolling: true, timestamp: Milli(860), wall: Milli(5000) }); + expect(sync.now()).toBe(Milli(860)); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(sync.now()).toBe(Milli(870)); + + // Video waits for the audio it accompanies, however early it arrived. + let presented: boolean | undefined; + void video.wait(Milli(900)).then((ok) => { + presented = ok; + }); + await clock.advance(Nano.fromMilli(Milli(29))); + expect(presented).toBeUndefined(); + await clock.advance(Nano.fromMilli(Milli(1))); + expect(presented).toBe(true); + + // An earlier video arrival no longer moves the playhead audio owns. + video.received(Milli(5000), local()); + expect(sync.now()).toBe(Milli(900)); + sync.close(); + }); + + it("holds the playhead through a re-stall and resumes with it", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + audio.playhead({ rolling: false, timestamp: Milli(1010) }); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(sync.now()).toBe(Milli(1010)); + + // A frame already due presents; the next waits for audio to resume. + expect(await video.wait(Milli(1010))).toBe(true); + let presented: boolean | undefined; + void video.wait(Milli(1020)).then((ok) => { + presented = ok; + }); + await clock.advance(Nano.fromMilli(Milli(500))); + expect(presented).toBeUndefined(); + audio.playhead({ rolling: true, timestamp: Milli(1015), wall: local() }); + await clock.advance(Nano.fromMilli(Milli(5))); + expect(presented).toBe(true); + sync.close(); + }); + + it("falls back to the wall clock where audio left it when audio stops", async () => { + for (const stalled of [false, true]) { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + video.received(Milli(1000), Milli(5000)); + audio.playhead({ rolling: true, timestamp: Milli(800), wall: local() }); + if (stalled) audio.playhead({ rolling: false, timestamp: Milli(800) }); + await clock.advance(Nano.fromMilli(Milli(50))); + const before = sync.now(); + + // Muted: no jump back to video's arrival anchor, and time runs on. + audio.playhead(undefined); + expect(sync.now()).toBe(before); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(sync.now()).toBe(Milli((before ?? 0) + 10)); + + // Video's own arrivals hold the clock now, so ending its timeline re-anchors it. + video.reset(); + expect(sync.now()).toBeUndefined(); + video.received(Milli(3000), local()); + expect(sync.now()).toBe(Milli(2900)); + sync.close(); + } + }); + + it("a buffered ring that drains hands the clock back instead of holding video", async () => { + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const audio = sync.track("audio", { target: target() }); + await flush(); + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + audio.playhead({ rolling: false, timestamp: Milli(1000) }); + await clock.advance(Nano.fromMilli(Milli(20))); + expect(sync.now()).toBe(Milli(1020)); + sync.close(); + }); + + it("a video-only discontinuity leaves audio's timeline alone", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + audio.received(Milli(1000), Milli(5000)); + video.received(Milli(1000), Milli(5000)); + audio.playhead({ rolling: true, timestamp: Milli(900), wall: local() }); + + // Video's parked frame belongs to the timeline that ended; audio keeps playing on its own. + const parked = video.wait(Milli(950)); + video.reset(); + expect(await parked).toBe(false); + expect(sync.now()).toBe(Milli(900)); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(sync.now()).toBe(Milli(910)); + + // The same holds before audio rolls: audio's arrivals keep the reference. + const idle = new Sync({ delay: Milli(100) }); + const quiet = idle.track("audio", { target: target() }); + const picture = idle.track("video", { target: target() }); + await flush(); + quiet.received(Milli(1000), Milli(5000)); + picture.received(Milli(990), Milli(5000)); + const anchored = idle.out.reference.peek(); + picture.reset(); + expect(idle.out.reference.peek()).toBe(anchored); + sync.close(); + idle.close(); + }); + + it("releases the reference of the earliest track when it leaves", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + // Video trails audio by 300ms, so audio anchors the clock and video is late against it. + audio.received(Milli(1000), Milli(5000)); + expect(video.received(Milli(1000), Milli(5300))).toBe(true); + + // Once audio stops arriving, video plays against its own arrivals instead. + audio.reset(); + expect(sync.out.reference.peek()).toBe(4300); + expect(video.received(Milli(1033), Milli(5333))).toBe(false); + + // Closing the last track leaves nothing anchored. + video.close(); + expect(sync.now()).toBeUndefined(); + sync.close(); + }); + + it("keeps the playhead still when the delay moves under it", async () => { + const delay = new Signal(Milli(100)); + const sync = new Sync({ delay }); + const audio = sync.track("audio", { target: target() }); + await flush(); + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + delay.set(Milli(300)); + await flush(); + expect(sync.now()).toBe(Milli(1000)); + sync.close(); + }); + + it("drops parked frames and stops following audio at a reset", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + const parked = video.wait(Milli(1500)); + sync.reset(); + expect(await parked).toBe(false); + expect(sync.now()).toBeUndefined(); + + // A stall reported after the reset is not a re-stall of the old timeline. + audio.playhead({ rolling: false, timestamp: Milli(1000) }); + video.received(Milli(3000), local()); + expect(sync.now()).toBe(Milli(2900)); + sync.close(); + }); + + it("reads zero until media time zero's turn, from either source", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + video.received(Milli(0), local()); + expect(sync.now()).toBe(Milli.zero); + // Output latency still covers the first sample. + audio.playhead({ rolling: true, timestamp: -30, wall: local() }); + expect(sync.now()).toBe(Milli.zero); + await clock.advance(Nano.fromMilli(Milli(40))); + expect(sync.now()).toBe(Milli(10)); + sync.close(); + }); + + it("allows one audio track at a time", () => { + const sync = new Sync(); + const audio = sync.track("audio", { target: target() }); + expect(() => sync.track("audio", { target: target() })).toThrow(); + audio.close(); + sync.track("audio", { target: target() }).close(); + sync.close(); + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 2a6fdf49f3f..71cfa4118f0 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -78,12 +78,13 @@ export type TrackProps = { }; /** - * Where audio output stands, as the render worklet last reported it: rolling, the sample at - * `timestamp` reaches the output at local time `wall` (on the `@moq/clock` timeline); stalled - * mid-playback, the output holds at `timestamp`. + * Where audio output stands, as the render worklet last reported it: rolling, the sample at media + * time `timestamp` ms (negative while output latency still covers the first sample) reaches the + * output at local time `wall` on the `@moq/clock` timeline; stalled mid-playback, the output holds + * at `timestamp`. * @internal */ -export type Playhead = { rolling: true; timestamp: Milli; wall: Milli } | { rolling: false; timestamp: Milli }; +export type Playhead = { rolling: true; timestamp: number; wall: Milli } | { rolling: false; timestamp: Milli }; /** What one track holds in the clock. @internal */ export type Part = { @@ -176,6 +177,9 @@ export class Clock { // and another track's arrivals must not move its timeline. if (this.driving) return late; + // A reference audio handed back belongs to whichever track arrives next. + this.owner ??= part; + const reference = this.fold(current, ref); if (reference === current) return late; this.out.reference.set(reference); @@ -212,7 +216,8 @@ export class Clock { } // Hand the clock back to the wall at the last audio-derived value, so video does not jump. The - // reference passes to another track still receiving, whose own timeline ending releases it. + // reference passes to another track still receiving, whose own timeline ending releases it, or + // else to the next track to arrive. release(): void { const held = this.paused.peek(); if (held !== undefined) this.out.reference.set(wall() - held - this.out.delay.peek()); @@ -230,8 +235,7 @@ export class Clock { if (this.audio !== part) return; this.release(); } - // A reference nobody holds, set before any arrival, is released by whichever track ends. - if (this.owner !== undefined && this.owner !== part) return; + if (this.owner !== part) return; // The earliest remaining track holds the reference, else the next frame anchors a new one. let earliest: Part | undefined; @@ -275,7 +279,8 @@ export class Clock { if (!this.playing.peek()) return undefined; const reference = this.out.reference.peek(); if (reference === undefined) return undefined; - return this.paused.peek() ?? Milli(wall() - reference - this.out.delay.peek()); + // Media time is nonnegative, so a playhead before the first sample's turn reads as zero. + return this.paused.peek() ?? Milli(Math.max(0, wall() - reference - this.out.delay.peek())); } async wait(part: Part | undefined, timestamp: Milli, signal?: AbortSignal): Promise { @@ -453,7 +458,9 @@ export class Sync { * @internal */ track(kind: "audio", props: TrackProps): AudioTrack; + /** Open a track's hold on the clock. @internal */ track(kind: "video" | "text", props: TrackProps): Track; + /** Open a track's hold on the clock. @internal */ track(kind: TrackKind, props: TrackProps): Track { return kind === "audio" ? new AudioTrack(this.#clock, props) : new Track(this.#clock, props); } @@ -544,8 +551,8 @@ export class Sync { /** * The PTS that should be rendering right now: the audio playhead while audio plays, else derived - * from the reference and delay; negative before the first frame's turn. Undefined before ingress - * or while buffered startup is filling. + * from the reference and delay; zero until media time zero's turn. Undefined before ingress or + * while buffered startup is filling. */ now(): Milli | undefined { return this.#clock.now(); diff --git a/js/watch/src/text/renderer.test.ts b/js/watch/src/text/renderer.test.ts index c54a8164989..1ca31f3b97a 100644 --- a/js/watch/src/text/renderer.test.ts +++ b/js/watch/src/text/renderer.test.ts @@ -1,5 +1,9 @@ import { beforeAll, describe, expect, test } from "bun:test"; +import { Milli, yieldNow } from "@moq/clock"; +import type * as Catalog from "@moq/hang/catalog"; +import { Signal } from "@moq/signals"; import { parseText, VTTCue } from "media-captions"; +import { Sync } from "../sync"; import { type CueStore, clearCue, @@ -7,9 +11,11 @@ import { insertCues, MAX_RETAINED, pruneCues, + Renderer, rollUp, sanitizeCue, } from "./renderer"; +import type { Source } from "./source"; function store(cues: VTTCue[] = []): CueStore { return { cues, clears: [] }; @@ -285,3 +291,32 @@ describe("sanitizeCue", () => { expect(html(sanitizeCue("[BUNNY SNORES]"))).toBe("[BUNNY SNORES]"); }); }); + +describe("Renderer", () => { + test("holds the selected rendition's catalog delay in the shared clock while enabled", async () => { + const config = new Signal({ delay: 300 } as Catalog.TextConfig); + const source = { + in: { broadcast: new Signal(undefined) }, + out: { config, track: new Signal(undefined) }, + } as unknown as Source; + const enabled = new Signal(true); + const sync = new Sync(); + const renderer = new Renderer({ source, sync, enabled }); + try { + await yieldNow(); + expect(sync.out.delay.peek()).toBe(Milli(300)); + + // Captions that are off hold nothing. + enabled.set(false); + await yieldNow(); + expect(sync.out.delay.peek()).toBe(Milli.zero); + enabled.set(true); + config.set(undefined); + await yieldNow(); + expect(sync.out.delay.peek()).toBe(Milli.zero); + } finally { + renderer.close(); + sync.close(); + } + }); +}); diff --git a/quest/m1/README.md b/quest/m1/README.md index 693e98413e6..32e65bdde37 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -159,7 +159,6 @@ their end report, so an embedder's drain alert pages on real cuts. - [Sockets](/quest/m1/socket/README.md) - a generic dial/accept `Socket` in moq-net, Rust, JS, and Python, with tracks as streams; moq-boy and CARLA move onto it - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; time-stretched browser playback without steady-state underruns and a native trace replay remain -- [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor - [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - an unmute is heard from its first frame with no catalog round trip: mute moves from holding a disabled rendition to suppressed silence with comfort noise, after a route spike; capture delivers frames on a steady cadence diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index c02205b885e..4656fc70329 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -12,7 +12,7 @@ Non-goals: redesigning independently scoped surfaces ([subscribe ranges](/quest/m1/subscribe-ranges/README.md), [archives](/doc/concept/archives.md), [compressed tracks](/doc/concept/compression.md), -[A/V clock](/quest/m1/av-clock.md)), and inherited debt that a later quest +and `Sync`'s per-track clock handles), and inherited debt that a later quest already reshapes. ## Plan diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index d2d108b062b..61cc75bca85 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -13,8 +13,8 @@ the same arrival trace. Boundaries: converging by playing slightly faster or slower is the browser's [Watch](/quest/m1/audio-jitter-target/watch.md) quest. No packet loss concealment. -Video keeps its own target; making the audio playhead the clock is [A/V -clock](/quest/m1/av-clock.md). +Video keeps its own target; the audio playhead already drives `Sync` while +audio plays. ## Plan diff --git a/quest/m1/audio-jitter-target/watch.md b/quest/m1/audio-jitter-target/watch.md index 74f5dc24c53..d192dd83b1b 100644 --- a/quest/m1/audio-jitter-target/watch.md +++ b/quest/m1/audio-jitter-target/watch.md @@ -159,10 +159,6 @@ A/V clock work. panel open for two minutes, and record its audio Underruns after 30 s and the latency tab's auto delay. -## Required - -- [A/V clock](/quest/m1/av-clock.md) - the clock the stretch converges toward - ## Related - [Capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) - re-measures the AAC browser budgets after the publisher change, no gate on this proof diff --git a/quest/m1/av-clock.md b/quest/m1/av-clock.md deleted file mode 100644 index 7a7726ec1fd..00000000000 --- a/quest/m1/av-clock.md +++ /dev/null @@ -1,72 +0,0 @@ -# [M] The audio playhead drives Sync.reference while audio plays - -## Goal - -`js/watch` keeps audio and video in sync from one clock. While audio plays, -`Sync.reference` derives from the audio playhead; muted or video-only playback -falls back to the wall clock. Today the ring is depth-driven and free-runs on -the AudioContext clock while video paces against a wall clock anchored at the -earliest arrival, so a ring that re-buffers or skips drifts against video until -the next re-anchor. - -Acceptance includes the 2026-10-07 audit's skip case: a video-only latency -skip (`js/watch/src/video/decoder.ts`, `sync.reset()` on a video -discontinuity) must not move audio's timeline. Today it re-anchors the shared -`Sync` while the audio ring stays on the old one. Folded in here rather than -fixed separately, since audio driving the clock removes the shared anchor. - -## Plan - -Moved from m0 to m1 in the 2026-09-30 audit: it waits on the whole jitter -line and is a published `@moq/watch` break, so it is not in flight. - -Settled: per-track handles, and this quest lands them. `sync.track("audio")` -and `sync.track("video")` each report their advertised delay and measured -spread, and one is nominated as the clock source. `SyncInput` -(`js/watch/src/sync.ts`, today `delay` and `buffer`) breaks once, -and a third track joins without another pair of inputs; `Sync.register` -already keeps one jitter entry per track and grows into the handles. The -measured target per track comes from the [Audio jitter -target](/quest/m1/audio-jitter-target/README.md) line: each decoder registers -its own target through `Sync.register`. This quest folds the registrations -into the handles. `SyncInput` is a published `@moq/watch` shape, so this is a -break. The decoder-facing calls (`received`, `wait`, `register`) are already -internal, with `wait` resolving whether to present or drop, so the handles build -on that. - -Recommendations for the implementation: - -- Playhead source. The worklet's ring (`js/watch/src/audio/ring-buffer.ts`) - posts an estimate and the main thread extrapolates between posts. -- Video reads a locally extrapolated clock, re-synced once per audio quantum, - so the per-frame `sync.wait()` (`js/watch/src/video/decoder.ts`) never - crosses a thread. -- Transitions. On mute or audio track end the reference falls back to the - wall clock at the last audio-derived value, so video does not jump. A ring - re-stall reads as the playhead pausing, and the reference pauses with it. -- Reset coupling stays: `Player.reset()` (`js/watch/src/player.ts`) already - flushes the audio ring alongside `sync.reset()`. -- The text renderer is the third track: it reads `sync.now()` - (`js/watch/src/text/renderer.ts`) to drive the cue clock and prune cues. -- Close the player gaps [#4170](https://github.com/moq-dev/moq/pull/4170) - left, since the handles own them. `Sync.received` only ever lowers its - reference, so after the earliest subscribed track leaves, playback stays - anchored to it; a track's handle going away must release its part of the - reference, the same expiry the jitter target's arrival minimum has. Text - renditions register no floor with `Sync`, so their catalog `delay` is - ignored; the text handle registers one like audio and video. The MSF - catalog schema (`js/msf/src/catalog.ts`) accepts a negative `delay` and - folds it into absent; refuse it on decode instead. - -Decided 2026-10-27 (maintainer): this waits only on the estimator's browser -budgets being re-measured, not the whole -[audio jitter target](/quest/m1/audio-jitter-target/README.md) epic, whose -remaining manual browser proof does not gate reshaping the clock. -[moq-dev/moq#7898](https://github.com/moq-dev/moq/pull/7898) re-measured every -browser row. What [capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) -still owes, the AAC rows after a publisher change, does not gate it either. - -## Related - -- [Watch](/quest/m1/audio-jitter-target/watch.md) - time-stretches the audio ring toward the clock this produces -- [Watch worker](/quest/m2/watch-worker.md) - moves `Sync` into a worker afterwards; keep the handles free of main-thread assumptions diff --git a/quest/m2/audio-bufferbloat-av.md b/quest/m2/audio-bufferbloat-av.md index daf7929ad38..f939068fda4 100644 --- a/quest/m2/audio-bufferbloat-av.md +++ b/quest/m2/audio-bufferbloat-av.md @@ -55,5 +55,4 @@ Public API: none. Wire: none. ## Related - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the estimator whose budgets these rows extend -- [A/V clock](/quest/m1/av-clock.md) - audio driving the clock should make these rows easier, not harder - [ViSQOL grading](/quest/m2/audio-visqol.md) - adds a quality score to the same harness diff --git a/quest/m2/moqsrc-av-sync.md b/quest/m2/moqsrc-av-sync.md index 6c19324b67c..ced9ec5e39b 100644 --- a/quest/m2/moqsrc-av-sync.md +++ b/quest/m2/moqsrc-av-sync.md @@ -26,4 +26,3 @@ Public API: none. Wire: none. ## Related - [moqsrc reconnect](/doc/bin/gstreamer.md#moqsrc) - redials and resumes on the same pads, a natural second scenario for this check -- [A/V clock](/quest/m1/av-clock.md) - the browser side of A/V sync diff --git a/quest/m2/plan-watch-worker.md b/quest/m2/plan-watch-worker.md index cfb378c446c..b11b2aaaff9 100644 --- a/quest/m2/plan-watch-worker.md +++ b/quest/m2/plan-watch-worker.md @@ -72,4 +72,3 @@ follow-up wires the capture worklet to the encoder worker with a ## Related - [Watch worker](/quest/m2/watch-worker.md) - the implementation this rewrites -- [A/V clock](/quest/m1/av-clock.md) - the `Sync` shape the implementation moves; the prototype can pace against today's diff --git a/quest/m2/watch-data-sync.md b/quest/m2/watch-data-sync.md index 5921c62196e..c5f207c6953 100644 --- a/quest/m2/watch-data-sync.md +++ b/quest/m2/watch-data-sync.md @@ -28,5 +28,4 @@ releases it. ## Required -- [A/V clock](/quest/m1/av-clock.md) - the playhead the reader releases against - [Watch worker](/quest/m2/watch-worker.md) - moves `Sync` into a worker, where the reader registers diff --git a/quest/m2/watch-decode-gate.md b/quest/m2/watch-decode-gate.md index ce3d8012d58..3fdb50d2d5d 100644 --- a/quest/m2/watch-decode-gate.md +++ b/quest/m2/watch-decode-gate.md @@ -26,7 +26,3 @@ Public API: none. Wire: none. ## Required - [Watch worker](/quest/m2/watch-worker.md) - moves decode and `Sync` into a worker - -## Related - -- [A/V clock](/quest/m1/av-clock.md) - changes what `Sync` waits on diff --git a/quest/m2/watch-worker.md b/quest/m2/watch-worker.md index 1b81e2469fb..08e74f12a6b 100644 --- a/quest/m2/watch-worker.md +++ b/quest/m2/watch-worker.md @@ -31,7 +31,6 @@ Gate on the plan's jank harness and N-player sweep, both nightly. ## Required - [Plan: watch worker](/quest/m2/plan-watch-worker.md) - picks the worker model and rewrites this quest -- [A/V clock](/quest/m1/av-clock.md) - reshapes `Sync` and the worklet playhead, so the move to the worker happens once ## Related diff --git a/quest/m2/webcodecs-priming.md b/quest/m2/webcodecs-priming.md index f2f50a709ca..e732e2c5692 100644 --- a/quest/m2/webcodecs-priming.md +++ b/quest/m2/webcodecs-priming.md @@ -17,8 +17,8 @@ gap ([w3c/webcodecs#626](https://github.com/w3c/webcodecs/issues/626)). Out of scope: Rust, where `moq-audio` already declares Opus lookahead as pre-skip and folds the AAC delay into timestamps (`folded_delay` in -`rs/moq-audio/src/encode/encoder.rs`); and playback-side sync, which -[A/V clock](/quest/m1/av-clock.md) owns. +`rs/moq-audio/src/encode/encoder.rs`); and playback-side sync, where +the audio playhead drives `Sync`. ## Plan diff --git a/test/video/src/captions.ts b/test/video/src/captions.ts index df6cbc31935..cdae42dcefd 100644 --- a/test/video/src/captions.ts +++ b/test/video/src/captions.ts @@ -30,7 +30,9 @@ export function captions() { const target = new Signal(undefined); const source = new Source({ broadcast, target }); const sync = new Sync({ delay: Milli.zero }); - sync.received(Milli(1000)); + // Stands in for the media that anchors the clock the cues show against. + const media = sync.track("video", { target: new Signal(undefined) }); + media.received(Milli(1000)); const container = document.createElement("div"); container.id = "captions"; container.style.cssText = "position:relative;width:640px;height:360px"; @@ -44,6 +46,7 @@ export function captions() { renderer.close(); source.close(); broadcast.close(); + media.close(); sync.close(); producer.close(); origin.close(); From 14f5bd40bbb1e925268518129a118b811b4dec5a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:01:29 -0800 Subject: [PATCH 4/9] fix(watch): release the audio clock at a declared end, hold re-stalls at the output, and forget wall-clock skips on takeover Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 39 ++++++++++++++++++++++++--- js/watch/src/audio/decoder.ts | 15 +++++++++-- js/watch/src/sync.test.ts | 43 ++++++++++++++++++++++++++++++ js/watch/src/sync.ts | 40 ++++++++++++++++++++------- 4 files changed, 121 insertions(+), 16 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index b4b477879ab..747914eab38 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1314,11 +1314,12 @@ it("drives the clock from the output while it plays, holds it at a re-stall, and await clock.advance(Nano.fromMilli(Milli(10))); expect(playback.sync.now()).toBeCloseTo(heard + 10, 6); - // The ring runs dry: the clock holds where it stopped, however long the refill takes. + // The ring runs dry: the clock runs on until the output reaches where it stopped, then holds + // there however long the refill takes. while (ring.read(quantum()) > 0); await microtasks(); - const held = playback.sync.now(); - expect(held).toBe(Milli.fromMicro(ring.ring.timestamp)); + const held = Milli.fromMicro(ring.ring.timestamp); + expect(playback.sync.now()).toBeLessThan(held); await clock.advance(Nano.fromMilli(Milli(500))); expect(playback.sync.now()).toBe(held); @@ -1326,7 +1327,37 @@ it("drives the clock from the output while it plays, holds it at a re-stall, and playback.decoding.set(false); await microtasks(); await clock.advance(Nano.fromMilli(Milli(10))); - expect(playback.sync.now()).toBe(Milli((held ?? 0) + 10)); + expect(playback.sync.now()).toBe(Milli(held + 10)); + } finally { + playback.close(); + await microtasks(); + installed.close(); + } +}); + +it("hands the clock back once the track declares its end, so video does not wait on the drained ring", async () => { + const clock = new Manual(); + clock.jump(Nano.fromMilli(Milli(5000))); + const installed = clock.install(); + const playback = await play(Milli(100)); + const quantum = () => [new Float32Array(960), new Float32Array(960)]; + try { + await playback.play(10); + const ring = playback.rings()[0]; + const context = playback.context.peek() as unknown as FakeContext; + if (!ring?.ring || !context) throw new Error("missing audio graph"); + context.output = { contextTime: 1, performanceTime: 5000 }; + ring.time = 1; + expect(ring.read(quantum())).toBe(960); + await microtasks(); + + playback.push({ kind: "end", group: 0n, end: Micro(frameTimestamp + 20_000) }); + await microtasks(); + while (ring.read(quantum()) > 0); + await microtasks(); + const drained = playback.sync.now() ?? 0; + await clock.advance(Nano.fromMilli(Milli(500))); + expect(playback.sync.now()).toBe(Milli(drained + 500)); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 318ef1b23df..0a61203c384 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -160,6 +160,10 @@ export class Decoder { // Ordered epoch and endpoint state from the container consumer. #terminal = new Terminal(); + // The subscription declared its end or finished, so a drained ring is the end of the audio, not a + // stall the clock should wait out. A later frame clears it. + #ended = new Signal(false); + // The container consumer's arrival estimate, unset while nothing is subscribed. #measured = new Signal(undefined); @@ -402,13 +406,14 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); - // Drive the clock while this rendition plays. Muted, disabled, or gone, it hands the clock - // back to the wall where it stands, instead of holding video on a ring that only drains. + // Drive the clock while this rendition plays. Muted, disabled, ended, or gone, it hands the + // clock back to the wall where it stands, instead of holding video on a ring that only drains. effect.run((inner) => { const config = inner.get(this.source.out.config); const live = inner.get(this.in.enabled) && !inner.get(this.sync.out.instant) && + !inner.get(this.#ended) && config !== undefined && config.enabled !== false; this.#track.playhead(live ? inner.get(ring.playhead) : undefined); @@ -571,6 +576,7 @@ export class Decoder { const preSkip = config.codec === "opus" && config.description ? Util.Opus.preSkip(Util.Hex.toBytes(config.description)) : 0; this.#terminal.clear(preSkip, Micro.fromMilli(Milli(config.warmup ?? 0))); + this.#ended.set(false); const format = config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. @@ -619,6 +625,7 @@ export class Decoder { if (next?.kind !== "frame") { // The track ending, or a gap or end, cuts a held head short. if (!(await play(order.flush()))) return; + if (!next || next.kind === "end") this.#ended.set(true); if (!next) break; if (this.#onRead(next)) { this.#reprime(decoder, decoderConfig, order); @@ -630,6 +637,7 @@ export class Decoder { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Milli.fromMicro(frame.timestamp as Micro); + this.#ended.set(false); const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); @@ -659,6 +667,7 @@ export class Decoder { const opusDescription = config.description ? Util.Hex.toBytes(config.description) : init.description; const preSkip = config.codec === "opus" && opusDescription ? Util.Opus.preSkip(opusDescription) : 0; this.#terminal.clear(preSkip, Micro.fromMilli(Milli(config.warmup ?? 0))); + this.#ended.set(false); // Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted. // The dOps box from the init segment is not a valid OGG Identification Header. const description = @@ -706,6 +715,7 @@ export class Decoder { if (next?.kind !== "frame") { // The track ending, or a gap or end, cuts a held head short. if (!(await play(order.flush()))) return; + if (!next || next.kind === "end") this.#ended.set(true); if (!next) break; if (this.#onRead(next)) { this.#reprime(decoder, decoderConfig, order); @@ -716,6 +726,7 @@ export class Decoder { const { frame } = next; const timestamp = Milli.fromMicro(frame.timestamp); + this.#ended.set(false); const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index c822efab63f..4864e71b7f6 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -511,6 +511,49 @@ describe("audio clock", () => { sync.close(); }); + it("holds a re-stall where the output stops, not where the ring did", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + await flush(); + // 30ms of output latency: the ring has read to 1030 while the output plays 1000. + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + audio.playhead({ rolling: false, timestamp: Milli(1030) }); + expect(sync.now()).toBe(Milli(1000)); + await clock.advance(Nano.fromMilli(Milli(20))); + expect(sync.now()).toBe(Milli(1020)); + await clock.advance(Nano.fromMilli(Milli(100))); + expect(sync.now()).toBe(Milli(1030)); + + // Refilled, the output plays the latency's silence first: the clock never steps back. + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + expect(sync.now()).toBe(Milli(1030)); + await clock.advance(Nano.fromMilli(Milli(40))); + expect(sync.now()).toBe(Milli(1040)); + audio.playhead({ rolling: true, timestamp: Milli(1040), wall: local() }); + await clock.advance(Nano.fromMilli(Milli(10))); + expect(sync.now()).toBe(Milli(1050)); + sync.close(); + }); + + it("forgets the wall clock's skips once audio takes over", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + const video = sync.track("video", { target: target() }); + await flush(); + // A burst skips the wall-clock playhead to 900, passing over everything before it. + video.received(Milli(0), local()); + video.received(Milli(1000), local()); + expect(video.presentable(Milli(100))).toBe(false); + + // Audio plays from 0, so a frame at 100 is still ahead of it. + audio.playhead({ rolling: true, timestamp: Milli(0), wall: local() }); + expect(video.presentable(Milli(100))).toBe(true); + const waiting = video.wait(Milli(100)); + await clock.advance(Nano.fromMilli(Milli(100))); + expect(await waiting).toBe(true); + sync.close(); + }); + it("falls back to the wall clock where audio left it when audio stops", async () => { for (const stalled of [false, true]) { const sync = new Sync({ delay: Milli(100) }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 71cfa4118f0..52b2b0954a0 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -130,8 +130,11 @@ export class Clock { audio?: Part; driving = false; - // The audio playhead a re-stall holds the clock at. - readonly paused = new Signal(undefined); + // Where a re-stall stopped the audio. Stalled, the clock runs on until the output reaches it, which + // output latency delays, and holds there. Once the ring refills, the output first plays that + // latency's worth of silence, so the clock holds until the reported playhead passes it again, + // rather than stepping back. + readonly paused = new Signal<{ timestamp: Milli; stalled: boolean } | undefined>(undefined); constructor(props?: Inputs) { this.in = { @@ -198,10 +201,14 @@ export class Clock { // Re-sync the reference to the audio playhead, hold it at a re-stall, or hand it back to arrivals. clock(part: Part, playhead: Playhead | undefined): void { if (playhead?.rolling) { + // The wall clock's skips passed over frames relative to a playhead audio now replaces. + if (!this.driving) this.floor = undefined; this.driving = true; this.owner = undefined; - this.paused.set(undefined); this.out.reference.set(playhead.wall - playhead.timestamp - this.out.delay.peek()); + const paused = this.paused.peek(); + if (paused && this.rolling() >= paused.timestamp) this.paused.set(undefined); + else if (paused) this.paused.set({ timestamp: paused.timestamp, stalled: false }); return; } if (this.audio !== part || !this.driving) return; @@ -209,18 +216,30 @@ export class Clock { // audio it accompanies. Buffered playback drains between bursts by design, so its ring // stalling means the audio ended, not that it is late. if (playhead && !this.out.buffered.peek()) { - this.paused.set(playhead.timestamp); + this.paused.set({ timestamp: playhead.timestamp, stalled: true }); return; } this.release(); } + // The playhead extrapolated from the reference, before any hold or clamp. + rolling(): number { + return wall() - (this.out.reference.peek() ?? 0) - this.out.delay.peek(); + } + + // The playhead, applying a re-stall's hold. + playhead(): number { + const rolling = this.rolling(); + const paused = this.paused.peek(); + if (!paused) return rolling; + return paused.stalled ? Math.min(rolling, paused.timestamp) : Math.max(rolling, paused.timestamp); + } + // Hand the clock back to the wall at the last audio-derived value, so video does not jump. The // reference passes to another track still receiving, whose own timeline ending releases it, or // else to the next track to arrive. release(): void { - const held = this.paused.peek(); - if (held !== undefined) this.out.reference.set(wall() - held - this.out.delay.peek()); + if (this.paused.peek()) this.out.reference.set(wall() - this.playhead() - this.out.delay.peek()); this.paused.set(undefined); this.driving = false; this.owner = this.parts.peek().find((part) => part !== this.audio && part.anchor !== undefined); @@ -280,7 +299,7 @@ export class Clock { const reference = this.out.reference.peek(); if (reference === undefined) return undefined; // Media time is nonnegative, so a playhead before the first sample's turn reads as zero. - return this.paused.peek() ?? Milli(Math.max(0, wall() - reference - this.out.delay.peek())); + return Milli(Math.max(0, this.playhead())); } async wait(part: Part | undefined, timestamp: Milli, signal?: AbortSignal): Promise { @@ -306,13 +325,14 @@ export class Clock { continue; } - // A stalled audio ring holds the playhead until it plays again. + // A stalled audio ring holds the playhead where it stopped until it plays again, and a + // refilled one holds it there until its output passes it. const paused = this.paused.peek(); - if (paused !== undefined) { - if (timestamp <= paused) return true; + if (paused?.stalled && timestamp > paused.timestamp) { await this.sleep(part, undefined, signal); continue; } + if (paused && !paused.stalled && timestamp <= paused.timestamp) return true; // NOTE: This function runs in parallel for each frame. const sleep = reference + timestamp - wall() + this.out.delay.peek(); From 5b705325a9693230bbf0272230409539c8a2d0a9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 00:08:03 -0800 Subject: [PATCH 5/9] fix(watch): keep a declared audio end through its terminal packets, and follow the output through a quick refill Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 2 ++ js/watch/src/audio/decoder.ts | 6 ++--- js/watch/src/sync.test.ts | 15 ++++++++++++ js/watch/src/sync.ts | 37 +++++++++++++++++++----------- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 747914eab38..7f8b39adac0 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1352,6 +1352,8 @@ it("hands the clock back once the track declares its end, so video does not wait await microtasks(); playback.push({ kind: "end", group: 0n, end: Micro(frameTimestamp + 20_000) }); + // The stopped encoder's last packet follows its end in the same group: still the old timeline. + playback.push(frame()); await microtasks(); while (ring.read(quantum()) > 0); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 0a61203c384..7b2299c2fc4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -161,7 +161,8 @@ export class Decoder { #terminal = new Terminal(); // The subscription declared its end or finished, so a drained ring is the end of the audio, not a - // stall the clock should wait out. A later frame clears it. + // stall the clock should wait out. The terminal packets after the end belong to the same timeline; + // only a frame starting another one clears it. #ended = new Signal(false); // The container consumer's arrival estimate, unset while nothing is subscribed. @@ -637,7 +638,6 @@ export class Decoder { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Milli.fromMicro(frame.timestamp as Micro); - this.#ended.set(false); const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); @@ -726,7 +726,6 @@ export class Decoder { const { frame } = next; const timestamp = Milli.fromMicro(frame.timestamp); - this.#ended.set(false); const late = this.#track.received(timestamp, frame.arrival); this.#arrivals.write({ kind: "frame", timestamp, arrival: frame.arrival, late }); @@ -1016,6 +1015,7 @@ export class Decoder { // A hole restarts codec prediction on the same timeline, keeping decoded PCM and parked video. // A frame after a declared end starts another audio timeline, which leaves video's alone. if (read.kind === "frame") { + this.#ended.set(false); this.#track.reset(); this.reset(); } diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 4864e71b7f6..d3711e8f5bf 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -535,6 +535,21 @@ describe("audio clock", () => { sync.close(); }); + it("follows the output through a refill that lands before it reaches the stop", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track("audio", { target: target() }); + await flush(); + audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); + audio.playhead({ rolling: false, timestamp: Milli(1030) }); + await clock.advance(Nano.fromMilli(Milli(5))); + // Refilled within one quantum: no silence reaches the output, and the clock never jumps. + audio.playhead({ rolling: true, timestamp: Milli(1005), wall: local() }); + expect(sync.now()).toBe(Milli(1005)); + await clock.advance(Nano.fromMilli(Milli(40))); + expect(sync.now()).toBe(Milli(1045)); + sync.close(); + }); + it("forgets the wall clock's skips once audio takes over", async () => { const sync = new Sync({ delay: Milli(100) }); const audio = sync.track("audio", { target: target() }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 52b2b0954a0..d0dc532c9f2 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -130,11 +130,14 @@ export class Clock { audio?: Part; driving = false; - // Where a re-stall stopped the audio. Stalled, the clock runs on until the output reaches it, which - // output latency delays, and holds there. Once the ring refills, the output first plays that - // latency's worth of silence, so the clock holds until the reported playhead passes it again, - // rather than stepping back. - readonly paused = new Signal<{ timestamp: Milli; stalled: boolean } | undefined>(undefined); + // Where a re-stall stopped the ring, and the output's progress when it did. The output keeps + // playing what the ring already read until it reaches `timestamp`, which output latency delays, + // so the clock follows `from` until then, refilled or not. Past it, the clock holds while the ring + // is stalled, and after a refill until the reported playhead passes it, since the output first + // plays the silence the stall left behind rather than stepping back. + readonly paused = new Signal< + { timestamp: Milli; from: { playhead: number; wall: number }; stalled: boolean } | undefined + >(undefined); constructor(props?: Inputs) { this.in = { @@ -207,8 +210,9 @@ export class Clock { this.owner = undefined; this.out.reference.set(playhead.wall - playhead.timestamp - this.out.delay.peek()); const paused = this.paused.peek(); - if (paused && this.rolling() >= paused.timestamp) this.paused.set(undefined); - else if (paused) this.paused.set({ timestamp: paused.timestamp, stalled: false }); + if (!paused) return; + if (this.playhead() >= paused.timestamp && this.rolling() >= paused.timestamp) this.paused.set(undefined); + else this.paused.set({ ...paused, stalled: false }); return; } if (this.audio !== part || !this.driving) return; @@ -216,7 +220,11 @@ export class Clock { // audio it accompanies. Buffered playback drains between bursts by design, so its ring // stalling means the audio ended, not that it is late. if (playhead && !this.out.buffered.peek()) { - this.paused.set({ timestamp: playhead.timestamp, stalled: true }); + this.paused.set({ + timestamp: playhead.timestamp, + from: { playhead: this.playhead(), wall: wall() }, + stalled: true, + }); return; } this.release(); @@ -232,7 +240,9 @@ export class Clock { const rolling = this.rolling(); const paused = this.paused.peek(); if (!paused) return rolling; - return paused.stalled ? Math.min(rolling, paused.timestamp) : Math.max(rolling, paused.timestamp); + const before = paused.from.playhead + wall() - paused.from.wall; + if (before < paused.timestamp) return before; + return paused.stalled ? paused.timestamp : Math.max(rolling, paused.timestamp); } // Hand the clock back to the wall at the last audio-derived value, so video does not jump. The @@ -325,17 +335,16 @@ export class Clock { continue; } - // A stalled audio ring holds the playhead where it stopped until it plays again, and a - // refilled one holds it there until its output passes it. + // A stalled audio ring holds the playhead where it stopped until it plays again. const paused = this.paused.peek(); if (paused?.stalled && timestamp > paused.timestamp) { await this.sleep(part, undefined, signal); continue; } - if (paused && !paused.stalled && timestamp <= paused.timestamp) return true; - // NOTE: This function runs in parallel for each frame. - const sleep = reference + timestamp - wall() + this.out.delay.peek(); + // NOTE: This function runs in parallel for each frame. The playhead moves at most as fast + // as the local clock, so sleeping the difference never overshoots; a hold re-checks. + const sleep = timestamp - this.playhead(); // Skip the timer for small sleeps; the host's timer resolution (~4ms) would overshoot. if (sleep < 5) return true; From c82ea3fb4fd228d4b34d1a7a88f475ea94b7865e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:20:16 -0800 Subject: [PATCH 6/9] refactor(watch): keep the shared clock on local time; drop the audio-driven clock The maintainer chose to keep one slow track from holding the others: the shared reference stays on the earliest arrival, as before, and steady-state lip sync comes from the audio ring time-stretching toward it (the Watch quest). Per-track timelines, a departing track releasing its part of the reference, captions' catalog delay, and the MSF negative-delay check stay. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 8 +- doc/lib/js/watch.md | 10 +- js/watch/src/audio/buffer.test.ts | 56 +---- js/watch/src/audio/buffer.ts | 32 --- js/watch/src/audio/decoder.test.ts | 105 ++------ js/watch/src/audio/decoder.ts | 32 +-- js/watch/src/audio/render-worklet.test.ts | 32 +-- js/watch/src/audio/render-worklet.ts | 2 - js/watch/src/audio/render.ts | 4 - js/watch/src/audio/replay.ts | 2 +- js/watch/src/retention.test.ts | 4 +- js/watch/src/sync.test.ts | 287 +++++----------------- js/watch/src/sync.ts | 195 +++------------ js/watch/src/text/renderer.ts | 2 +- js/watch/src/video/decoder.test.ts | 14 +- js/watch/src/video/decoder.ts | 2 +- quest/m1/audio-jitter-target/README.md | 4 +- quest/m1/audio-jitter-target/watch.md | 10 + quest/m2/webcodecs-priming.md | 4 +- test/video/src/captions.ts | 2 +- 20 files changed, 145 insertions(+), 662 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index c613a91c54b..9c44549f6ef 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -383,10 +383,10 @@ them. The cost is over-buffering by the smallest subscribed `delay` when the broadcast's earliest rendition is not subscribed. The playout is recomputed when a rendition is subscribed, unsubscribed, or its -catalog entry rises, so dropping a slow rendition lowers latency. The shared -reference itself only ever moves earlier, so once the earliest subscribed track -leaves, the clock stays anchored to it until playback re-anchors; that errs on -the safe side. +catalog entry rises, so dropping a slow rendition lowers latency. Each track's +own arrivals only ever move its part of the shared reference earlier. A track +that leaves, or whose timeline restarts, releases its part: the earliest +remaining track's anchor takes over, or the next frame re-anchors. A track that replaces another in the same slot and on the same broadcast path, such as the new name a codec change publishes under, continues the outgoing diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index b753815114e..ab2539f5ac4 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -49,12 +49,10 @@ suspends the element until corrected or removed. The The overlay adds play/pause, volume, fullscreen, a quality selector, a buffering indicator, and a stats panel. -While audio plays, it is the clock: video and captions present against the -sample reaching the speakers, output latency included, so a re-buffering or -skipping audio ring cannot drift away from the picture. When the audio runs dry -mid-stream, the picture holds with it. Muted, paused, or without audio, playback -paces on the local clock, continuing from where audio left it. A video -discontinuity restarts only video's timeline. +Audio, video, and captions play from one clock on local time, anchored at the +earliest arrival, so a congested track falls behind on its own instead of +stalling the others. A discontinuity restarts only its own track's timeline, +and a track that stops arriving releases its hold on the clock. `el.audio.out.error` and `el.video.out.error` report a fatal decode error. The failing track closes its subscription and stops decoding. Select another diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index e1d339eb678..aacabd46e21 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -5,9 +5,6 @@ import type { Init, Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; const HOUR = Milli(60 * 60 * 1000); - -// A context whose output has not started, so no state report maps to a rolling playhead. -const silent = { getOutputTimestamp: () => ({ contextTime: 0, performanceTime: 0 }) } as unknown as AudioContext; const ms = (value: number) => Micro.fromMilli(Milli(value)); // A buffered buffer over a real port, with the worklet's end of it left to the test. @@ -19,7 +16,6 @@ function open(): { } { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { - context: silent, channels: 1, rate: 48_000, delay: Milli(100), @@ -33,9 +29,7 @@ function open(): { const msg: State = { type: "state", timestamp: (timestamp ?? 0) as Micro, - time: 0, stalled, - playing: true, anchored: timestamp !== undefined, underruns: 0, epoch, @@ -58,7 +52,6 @@ describe("AudioBuffer", () => { it("transfers PCM ownership and orders control messages after the data they change", async () => { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { - context: silent, channels: 2, rate: 48_000, delay: Milli(100), @@ -426,21 +419,20 @@ describe("Backpressure", () => { }); // A worklet whose port records what the buffer posts and delivers state reports synchronously. -function fake(delay: Milli, buffered = false, context = silent) { +function fake(delay: Milli, buffered = false) { const sent: Message[] = []; const port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => sent.push(msg), start() {}, }); const buffer = new AudioBuffer({ port } as unknown as AudioWorkletNode, { - context, channels: 1, rate: 1000, delay, buffered, }); - const state = (timestamp: Micro, stalled: boolean, epoch: number, time = 0, playing = true) => { - const data: State = { type: "state", timestamp, time, stalled, playing, anchored: true, underruns: 0, epoch }; + const state = (timestamp: Micro, stalled: boolean, epoch: number) => { + const data: State = { type: "state", timestamp, stalled, anchored: true, underruns: 0, epoch }; port.dispatchEvent(new MessageEvent("message", { data })); }; return { buffer, sent, state }; @@ -456,7 +448,6 @@ describe("AudioBuffer past its decoded floor", () => { start() {}, }); new AudioBuffer({ port } as unknown as AudioWorkletNode, { - context: silent, channels: 2, rate: 48_000, delay, @@ -547,44 +538,3 @@ describe("AudioBuffer past its decoded floor", () => { } }); }); - -describe("AudioBuffer playhead", () => { - it("maps each report to the sample reaching the output, on the current epoch only", () => { - // Context time 1 s reaches the output at local time 5000 ms. - const output = { contextTime: 0, performanceTime: 0 }; - const context = { getOutputTimestamp: () => output } as unknown as AudioContext; - const { buffer, state } = fake(Milli(100), false, context); - try { - // Nothing has reached the output yet, so there is no mapping to report. - state(ms(2000), false, 0, 1.03); - expect(buffer.playhead.peek()).toBeUndefined(); - - // The next sample, 2 s into the media, renders at context time 1.03 s: 30 ms of output - // latency behind it, the output is at 1.97 s. - Object.assign(output, { contextTime: 1, performanceTime: 5000 }); - state(ms(2000), false, 0, 1.03); - const rolling = buffer.playhead.peek(); - expect(rolling?.rolling).toBe(true); - expect(rolling?.timestamp).toBeCloseTo(1970, 6); - expect(rolling?.rolling && rolling.wall).toBe(Milli(5000)); - - // A stall holds where the ring stopped reading. - state(ms(2010), true, 0, 1.05); - expect(buffer.playhead.peek()).toEqual({ rolling: false, timestamp: Milli(2010) }); - - // A ring held for buffered startup drives nothing. - state(ms(2010), false, 0, 1.06, false); - expect(buffer.playhead.peek()).toBeUndefined(); - - // A reset flushes the timeline, and a report the worklet sent before seeing it is ignored. - state(ms(2020), false, 0, 1.07); - expect(buffer.playhead.peek()?.rolling).toBe(true); - buffer.reset(); - expect(buffer.playhead.peek()).toBeUndefined(); - state(ms(2030), false, 0, 1.08); - expect(buffer.playhead.peek()).toBeUndefined(); - } finally { - buffer.close(); - } - }); -}); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 0005e5f6c9c..1bd9e2641bf 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,6 +1,5 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Playhead } from "../sync"; import { ringHold, ringSamples } from "./latency"; import type { Data, Init, Latency, Playing, Reset, State, Truncate } from "./render"; @@ -271,12 +270,6 @@ export class AudioBuffer { /** How many times the ring has run dry mid-playback, cumulative. */ readonly underruns: Getter = this.#underruns; - readonly #playhead = new Signal(undefined); - /** Where the output stands on the current timeline, undefined until the ring anchors and is released. */ - readonly playhead: Getter = this.#playhead; - - readonly #context: AudioContext; - // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; // Bumped per reset or park. A state message posted before the worklet saw it reports the ring @@ -289,8 +282,6 @@ export class AudioBuffer { constructor( worklet: AudioWorkletNode, props: { - /** The context the worklet renders in, which maps its output time to the local clock. */ - context: AudioContext; channels: number; rate: number; /** The target delay. The ring holds up to a fixed floor of it decoded; the rest stays encoded. */ @@ -303,7 +294,6 @@ export class AudioBuffer { ) { const { channels, rate, buffered, playing } = props; this.#playing = props.playing; - this.#context = props.context; this.#worklet = worklet; this.channels = channels; this.rate = rate; @@ -334,7 +324,6 @@ export class AudioBuffer { this.#underruns.set(data.underruns); if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); - this.#observe(data); } } }); @@ -342,26 +331,6 @@ export class AudioBuffer { worklet.port.start(); } - // Map a state report to the playhead at the output. The worklet reports the context time its next - // sample renders at, and the context reports which context time is reaching the output at which - // local time, so the output latency is accounted for. - #observe(data: State): void { - if (!data.anchored || !data.playing) { - this.#playhead.set(undefined); - return; - } - const timestamp = Milli.fromMicro(data.timestamp); - if (data.stalled) { - this.#playhead.set({ rolling: false, timestamp }); - return; - } - const output = this.#context.getOutputTimestamp(); - // Nothing has reached the output yet, so there is no mapping to report. - if (!output.performanceTime || output.contextTime === undefined) return; - const behind = Milli.fromSecond((data.time - output.contextTime) as Second); - this.#playhead.set({ rolling: true, timestamp: timestamp - behind, wall: Milli(output.performanceTime) }); - } - // The floor as the worklet takes it. #latency(): Milli { return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); @@ -430,7 +399,6 @@ export class AudioBuffer { /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void { this.#epoch++; - this.#playhead.set(undefined); const msg: Reset = { type: "reset", epoch: this.#epoch, diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 7f8b39adac0..9131208046f 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -33,10 +33,6 @@ class FakeContext extends EventTarget { readonly sampleRate: number; readonly audioWorklet = { addModule: () => moduleLoaded }; readonly calls = { suspend: 0, resume: 0, close: 0 }; - // Which context time reaches the output at which local time. Nothing has until a test says so, - // so the audio playhead drives the clock only in the tests about that. - output: AudioTimestamp = { contextTime: 0, performanceTime: 0 }; - getOutputTimestamp = () => this.output; constructor(options: AudioContextOptions) { super(); this.sampleRate = options.sampleRate ?? 48_000; @@ -67,8 +63,6 @@ let worklets: FakeWorklet[] = []; // every quantum read reports the ring's state back, the way `render-worklet.ts` does. class FakeWorklet { ring?: AudioRingBuffer; - // The context time each reported playhead renders at. - time = 0; #epoch = 0; #playing = true; readonly port = Object.assign(new EventTarget(), { @@ -109,9 +103,7 @@ class FakeWorklet { const state: State = { type: "state", timestamp: ring.timestamp, - time: this.time, stalled: ring.stalled, - playing: this.#playing, anchored: ring.anchored, underruns: ring.underruns, epoch: this.#epoch, @@ -563,19 +555,27 @@ for (const cmaf of [false, true]) { } // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder - // resets for it before anything is judged out of order. + // resets for it before anything is judged out of order. Only audio's timeline restarts. it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { const playback = await play(Milli(100), false, cmaf); + const reset = spyOn(playback.sync, "reset"); + const video = playback.sync.track({ target: new Signal(undefined) }); try { frameTimestamp = 480_000; playback.push({ ...frame(), group: 0n }); await microtasks(); + video.received(Milli(500)); + const parked = video.wait(Milli(10_000)); playback.push({ kind: "end", group: 0n, end: Micro(520_000) }); frameTimestamp = 0; playback.push({ ...frame(), group: 1n }); await microtasks(); expect(playback.inserted()).toEqual([500_000, 20_000]); + expect(reset).not.toHaveBeenCalled(); + expect(Bun.peek.status(parked)).toBe("pending"); } finally { + reset.mockRestore(); + video.close(); playback.close(); await microtasks(); } @@ -1292,87 +1292,12 @@ describe("Decoder across a rename", () => { }); }); -it("drives the clock from the output while it plays, holds it at a re-stall, and hands it back when muted", async () => { - const clock = new Manual(); - clock.jump(Nano.fromMilli(Milli(5000))); - const installed = clock.install(); - const playback = await play(Milli(100)); - const quantum = () => [new Float32Array(960), new Float32Array(960)]; - try { - await playback.play(10); - const ring = playback.rings()[0]; - const context = playback.context.peek() as unknown as FakeContext; - if (!ring?.ring || !context) throw new Error("missing audio graph"); - - // Context time 1 s reaches the output now, and the next sample renders 30 ms after it. - context.output = { contextTime: 1, performanceTime: 5000 }; - ring.time = 1.03; - expect(ring.read(quantum())).toBe(960); - await microtasks(); - const heard = Milli.fromMicro(ring.ring.timestamp) - 30; - expect(playback.sync.now()).toBeCloseTo(heard, 6); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(playback.sync.now()).toBeCloseTo(heard + 10, 6); - - // The ring runs dry: the clock runs on until the output reaches where it stopped, then holds - // there however long the refill takes. - while (ring.read(quantum()) > 0); - await microtasks(); - const held = Milli.fromMicro(ring.ring.timestamp); - expect(playback.sync.now()).toBeLessThan(held); - await clock.advance(Nano.fromMilli(Milli(500))); - expect(playback.sync.now()).toBe(held); - - // Muted, the ring only drains, so the clock runs on from where audio left it. - playback.decoding.set(false); - await microtasks(); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(playback.sync.now()).toBe(Milli(held + 10)); - } finally { - playback.close(); - await microtasks(); - installed.close(); - } -}); - -it("hands the clock back once the track declares its end, so video does not wait on the drained ring", async () => { - const clock = new Manual(); - clock.jump(Nano.fromMilli(Milli(5000))); - const installed = clock.install(); - const playback = await play(Milli(100)); - const quantum = () => [new Float32Array(960), new Float32Array(960)]; - try { - await playback.play(10); - const ring = playback.rings()[0]; - const context = playback.context.peek() as unknown as FakeContext; - if (!ring?.ring || !context) throw new Error("missing audio graph"); - context.output = { contextTime: 1, performanceTime: 5000 }; - ring.time = 1; - expect(ring.read(quantum())).toBe(960); - await microtasks(); - - playback.push({ kind: "end", group: 0n, end: Micro(frameTimestamp + 20_000) }); - // The stopped encoder's last packet follows its end in the same group: still the old timeline. - playback.push(frame()); - await microtasks(); - while (ring.read(quantum()) > 0); - await microtasks(); - const drained = playback.sync.now() ?? 0; - await clock.advance(Nano.fromMilli(Milli(500))); - expect(playback.sync.now()).toBe(Milli(drained + 500)); - } finally { - playback.close(); - await microtasks(); - installed.close(); - } -}); - it("keeps audio's opening buffered while an enabled video track fills", async () => { const clock = new Manual(); const installed = clock.install(); const playback = await play(Milli(100), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("video", { target: new Signal(Milli(100)), startup: video }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: video }); try { await playback.play(8); await clock.advance(Nano.zero); @@ -1399,7 +1324,7 @@ it("starts above the decoded audio floor without expanding PCM storage", async ( const installed = clock.install(); const playback = await play(Milli(1000), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("video", { target: new Signal(Milli(1000)), startup: video }); + const remove = playback.sync.track({ target: new Signal(Milli(1000)), startup: video }); try { await playback.play(80); await clock.advance(Nano.zero); @@ -1467,7 +1392,7 @@ for (const cmaf of [false, true]) { const installed = clock.install(); const playback = await play(Milli(100), true, cmaf); const reset = spyOn(playback.sync, "reset"); - const video = playback.sync.track("video", { target: new Signal(undefined) }); + const video = playback.sync.track({ target: new Signal(undefined) }); const prime = spyOn(AudioDecoder.prototype, "reset"); try { await playback.play(7); @@ -1509,7 +1434,7 @@ it.each([100, 1000])( async (delay) => { const playback = await play(Milli(delay), true); const video = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("video", { target: new Signal(Milli(delay)), startup: video }); + const remove = playback.sync.track({ target: new Signal(Milli(delay)), startup: video }); const disconnect = spyOn(FakeWorklet.prototype, "disconnect"); const closeTrack = spyOn(Moq.Track.Subscriber.prototype, "close"); try { @@ -1539,7 +1464,7 @@ it.each([100, 1000])( it("a rejected audio support probe does not leave buffered video waiting on a pending track", async () => { const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const remove = sync.track("video", { + const remove = sync.track({ target: new Signal(Milli(100)), startup: new Signal({ start: Milli(0), end: Milli(100) }), }); @@ -1595,7 +1520,7 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C const installed = clock.install(); const playback = await play(Milli(1500), true, cmaf); const video = new Signal<{ start: Milli; end: Milli } | null>({ start: Milli(20), end: Milli(1520) }); - const remove = playback.sync.track("video", { target: new Signal(Milli(1500)), startup: video }); + const remove = playback.sync.track({ target: new Signal(Milli(1500)), startup: video }); try { await playback.play(25); // The audio after the hole counts, the hole itself does not. diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 7b2299c2fc4..bfa43cb787b 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -20,7 +20,7 @@ import { base64ToBytes } from "../base64"; import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { supportCacheKey } from "../support-probe"; -import type { AudioTrack, Sync } from "../sync"; +import type { Sync, Track } from "../sync"; import { AudioBuffer } from "./buffer"; import { type DecoderConfig, @@ -127,8 +127,8 @@ export class Decoder { readonly source: Source; /** The clock that schedules playback. */ readonly sync: Sync; - // This track's hold on the clock, which its playhead drives while it plays. - readonly #track: AudioTrack; + // Audio's hold on the clock: its target, startup span, and arrivals. + readonly #track: Track; readonly #out: DecoderOutput = { error: new Signal(undefined), @@ -160,11 +160,6 @@ export class Decoder { // Ordered epoch and endpoint state from the container consumer. #terminal = new Terminal(); - // The subscription declared its end or finished, so a drained ring is the end of the audio, not a - // stall the clock should wait out. The terminal packets after the end belong to the same timeline; - // only a frame starting another one clears it. - #ended = new Signal(false); - // The container consumer's arrival estimate, unset while nothing is subscribed. #measured = new Signal(undefined); @@ -239,7 +234,7 @@ export class Decoder { const hold = Milli.fromMicro(ringHold(effect.get(this.sync.out.delay))); return { start: span.start, end: Milli(span.end + Math.min(queued, hold)) }; }); - this.#track = this.sync.track("audio", { target: playout, startup }); + this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); @@ -388,7 +383,6 @@ export class Decoder { effect.cleanup(() => worklet.disconnect()); const ring = new AudioBuffer(worklet, { - context, channels: channelCount, rate: sampleRate, delay: this.sync.out.delay.peek(), @@ -407,19 +401,6 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); - // Drive the clock while this rendition plays. Muted, disabled, ended, or gone, it hands the - // clock back to the wall where it stands, instead of holding video on a ring that only drains. - effect.run((inner) => { - const config = inner.get(this.source.out.config); - const live = - inner.get(this.in.enabled) && - !inner.get(this.sync.out.instant) && - !inner.get(this.#ended) && - config !== undefined && - config.enabled !== false; - this.#track.playhead(live ? inner.get(ring.playhead) : undefined); - }); - effect.cleanup(() => this.#track.playhead(undefined)); // Each ring counts from zero, so add only what this one counted to the cumulative stats. let counted = 0; effect.run((inner) => { @@ -577,7 +558,6 @@ export class Decoder { const preSkip = config.codec === "opus" && config.description ? Util.Opus.preSkip(Util.Hex.toBytes(config.description)) : 0; this.#terminal.clear(preSkip, Micro.fromMilli(Milli(config.warmup ?? 0))); - this.#ended.set(false); const format = config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. @@ -626,7 +606,6 @@ export class Decoder { if (next?.kind !== "frame") { // The track ending, or a gap or end, cuts a held head short. if (!(await play(order.flush()))) return; - if (!next || next.kind === "end") this.#ended.set(true); if (!next) break; if (this.#onRead(next)) { this.#reprime(decoder, decoderConfig, order); @@ -667,7 +646,6 @@ export class Decoder { const opusDescription = config.description ? Util.Hex.toBytes(config.description) : init.description; const preSkip = config.codec === "opus" && opusDescription ? Util.Opus.preSkip(opusDescription) : 0; this.#terminal.clear(preSkip, Micro.fromMilli(Milli(config.warmup ?? 0))); - this.#ended.set(false); // Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted. // The dOps box from the init segment is not a valid OGG Identification Header. const description = @@ -715,7 +693,6 @@ export class Decoder { if (next?.kind !== "frame") { // The track ending, or a gap or end, cuts a held head short. if (!(await play(order.flush()))) return; - if (!next || next.kind === "end") this.#ended.set(true); if (!next) break; if (this.#onRead(next)) { this.#reprime(decoder, decoderConfig, order); @@ -1015,7 +992,6 @@ export class Decoder { // A hole restarts codec prediction on the same timeline, keeping decoded PCM and parked video. // A frame after a declared end starts another audio timeline, which leaves video's alone. if (read.kind === "frame") { - this.#ended.set(false); this.#track.reset(); this.reset(); } diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 7276d65f1db..b6afeadb67e 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { describe, expect, it } from "bun:test"; import { Micro, Milli } from "@moq/clock"; import type { Message, State } from "./render"; @@ -37,14 +37,6 @@ try { } } -// The worklet scope's context time, which each quantum starts at; held at zero. -const clock = Object.getOwnPropertyDescriptor(globalThis, "currentTime"); -beforeEach(() => Object.defineProperty(globalThis, "currentTime", { configurable: true, value: 0 })); -afterEach(() => { - if (clock) Object.defineProperty(globalThis, "currentTime", clock); - else Reflect.deleteProperty(globalThis, "currentTime"); -}); - function renderer() { const render = new Render(); const send = (data: Message) => render.port.onmessage?.({ data } as MessageEvent); @@ -86,16 +78,7 @@ describe("render dispatch", () => { send({ type: "data", timestamp: Micro(500_000), data: [new Float32Array(100).fill(2)] }); for (let i = 0; i < 3; i++) read(20); expect(sent).toEqual([ - { - type: "state", - epoch: 7, - timestamp: Micro(560_000), - time: 0.02, - stalled: false, - playing: true, - anchored: true, - underruns: 0, - }, + { type: "state", epoch: 7, timestamp: Micro(560_000), stalled: false, anchored: true, underruns: 0 }, ]); }); @@ -115,16 +98,7 @@ describe("render dispatch", () => { expect(read(20)[19]).toBe(1); read(20); expect(sent).toEqual([ - { - type: "state", - epoch: 3, - timestamp: Micro(60_000), - time: 0.02, - stalled: false, - playing: true, - anchored: true, - underruns: 0, - }, + { type: "state", epoch: 3, timestamp: Micro(60_000), stalled: false, anchored: true, underruns: 0 }, ]); }); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 87ffb904f36..6cafe639a57 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -85,9 +85,7 @@ class Render extends AudioWorkletProcessor { const state: State = { type: "state", timestamp: ring.timestamp, - time: currentTime + output[0].length / ring.rate, stalled: ring.stalled, - playing: this.#playing, anchored: ring.anchored, underruns: ring.underruns, epoch: this.#epoch, diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 1000dec17b7..13689ab7f76 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -49,11 +49,7 @@ export interface Latency { export interface State { type: "state"; timestamp: Micro; - // The context time, in seconds, at which the sample at `timestamp` is rendered. - time: number; stalled: boolean; - // Whether consumption is released; false holds the ring for buffered startup. - playing: boolean; // Whether `timestamp` is on the current timeline: it reads 0 until a frame anchors the ring. anchored: boolean; // How many times the ring has run dry mid-playback, cumulative. diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index f6b2d9ac7a2..d8a6766cfc9 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -150,7 +150,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato delay: config.delay !== undefined ? Milli(config.delay) : undefined, }), ); - const audio = sync.track("audio", { target: playout }); + const audio = sync.track({ target: playout }); signals.cleanup(() => audio.close()); await settle(); diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index a9879a83bfe..85a3d072822 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -59,7 +59,7 @@ async function pendingListeners(run: () => Promise): Promise { // piles it up for the life of the player. test("waits on a stable clock leave nothing behind", async () => { const sync = new Sync({ delay: Milli(10) }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); // Let the delay effect run before anchoring. await yieldNow(); video.received(Nano.toMilli(Nano(now()))); @@ -88,7 +88,7 @@ test("a long subscription through the player path leaves nothing behind", async const producer = new Container.Legacy.Producer(track, format); const sync = new Sync({ delay: Milli(10) }); - const audio = sync.track("audio", { target: new Signal(undefined) }); + const audio = sync.track({ target: new Signal(undefined) }); const effect = new Effect(); const sub = subscribeMedia(effect, { broadcast: broadcast.consume(), diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index d3711e8f5bf..7fa6736e20f 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -77,8 +77,8 @@ describe("auto delay", () => { const audio = new Signal(120 as Milli); const video = new Signal(40 as Milli); const sync = new Sync(); - const unregisterAudio = sync.track("audio", { target: audio }); - sync.track("video", { target: video }); + const unregisterAudio = sync.track({ target: audio }); + sync.track({ target: video }); await flush(); expect(sync.out.delay.peek()).toBe(120 as Milli); @@ -96,8 +96,8 @@ describe("auto delay", () => { it("unregisters duplicate targets independently", async () => { const media = new Signal(20 as Milli); const sync = new Sync(); - const unregisterFirst = sync.track("video", { target: media }); - const unregisterSecond = sync.track("video", { target: media }); + const unregisterFirst = sync.track({ target: media }); + const unregisterSecond = sync.track({ target: media }); unregisterFirst.close(); await flush(); @@ -112,7 +112,7 @@ describe("auto delay", () => { it("takes a fixed delay literally, ignoring the measured targets", async () => { const media = new Signal(250 as Milli); const sync = new Sync({ delay: 100 as Milli }); - sync.track("video", { target: media }); + sync.track({ target: media }); await flush(); expect(sync.out.delay.peek()).toBe(100 as Milli); sync.close(); @@ -133,7 +133,7 @@ describe("wait", () => { it("presents at once when constructed instant, before effects flush", async () => { const sync = new Sync({ delay: "instant" }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); video.received(nowMs()); expect(await video.wait(Milli(nowMs() + 10_000))).toBe(true); sync.close(); @@ -141,14 +141,14 @@ describe("wait", () => { it("drops an unanchored frame", async () => { const sync = new Sync({ delay: "instant" }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); expect(await video.wait(nowMs())).toBe(false); sync.close(); }); it("presents a frame once its deadline comes due", async () => { const sync = new Sync({ delay: 100 as Milli }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); video.received(timestamp); @@ -165,7 +165,7 @@ describe("wait", () => { it("wakes a sleeping wait to present when the delay switches to instant", async () => { const delay = new Signal(10_000 as Milli); const sync = new Sync({ delay }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); video.received(nowMs()); @@ -180,7 +180,7 @@ describe("wait", () => { it("drops a frame a skip passed over, before and after it parks", async () => { const sync = new Sync({ delay: 100 as Milli }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); video.received(timestamp); @@ -198,7 +198,7 @@ describe("wait", () => { it("drops a sleeping wait at a reset, even when the next anchor restores the reference", async () => { const sync = new Sync({ delay: 10_000 as Milli }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); video.received(timestamp); @@ -215,7 +215,7 @@ describe("wait", () => { it("drops a sleeping wait when its signal aborts", async () => { const sync = new Sync({ delay: 10_000 as Milli }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); video.received(timestamp); @@ -230,7 +230,7 @@ describe("wait", () => { it("drops a sleeping wait when closed", async () => { const sync = new Sync({ delay: 10_000 as Milli }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); const timestamp = nowMs(); video.received(timestamp); @@ -248,7 +248,7 @@ describe("recorded ingress", () => { clock.jump(Nano.fromMilli(Milli(10_000))); const installed = clock.install(); const sync = new Sync({ delay: Milli(100) }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); try { for (let i = 0; i < 20; i++) await Promise.resolve(); expect(video.received(Milli(0), Milli(1000))).toBe(false); @@ -275,7 +275,7 @@ describe("skip floor", () => { it("a skip passes over only frames behind the playhead it jumped to", async () => { const sync = new Sync({ delay: Milli(100) }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); video.received(Milli(1000), Milli(5000)); expect(video.presentable(Milli(0))).toBe(true); @@ -295,7 +295,7 @@ describe("skip floor", () => { it("a skip never lowers the floor and a reset clears it", async () => { const delay = new Signal(Milli(100)); const sync = new Sync({ delay }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); video.received(Milli(0), Milli(5000)); video.received(Milli(10_000), Milli(5000)); @@ -317,7 +317,7 @@ describe("skip floor", () => { it("a skip counts how long the skipping frame waited after ingress", async () => { const sync = new Sync({ delay: Milli(100) }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); video.received(Milli(0), Milli(5000)); @@ -331,7 +331,7 @@ describe("skip floor", () => { it("instant skips pass over nothing", async () => { const sync = new Sync({ delay: "instant" }); - const video = sync.track("video", { target: new Signal(undefined) }); + const video = sync.track({ target: new Signal(undefined) }); await flush(); video.received(Milli(0), Milli(5000)); video.received(Milli(10_000), Milli(5000)); @@ -351,7 +351,7 @@ describe("buffered startup", () => { video: new Signal<{ start: Milli; end: Milli } | null>(null), }; const tracks = Object.values(spans).map((startup) => - sync.track("video", { target: new Signal(Milli(100)), startup }), + sync.track({ target: new Signal(Milli(100)), startup }), ); const [video] = tracks; try { @@ -385,7 +385,7 @@ describe("buffered startup", () => { const clock = new Manual(); const installed = clock.install(); const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const video = sync.track("video", { target: new Signal(Milli(100)), startup: new Signal(null) }); + const video = sync.track({ target: new Signal(Milli(100)), startup: new Signal(null) }); try { await clock.advance(Nano.zero); video.received(Milli(0)); @@ -413,8 +413,8 @@ describe("buffered startup", () => { const buffer = new Signal(Milli(1000)); const sync = new Sync({ delay: Milli(100), buffer }); const absent = new Signal<{ start: Milli; end: Milli } | null | undefined>(null); - const remove = sync.track("audio", { target: new Signal(Milli(100)), startup: absent }); - const video = sync.track("video", { + const remove = sync.track({ target: new Signal(Milli(100)), startup: absent }); + const video = sync.track({ target: new Signal(Milli(100)), startup: new Signal({ start: Milli(0), end: Milli(100) }), }); @@ -443,7 +443,7 @@ describe("buffered startup", () => { }); }); -describe("audio clock", () => { +describe("tracks", () => { // Local time is pinned, so every playhead and deadline below is exact. let clock: Manual; let installed: Installed; @@ -455,193 +455,40 @@ describe("audio clock", () => { afterEach(() => installed.close()); const target = () => new Signal(undefined); - const local = () => Nano.toMilli(Nano(now())); - - it("follows the audio playhead while it rolls, not the earliest arrival", async () => { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); - await flush(); - video.received(Milli(1000), Milli(5000)); - expect(sync.now()).toBe(Milli(900)); - - // The ring re-buffered and plays 40ms behind where the wall clock would be. - audio.playhead({ rolling: true, timestamp: Milli(860), wall: Milli(5000) }); - expect(sync.now()).toBe(Milli(860)); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(sync.now()).toBe(Milli(870)); - - // Video waits for the audio it accompanies, however early it arrived. - let presented: boolean | undefined; - void video.wait(Milli(900)).then((ok) => { - presented = ok; - }); - await clock.advance(Nano.fromMilli(Milli(29))); - expect(presented).toBeUndefined(); - await clock.advance(Nano.fromMilli(Milli(1))); - expect(presented).toBe(true); - - // An earlier video arrival no longer moves the playhead audio owns. - video.received(Milli(5000), local()); - expect(sync.now()).toBe(Milli(900)); - sync.close(); - }); - - it("holds the playhead through a re-stall and resumes with it", async () => { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); - await flush(); - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - audio.playhead({ rolling: false, timestamp: Milli(1010) }); - await clock.advance(Nano.fromMilli(Milli(500))); - expect(sync.now()).toBe(Milli(1010)); - - // A frame already due presents; the next waits for audio to resume. - expect(await video.wait(Milli(1010))).toBe(true); - let presented: boolean | undefined; - void video.wait(Milli(1020)).then((ok) => { - presented = ok; - }); - await clock.advance(Nano.fromMilli(Milli(500))); - expect(presented).toBeUndefined(); - audio.playhead({ rolling: true, timestamp: Milli(1015), wall: local() }); - await clock.advance(Nano.fromMilli(Milli(5))); - expect(presented).toBe(true); - sync.close(); - }); - - it("holds a re-stall where the output stops, not where the ring did", async () => { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - await flush(); - // 30ms of output latency: the ring has read to 1030 while the output plays 1000. - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - audio.playhead({ rolling: false, timestamp: Milli(1030) }); - expect(sync.now()).toBe(Milli(1000)); - await clock.advance(Nano.fromMilli(Milli(20))); - expect(sync.now()).toBe(Milli(1020)); - await clock.advance(Nano.fromMilli(Milli(100))); - expect(sync.now()).toBe(Milli(1030)); - - // Refilled, the output plays the latency's silence first: the clock never steps back. - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - expect(sync.now()).toBe(Milli(1030)); - await clock.advance(Nano.fromMilli(Milli(40))); - expect(sync.now()).toBe(Milli(1040)); - audio.playhead({ rolling: true, timestamp: Milli(1040), wall: local() }); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(sync.now()).toBe(Milli(1050)); - sync.close(); - }); - - it("follows the output through a refill that lands before it reaches the stop", async () => { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - await flush(); - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - audio.playhead({ rolling: false, timestamp: Milli(1030) }); - await clock.advance(Nano.fromMilli(Milli(5))); - // Refilled within one quantum: no silence reaches the output, and the clock never jumps. - audio.playhead({ rolling: true, timestamp: Milli(1005), wall: local() }); - expect(sync.now()).toBe(Milli(1005)); - await clock.advance(Nano.fromMilli(Milli(40))); - expect(sync.now()).toBe(Milli(1045)); - sync.close(); - }); - - it("forgets the wall clock's skips once audio takes over", async () => { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); - await flush(); - // A burst skips the wall-clock playhead to 900, passing over everything before it. - video.received(Milli(0), local()); - video.received(Milli(1000), local()); - expect(video.presentable(Milli(100))).toBe(false); - - // Audio plays from 0, so a frame at 100 is still ahead of it. - audio.playhead({ rolling: true, timestamp: Milli(0), wall: local() }); - expect(video.presentable(Milli(100))).toBe(true); - const waiting = video.wait(Milli(100)); - await clock.advance(Nano.fromMilli(Milli(100))); - expect(await waiting).toBe(true); - sync.close(); - }); - - it("falls back to the wall clock where audio left it when audio stops", async () => { - for (const stalled of [false, true]) { - const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); - await flush(); - video.received(Milli(1000), Milli(5000)); - audio.playhead({ rolling: true, timestamp: Milli(800), wall: local() }); - if (stalled) audio.playhead({ rolling: false, timestamp: Milli(800) }); - await clock.advance(Nano.fromMilli(Milli(50))); - const before = sync.now(); - - // Muted: no jump back to video's arrival anchor, and time runs on. - audio.playhead(undefined); - expect(sync.now()).toBe(before); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(sync.now()).toBe(Milli((before ?? 0) + 10)); - - // Video's own arrivals hold the clock now, so ending its timeline re-anchors it. - video.reset(); - expect(sync.now()).toBeUndefined(); - video.received(Milli(3000), local()); - expect(sync.now()).toBe(Milli(2900)); - sync.close(); - } - }); - - it("a buffered ring that drains hands the clock back instead of holding video", async () => { - const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); - const audio = sync.track("audio", { target: target() }); - await flush(); - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - audio.playhead({ rolling: false, timestamp: Milli(1000) }); - await clock.advance(Nano.fromMilli(Milli(20))); - expect(sync.now()).toBe(Milli(1020)); - sync.close(); - }); it("a video-only discontinuity leaves audio's timeline alone", async () => { const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); + const audio = sync.track({ target: target() }); + const video = sync.track({ target: target() }); await flush(); + // Video arrived earliest and anchors the clock. audio.received(Milli(1000), Milli(5000)); - video.received(Milli(1000), Milli(5000)); - audio.playhead({ rolling: true, timestamp: Milli(900), wall: local() }); + video.received(Milli(1010), Milli(5000)); + expect(sync.now()).toBe(Milli(910)); - // Video's parked frame belongs to the timeline that ended; audio keeps playing on its own. + // Video's parked frame belongs to the timeline that ended; audio's arrivals take the reference + // and its parked frame stays on its own timeline. const parked = video.wait(Milli(950)); + const playing = audio.wait(Milli(950)); video.reset(); expect(await parked).toBe(false); expect(sync.now()).toBe(Milli(900)); - await clock.advance(Nano.fromMilli(Milli(10))); - expect(sync.now()).toBe(Milli(910)); + await clock.advance(Nano.fromMilli(Milli(50))); + expect(await playing).toBe(true); - // The same holds before audio rolls: audio's arrivals keep the reference. - const idle = new Sync({ delay: Milli(100) }); - const quiet = idle.track("audio", { target: target() }); - const picture = idle.track("video", { target: target() }); - await flush(); - quiet.received(Milli(1000), Milli(5000)); - picture.received(Milli(990), Milli(5000)); - const anchored = idle.out.reference.peek(); - picture.reset(); - expect(idle.out.reference.peek()).toBe(anchored); + // An audio break likewise leaves video's parked frames alone. + video.received(Milli(1100), Milli(5050)); + const picture = video.wait(Milli(1040)); + audio.reset(); + await clock.advance(Nano.fromMilli(Milli(50))); + expect(await picture).toBe(true); sync.close(); - idle.close(); }); it("releases the reference of the earliest track when it leaves", async () => { const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); + const audio = sync.track({ target: target() }); + const video = sync.track({ target: target() }); await flush(); // Video trails audio by 300ms, so audio anchors the clock and video is late against it. audio.received(Milli(1000), Milli(5000)); @@ -658,57 +505,33 @@ describe("audio clock", () => { sync.close(); }); - it("keeps the playhead still when the delay moves under it", async () => { - const delay = new Signal(Milli(100)); - const sync = new Sync({ delay }); - const audio = sync.track("audio", { target: target() }); - await flush(); - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - delay.set(Milli(300)); - await flush(); - expect(sync.now()).toBe(Milli(1000)); - sync.close(); - }); - - it("drops parked frames and stops following audio at a reset", async () => { + it("drops every track's parked frames at a reset", async () => { const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); + const audio = sync.track({ target: target() }); + const video = sync.track({ target: target() }); await flush(); - audio.playhead({ rolling: true, timestamp: Milli(1000), wall: local() }); - const parked = video.wait(Milli(1500)); + audio.received(Milli(1000), Milli(5000)); + const sound = audio.wait(Milli(1500)); + const picture = video.wait(Milli(1500)); sync.reset(); - expect(await parked).toBe(false); + expect(await sound).toBe(false); + expect(await picture).toBe(false); expect(sync.now()).toBeUndefined(); - // A stall reported after the reset is not a re-stall of the old timeline. - audio.playhead({ rolling: false, timestamp: Milli(1000) }); - video.received(Milli(3000), local()); + // The next frame anchors a new timeline, ignoring the anchors before the reset. + video.received(Milli(3000), Milli(5000)); expect(sync.now()).toBe(Milli(2900)); sync.close(); }); - it("reads zero until media time zero's turn, from either source", async () => { + it("reads zero until media time zero's turn", async () => { const sync = new Sync({ delay: Milli(100) }); - const audio = sync.track("audio", { target: target() }); - const video = sync.track("video", { target: target() }); + const video = sync.track({ target: target() }); await flush(); - video.received(Milli(0), local()); - expect(sync.now()).toBe(Milli.zero); - // Output latency still covers the first sample. - audio.playhead({ rolling: true, timestamp: -30, wall: local() }); + video.received(Milli(0), Milli(5000)); expect(sync.now()).toBe(Milli.zero); - await clock.advance(Nano.fromMilli(Milli(40))); + await clock.advance(Nano.fromMilli(Milli(110))); expect(sync.now()).toBe(Milli(10)); sync.close(); }); - - it("allows one audio track at a time", () => { - const sync = new Sync(); - const audio = sync.track("audio", { target: target() }); - expect(() => sync.track("audio", { target: target() })).toThrow(); - audio.close(); - sync.track("audio", { target: target() }).close(); - sync.close(); - }); }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index d0dc532c9f2..2a6d02dd84a 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -39,11 +39,10 @@ export type SyncOutput = { // that only care about the mode don't rerun when the number changes. instant: Signal; - // Local time minus media time, less the delay: `now()` is the local time minus this minus the - // delay. While audio plays it follows the audio playhead, re-synced on every report from the - // render worklet. Otherwise it is the earliest a frame has arrived relative to its timestamp, - // negative when timestamps run ahead of the local clock, and it settles once playback catches up - // to the live edge. + // The earliest time we've received a frame, relative to its timestamp: arrival minus timestamp + // in milliseconds, an offset that is negative when the timestamps run ahead of the local clock. + // This will keep being updated as we catch up to the live playhead then will be relatively static. + // Each track holds its own part of it; see `Track.reset()`. reference: Signal; // The resolved delay from the live edge to the playhead: the configured number, or in "auto" the @@ -63,9 +62,6 @@ export type SyncOutput = { maxDelay: Signal; }; -/** The media kinds that pace against the clock. @internal */ -export type TrackKind = "audio" | "video" | "text"; - /** A decoded span from the opening of a track's timeline. @internal */ export type Span = { start: Milli; end: Milli }; @@ -77,15 +73,6 @@ export type TrackProps = { startup?: Getter; }; -/** - * Where audio output stands, as the render worklet last reported it: rolling, the sample at media - * time `timestamp` ms (negative while output latency still covers the first sample) reaches the - * output at local time `wall` on the `@moq/clock` timeline; stalled mid-playback, the output holds - * at `timestamp`. - * @internal - */ -export type Playhead = { rolling: true; timestamp: number; wall: Milli } | { rolling: false; timestamp: Milli }; - /** What one track holds in the clock. @internal */ export type Part = { props: TrackProps; @@ -126,19 +113,6 @@ export class Clock { // The track whose arrivals set the reference, which releases it when its timeline ends. owner?: Part; - // The audio track, and whether its playhead drives the reference. - audio?: Part; - driving = false; - - // Where a re-stall stopped the ring, and the output's progress when it did. The output keeps - // playing what the ring already read until it reaches `timestamp`, which output latency delays, - // so the clock follows `from` until then, refilled or not. Past it, the clock holds while the ring - // is stalled, and after a refill until the reported playhead passes it, since the output first - // plays the silence the stall left behind rather than stepping back. - readonly paused = new Signal< - { timestamp: Milli; from: { playhead: number; wall: number }; stalled: boolean } | undefined - >(undefined); - constructor(props?: Inputs) { this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), @@ -179,13 +153,6 @@ export class Clock { const delay = this.out.delay.peek(); const late = current - ref + delay < 0; - // The audio playhead owns the reference while it plays: the ring skips and refills on its own, - // and another track's arrivals must not move its timeline. - if (this.driving) return late; - - // A reference audio handed back belongs to whichever track arrives next. - this.owner ??= part; - const reference = this.fold(current, ref); if (reference === current) return late; this.out.reference.set(reference); @@ -201,69 +168,10 @@ export class Clock { return late; } - // Re-sync the reference to the audio playhead, hold it at a re-stall, or hand it back to arrivals. - clock(part: Part, playhead: Playhead | undefined): void { - if (playhead?.rolling) { - // The wall clock's skips passed over frames relative to a playhead audio now replaces. - if (!this.driving) this.floor = undefined; - this.driving = true; - this.owner = undefined; - this.out.reference.set(playhead.wall - playhead.timestamp - this.out.delay.peek()); - const paused = this.paused.peek(); - if (!paused) return; - if (this.playhead() >= paused.timestamp && this.rolling() >= paused.timestamp) this.paused.set(undefined); - else this.paused.set({ ...paused, stalled: false }); - return; - } - if (this.audio !== part || !this.driving) return; - // A ring that runs dry while it still plays holds the clock with it, so video waits for the - // audio it accompanies. Buffered playback drains between bursts by design, so its ring - // stalling means the audio ended, not that it is late. - if (playhead && !this.out.buffered.peek()) { - this.paused.set({ - timestamp: playhead.timestamp, - from: { playhead: this.playhead(), wall: wall() }, - stalled: true, - }); - return; - } - this.release(); - } - - // The playhead extrapolated from the reference, before any hold or clamp. - rolling(): number { - return wall() - (this.out.reference.peek() ?? 0) - this.out.delay.peek(); - } - - // The playhead, applying a re-stall's hold. - playhead(): number { - const rolling = this.rolling(); - const paused = this.paused.peek(); - if (!paused) return rolling; - const before = paused.from.playhead + wall() - paused.from.wall; - if (before < paused.timestamp) return before; - return paused.stalled ? paused.timestamp : Math.max(rolling, paused.timestamp); - } - - // Hand the clock back to the wall at the last audio-derived value, so video does not jump. The - // reference passes to another track still receiving, whose own timeline ending releases it, or - // else to the next track to arrive. - release(): void { - if (this.paused.peek()) this.out.reference.set(wall() - this.playhead() - this.out.delay.peek()); - this.paused.set(undefined); - this.driving = false; - this.owner = this.parts.peek().find((part) => part !== this.audio && part.anchor !== undefined); - } - // End one track's timeline: its parked frames drop and it releases its part of the reference. end(part: Part): void { part.epoch.update((epoch) => epoch + 1); part.anchor = undefined; - if (this.driving) { - // The audio playhead holds the reference, which another track's timeline leaves alone. - if (this.audio !== part) return; - this.release(); - } if (this.owner !== part) return; // The earliest remaining track holds the reference, else the next frame anchors a new one. @@ -290,8 +198,6 @@ export class Clock { reset(): void { for (const part of this.parts.peek()) part.anchor = undefined; - this.driving = false; - this.paused.set(undefined); this.unanchor(); } @@ -308,13 +214,13 @@ export class Clock { if (!this.playing.peek()) return undefined; const reference = this.out.reference.peek(); if (reference === undefined) return undefined; - // Media time is nonnegative, so a playhead before the first sample's turn reads as zero. - return Milli(Math.max(0, this.playhead())); + // Media time is nonnegative, so a playhead before the first frame's turn reads as zero. + return Milli(Math.max(0, wall() - reference - this.out.delay.peek())); } - async wait(part: Part | undefined, timestamp: Milli, signal?: AbortSignal): Promise { + async wait(part: Part, timestamp: Milli, signal?: AbortSignal): Promise { const epoch = this.epoch.peek(); - const own = part?.epoch.peek(); + const own = part.epoch.peek(); for (;;) { // Closing or aborting wakes the sleep below, so frames parked here leave. @@ -323,7 +229,7 @@ export class Clock { // A reset clears the reference, so an unanchored frame predates the current timeline. const reference = this.out.reference.peek(); if (reference === undefined || this.epoch.peek() !== epoch || this.skipped(timestamp)) return false; - if (part && part.epoch.peek() !== own) return false; + if (part.epoch.peek() !== own) return false; // A zero delay still sleeps: the sleep comes from the reference, which holds an early frame // until its timestamp comes up. "instant" is the only thing that skips the wait itself. @@ -335,16 +241,8 @@ export class Clock { continue; } - // A stalled audio ring holds the playhead where it stopped until it plays again. - const paused = this.paused.peek(); - if (paused?.stalled && timestamp > paused.timestamp) { - await this.sleep(part, undefined, signal); - continue; - } - - // NOTE: This function runs in parallel for each frame. The playhead moves at most as fast - // as the local clock, so sleeping the difference never overshoots; a hold re-checks. - const sleep = timestamp - this.playhead(); + // NOTE: This function runs in parallel for each frame. + const sleep = reference + timestamp - wall() + this.out.delay.peek(); // Skip the timer for small sleeps; the host's timer resolution (~4ms) would overshoot. if (sleep < 5) return true; @@ -356,7 +254,7 @@ export class Clock { // Sleeps for `ms`, or wakes early once anything the sleep was computed from changes or `signal` // aborts. Releases every listener either way: frames sleep once each, so a listener left on a // signal that never changes would pile up for the life of the player. - sleep(part: Part | undefined, ms: number | undefined, signal?: AbortSignal): Promise { + sleep(part: Part, ms: number | undefined, signal?: AbortSignal): Promise { return new Promise((resolve) => { const timer = new AbortController(); const wake = () => { @@ -372,9 +270,8 @@ export class Clock { this.epoch.changed(wake), this.closed.changed(wake), this.playing.changed(wake), - this.paused.changed(wake), + part.epoch.changed(wake), ]; - if (part) disposes.push(part.epoch.changed(wake)); if (signal) { signal.addEventListener("abort", wake, { once: true }); disposes.push(() => signal.removeEventListener("abort", wake)); @@ -389,19 +286,19 @@ export class Clock { * @internal */ export class Track { - protected readonly clock: Clock; - protected readonly part: Part; + readonly #clock: Clock; + readonly #part: Part; /** @internal */ constructor(clock: Clock, props: TrackProps) { - this.clock = clock; - this.part = { props, epoch: new Signal(0) }; - clock.parts.update((parts) => [...parts, this.part]); + this.#clock = clock; + this.#part = { props, epoch: new Signal(0) }; + clock.parts.update((parts) => [...parts, this.#part]); } /** Fold a received frame into the clock and report whether it arrived too late for its turn. */ received(timestamp: Milli, arrival = wall()): boolean { - return this.clock.received(this.part, timestamp, arrival); + return this.#clock.received(this.#part, timestamp, arrival); } /** @@ -409,7 +306,7 @@ export class Track { * may still present. Advisory, for showing a frame early; {@link Track.wait} decides on its own. */ presentable(timestamp: Milli): boolean { - return this.clock.presentable(timestamp); + return this.#clock.presentable(timestamp); } /** @@ -419,7 +316,7 @@ export class Track { * reset ended its timeline, or `close()` or `signal` released the wait. */ wait(timestamp: Milli, signal?: AbortSignal): Promise { - return this.clock.wait(this.part, timestamp, signal); + return this.#clock.wait(this.#part, timestamp, signal); } /** @@ -427,32 +324,13 @@ export class Track { * drop, and its arrivals stop holding the reference; the other tracks' timelines are untouched. */ reset(): void { - this.clock.end(this.part); + this.#clock.end(this.#part); } /** Release this track's hold on the clock. */ close(): void { - this.clock.end(this.part); - this.clock.parts.update((parts) => parts.filter((part) => part !== this.part)); - if (this.clock.audio === this.part) this.clock.audio = undefined; - } -} - -/** The audio track, whose playhead drives the clock while it plays. @internal */ -export class AudioTrack extends Track { - /** @internal */ - constructor(clock: Clock, props: TrackProps) { - if (clock.audio) throw new Error("the clock already has an audio track"); - super(clock, props); - clock.audio = this.part; - } - - /** - * Report the audio playhead. A rolling one drives the clock, a stalled one holds it, and undefined - * (muted, disabled, or flushed) hands it back to the wall clock where it stands. - */ - playhead(playhead: Playhead | undefined): void { - this.clock.clock(this.part, playhead); + this.#clock.end(this.#part); + this.#clock.parts.update((parts) => parts.filter((part) => part !== this.#part)); } } @@ -482,16 +360,11 @@ export class Sync { } /** - * Open a track's hold on the clock. The audio track's playhead drives the clock while it plays; - * only one may be open at a time. + * Open a track's hold on the clock: its playout target, startup span, arrivals, and paced frames. * @internal */ - track(kind: "audio", props: TrackProps): AudioTrack; - /** Open a track's hold on the clock. @internal */ - track(kind: "video" | "text", props: TrackProps): Track; - /** Open a track's hold on the clock. @internal */ - track(kind: TrackKind, props: TrackProps): Track { - return kind === "audio" ? new AudioTrack(this.#clock, props) : new Track(this.#clock, props); + track(props: TrackProps): Track { + return new Track(this.#clock, props); } // Startup is data-driven: time passing cannot fill a missing track. The first picture still @@ -559,14 +432,7 @@ export class Sync { } } - // The audio playhead does not move with the delay: the ring parks or sheds to reach it, and - // reports where it went. Keep `now()` on the playhead until it does. - const clock = this.#clock; - const reference = clock.out.reference.peek(); - if (clock.driving && reference !== undefined) { - clock.out.reference.set(reference - (delay - clock.out.delay.peek())); - } - clock.out.delay.set(delay); + this.#clock.out.delay.set(delay); } /** @@ -579,9 +445,8 @@ export class Sync { } /** - * The PTS that should be rendering right now: the audio playhead while audio plays, else derived - * from the reference and delay; zero until media time zero's turn. Undefined before ingress or - * while buffered startup is filling. + * The PTS that should be rendering right now, derived from the reference and delay; zero until + * media time zero's turn. Undefined before ingress or while buffered startup is filling. */ now(): Milli | undefined { return this.#clock.now(); diff --git a/js/watch/src/text/renderer.ts b/js/watch/src/text/renderer.ts index c9a8ee1a814..0e10efe3900 100644 --- a/js/watch/src/text/renderer.ts +++ b/js/watch/src/text/renderer.ts @@ -248,7 +248,7 @@ export class Renderer { const delay = effect.get(this.source.out.config)?.delay; return delay !== undefined ? Milli(delay) : undefined; }); - this.#clock = this.sync.track("text", { target }); + this.#clock = this.sync.track({ target }); this.#signals.cleanup(() => this.#clock.close()); this.#signals.run(this.#run.bind(this)); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index dbea56d638b..c3e590e978d 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -381,7 +381,7 @@ async function guardedPlayback( : undefined; const decoder = new Decoder({ source, sync, enabled }); // Another track on the same clock, whose arrivals the tests drive by hand. - const audio = sync.track("audio", { target: new Signal(undefined) }); + const audio = sync.track({ target: new Signal(undefined) }); await microtasks(); return { decoder, @@ -1213,7 +1213,7 @@ for (const kind of ["legacy", "cmaf"] as const) { const reads: Read[] = []; const playback = await guardedPlayback(kind, reads, [], false, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: audio }); const reset = spyOn(playback.sync, "reset"); try { playback.enabled.set(false); @@ -1261,7 +1261,7 @@ for (const kind of ["legacy", "cmaf"] as const) { reads.push(sample(group * 2n, Number(group) * 40_000)); } const playback = await guardedPlayback(kind, reads, [], false, 1000); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: new Signal(null) }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: new Signal(null) }); try { await microtasks(); await clock.advance(0); @@ -1281,7 +1281,7 @@ for (const kind of ["legacy", "cmaf"] as const) { it("bounds startup decoding while another track is missing and releases it on close", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: new Signal(null) }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: new Signal(null) }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1307,7 +1307,7 @@ it("resumes decoding the moment shared startup releases", async () => { const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: audio }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1355,7 +1355,7 @@ it("continues filling when the shared floor rises while video waits on audio", a const clock = manualAt(0); const playback = await guardedPlayback("legacy", [], [], true, 1000); const audio = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: audio }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: audio }); if (!playback.track) throw new Error("missing video track"); const group = playback.track.appendGroup(); try { @@ -1400,7 +1400,7 @@ for (const kind of ["legacy", "cmaf"] as const) { const clock = manualAt(0); const playback = await guardedPlayback(kind, [sample(0n, 0), sample(0n, 100_000)], [0, 100_000], false, 1000); const other = new Signal<{ start: Milli; end: Milli } | null>(null); - const remove = playback.sync.track("text", { target: new Signal(Milli(100)), startup: other }); + const remove = playback.sync.track({ target: new Signal(Milli(100)), startup: other }); try { playback.release(); await microtasks(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 340c1e9d257..ca5b369d7a2 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -190,7 +190,7 @@ export class Decoder { const ranges = effect.get(active.buffered); return startupSpan(ranges, timestamp); }); - this.#clock = this.sync.track("video", { target, startup }); + this.#clock = this.sync.track({ target, startup }); this.#signals.cleanup(() => this.#clock.close()); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index 61cc75bca85..05bc0f3b2dc 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -13,8 +13,8 @@ the same arrival trace. Boundaries: converging by playing slightly faster or slower is the browser's [Watch](/quest/m1/audio-jitter-target/watch.md) quest. No packet loss concealment. -Video keeps its own target; the audio playhead already drives `Sync` while -audio plays. +Video keeps its own target; the shared clock stays on local time, and +[Watch](/quest/m1/audio-jitter-target/watch.md) converges the audio ring on it. ## Plan diff --git a/quest/m1/audio-jitter-target/watch.md b/quest/m1/audio-jitter-target/watch.md index d192dd83b1b..939cde0ba6c 100644 --- a/quest/m1/audio-jitter-target/watch.md +++ b/quest/m1/audio-jitter-target/watch.md @@ -150,6 +150,16 @@ path, so the 20 ms default stays. Decided 2026-10-15: keep this proof in m1 with the existing playback and A/V clock work. +Decided 2026-11-05 (maintainer, /quest-plan): steady-state lip sync comes +from this stretch converging the ring's playhead on `Sync.now()`, the shared +clock on local time anchored at the earliest arrival. The clock never follows +audio: one slow track must not hold the others, so audio and video stay in sync +in the steady state and drift apart only while a track is congested. +[moq-dev/moq#7941](https://github.com/moq-dev/moq/pull/7941) dropped an +audio-driven clock for this. Recommendation: measure the gap against the sample +reaching the output, through `AudioContext.getOutputTimestamp()`, so output +latency, Bluetooth included, is part of the lip sync. + ## Waiting - Maintainer: Safari with 40 ms or more of added RTT, the baseline row the diff --git a/quest/m2/webcodecs-priming.md b/quest/m2/webcodecs-priming.md index e732e2c5692..050390794bd 100644 --- a/quest/m2/webcodecs-priming.md +++ b/quest/m2/webcodecs-priming.md @@ -17,8 +17,8 @@ gap ([w3c/webcodecs#626](https://github.com/w3c/webcodecs/issues/626)). Out of scope: Rust, where `moq-audio` already declares Opus lookahead as pre-skip and folds the AAC delay into timestamps (`folded_delay` in -`rs/moq-audio/src/encode/encoder.rs`); and playback-side sync, where -the audio playhead drives `Sync`. +`rs/moq-audio/src/encode/encoder.rs`); and playback-side sync, which +[Watch](/quest/m1/audio-jitter-target/watch.md) converges by time-stretching. ## Plan diff --git a/test/video/src/captions.ts b/test/video/src/captions.ts index cdae42dcefd..0a46bf4fc7f 100644 --- a/test/video/src/captions.ts +++ b/test/video/src/captions.ts @@ -31,7 +31,7 @@ export function captions() { const source = new Source({ broadcast, target }); const sync = new Sync({ delay: Milli.zero }); // Stands in for the media that anchors the clock the cues show against. - const media = sync.track("video", { target: new Signal(undefined) }); + const media = sync.track({ target: new Signal(undefined) }); media.received(Milli(1000)); const container = document.createElement("div"); container.id = "captions"; From 06831c8c62c02eda2bca5690811c60968c810924 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:28:52 -0800 Subject: [PATCH 7/9] fix(watch): reconcile the skip floor and startup anchors when the reference changes owner Co-Authored-By: Claude Opus 5.5 --- js/watch/src/sync.test.ts | 40 +++++++++++++++++++++++++++++++++++++++ js/watch/src/sync.ts | 24 ++++++++++++++++++++--- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 7fa6736e20f..e13c4ec393c 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -505,6 +505,46 @@ describe("tracks", () => { sync.close(); }); + it("forgets a departed track's skip once the reference moves back to the survivor's", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track({ target: target() }); + const video = sync.track({ target: target() }); + await flush(); + // Audio's burst skips the playhead to 1900; video's own arrivals put it at 900. + video.received(Milli(1000), Milli(5000)); + audio.received(Milli(2000), Milli(5000)); + expect(video.presentable(Milli(1100))).toBe(false); + + audio.reset(); + expect(sync.now()).toBe(Milli(900)); + const waiting = video.wait(Milli(1100)); + await clock.advance(Nano.fromMilli(Milli(200))); + expect(await waiting).toBe(true); + sync.close(); + }); + + it("keeps the playhead where buffered startup put it when the owner leaves", async () => { + const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); + const audioSpan = new Signal<{ start: Milli; end: Milli } | null>(null); + const videoSpan = new Signal<{ start: Milli; end: Milli } | null>(null); + const audio = sync.track({ target: target(), startup: audioSpan }); + const video = sync.track({ target: target(), startup: videoSpan }); + await flush(); + audio.received(Milli(500), Milli(5000)); + video.received(Milli(500), Milli(5000)); + + // Startup releases a second later, anchoring at the retained opening rather than its arrival. + await clock.advance(Nano.fromMilli(Milli(1000))); + audioSpan.set({ start: Milli(500), end: Milli(700) }); + videoSpan.set({ start: Milli(500), end: Milli(700) }); + await flush(); + expect(sync.now()).toBe(Milli(500)); + + audio.reset(); + expect(sync.now()).toBe(Milli(500)); + sync.close(); + }); + it("drops every track's parked frames at a reset", async () => { const sync = new Sync({ delay: Milli(100) }); const audio = sync.track({ target: target() }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 2a6d02dd84a..5b199d11454 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -181,8 +181,20 @@ export class Clock { if (earliest?.anchor === undefined || other.anchor < earliest.anchor) earliest = other; } this.owner = earliest; - if (earliest) this.out.reference.set(earliest.anchor); - else this.unanchor(); + if (earliest?.anchor === undefined) { + this.unanchor(); + return; + } + const current = this.out.reference.peek(); + const reference = earliest.anchor; + this.out.reference.set(reference); + + // The floor marked frames a skip on the departed track's reference passed over. Moving back + // to the survivor's puts its frames ahead of the playhead again; moving forward is a skip. + this.floor = undefined; + if (current !== undefined && reference < current && this.in.delay.peek() !== "instant") { + this.floor = wall() - reference - this.out.delay.peek(); + } } // Clear the reference so the next frame anchors a new timeline. Every parked frame belongs to @@ -399,7 +411,13 @@ export class Sync { clock.started = true; clock.owner = earliest.part; // Anchor at the retained opening, not at its arrival: startup may have waited on another track. - clock.out.reference.set(wall() - first - delay); + // Each track's own anchor moves with it, so handing the reference to another track later keeps + // the playhead where startup put it. + const anchored = wall() - first - delay; + for (const part of clock.parts.peek()) { + if (part.anchor !== undefined) part.anchor += anchored - reference; + } + clock.out.reference.set(anchored); clock.playing.set(true); } From 103d06b24c595dc65ba25df2d7a9bb0a47fb6f19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 11:00:10 -0800 Subject: [PATCH 8/9] fix(watch): address CodeRabbit on per-track timelines - Keep a survivor's own skip floor when the reference moves back to it. - Release video's clock hold when its last rendition fails. - Captions with no overlay container hold no playout target. - MSF encode refuses a catalog decode would reject. - Docs and the bufferbloat quest state the clock's conditions. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 9 ++++++--- js/msf/src/catalog.test.ts | 5 +++++ js/msf/src/catalog.ts | 3 +++ js/watch/src/sync.test.ts | 17 +++++++++++++++++ js/watch/src/sync.ts | 18 +++++++++++++----- js/watch/src/text/renderer.test.ts | 12 ++++++++++-- js/watch/src/text/renderer.ts | 3 ++- js/watch/src/video/decoder.test.ts | 5 +++++ js/watch/src/video/decoder.ts | 4 ++++ quest/m2/audio-bufferbloat-av.md | 18 +++++++++--------- 10 files changed, 74 insertions(+), 20 deletions(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index ab2539f5ac4..aba5df261ab 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -50,9 +50,12 @@ The overlay adds play/pause, volume, fullscreen, a quality selector, a buffering indicator, and a stats panel. Audio, video, and captions play from one clock on local time, anchored at the -earliest arrival, so a congested track falls behind on its own instead of -stalling the others. A discontinuity restarts only its own track's timeline, -and a track that stops arriving releases its hold on the clock. +earliest arrival (with `buffer`, at the earliest retained frame once startup +fills). A congested track falls behind on its own instead of stalling the +others; in `auto` delay, its rising playout target still raises the delay +every track plays at. A discontinuity restarts only its own track's timeline, +and a track that is disabled, deselected, or fails releases its hold on the +clock. `el.audio.out.error` and `el.video.out.error` report a fatal decode error. The failing track closes its subscription and stops decoding. Select another diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index bbffe319a7a..ab97095fae1 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -245,6 +245,11 @@ test("rejects a negative delay", () => { tracks: [{ name: "video0", packaging: "loc", isLive: true, role: "video", codec: "avc1.640028", delay: -1 }], }); expect(() => decode(raw)).toThrow(); + expect(() => + encode({ + tracks: [{ name: "video0", packaging: "loc", isLive: true, role: "video", codec: "avc1.640028", delay: -1 }], + }), + ).toThrow(); }); test("carries extension root members through decode and encode", () => { diff --git a/js/msf/src/catalog.ts b/js/msf/src/catalog.ts index 37be0bb6f0e..dc79ac70bc3 100644 --- a/js/msf/src/catalog.ts +++ b/js/msf/src/catalog.ts @@ -128,6 +128,9 @@ const WireCatalogSchema = z.object({ /** Serialize a catalog to its JSON wire representation (draft-01). */ export function encode(catalog: Catalog): Uint8Array { + // Refuse what `decode` would reject, so a publisher fails here instead of its subscribers. + CatalogSchema.parse(catalog); + // Hoist inline init payloads into a shared, deduplicated initDataList and // reference each from its track via initRef (the draft-01 wire shape). const initDataList: z.infer[] = []; diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index e13c4ec393c..2ba372cb264 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -523,6 +523,23 @@ describe("tracks", () => { sync.close(); }); + it("keeps the survivor's own skip when the reference moves back to it", async () => { + const sync = new Sync({ delay: Milli(100) }); + const audio = sync.track({ target: target() }); + const video = sync.track({ target: target() }); + await flush(); + // Video skips its own playhead from 900 to 1400, then audio's burst skips it to 2900. + video.received(Milli(1000), Milli(5000)); + video.received(Milli(1500), Milli(5000)); + audio.received(Milli(3000), Milli(5000)); + + audio.reset(); + expect(sync.now()).toBe(Milli(1400)); + expect(video.presentable(Milli(1300))).toBe(false); + expect(video.presentable(Milli(1450))).toBe(true); + sync.close(); + }); + it("keeps the playhead where buffered startup put it when the owner leaves", async () => { const sync = new Sync({ delay: Milli(100), buffer: Milli(1000) }); const audioSpan = new Signal<{ start: Milli; end: Milli } | null>(null); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 5b199d11454..06398a92528 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -80,6 +80,8 @@ export type Part = { epoch: Signal; // The reference this track's own arrivals hold on its current timeline. anchor?: number; + // The furthest playhead this track's own skips jumped to, restored when it takes the reference. + floor?: number; }; const wall = (): Milli => Nano.toMilli(Nano(now())); @@ -164,6 +166,7 @@ export class Clock { if (this.in.delay.peek() !== "instant") { const floor = wall() - reference - delay; if (this.floor === undefined || floor > this.floor) this.floor = floor; + if (part.floor === undefined || floor > part.floor) part.floor = floor; } return late; } @@ -172,6 +175,7 @@ export class Clock { end(part: Part): void { part.epoch.update((epoch) => epoch + 1); part.anchor = undefined; + part.floor = undefined; if (this.owner !== part) return; // The earliest remaining track holds the reference, else the next frame anchors a new one. @@ -190,8 +194,9 @@ export class Clock { this.out.reference.set(reference); // The floor marked frames a skip on the departed track's reference passed over. Moving back - // to the survivor's puts its frames ahead of the playhead again; moving forward is a skip. - this.floor = undefined; + // to the survivor's puts its frames ahead of the playhead again, except those its own skips + // passed over; moving forward is a skip. + this.floor = earliest.floor; if (current !== undefined && reference < current && this.in.delay.peek() !== "instant") { this.floor = wall() - reference - this.out.delay.peek(); } @@ -209,7 +214,10 @@ export class Clock { } reset(): void { - for (const part of this.parts.peek()) part.anchor = undefined; + for (const part of this.parts.peek()) { + part.anchor = undefined; + part.floor = undefined; + } this.unanchor(); } @@ -411,8 +419,8 @@ export class Sync { clock.started = true; clock.owner = earliest.part; // Anchor at the retained opening, not at its arrival: startup may have waited on another track. - // Each track's own anchor moves with it, so handing the reference to another track later keeps - // the playhead where startup put it. + // Every track's own anchor moves by the same offset, keeping the gaps between tracks that + // playing would have held, so a later handoff moves the playhead only by that gap. const anchored = wall() - first - delay; for (const part of clock.parts.peek()) { if (part.anchor !== undefined) part.anchor += anchored - reference; diff --git a/js/watch/src/text/renderer.test.ts b/js/watch/src/text/renderer.test.ts index 1ca31f3b97a..13c27a18d85 100644 --- a/js/watch/src/text/renderer.test.ts +++ b/js/watch/src/text/renderer.test.ts @@ -293,7 +293,7 @@ describe("sanitizeCue", () => { }); describe("Renderer", () => { - test("holds the selected rendition's catalog delay in the shared clock while enabled", async () => { + test("holds the selected rendition's catalog delay in the shared clock while it can render", async () => { const config = new Signal({ delay: 300 } as Catalog.TextConfig); const source = { in: { broadcast: new Signal(undefined) }, @@ -301,7 +301,9 @@ describe("Renderer", () => { } as unknown as Source; const enabled = new Signal(true); const sync = new Sync(); - const renderer = new Renderer({ source, sync, enabled }); + // Never touched: with no track selected, nothing subscribes or renders. + const container = new Signal({} as HTMLElement); + const renderer = new Renderer({ source, sync, enabled, container }); try { await yieldNow(); expect(sync.out.delay.peek()).toBe(Milli(300)); @@ -311,6 +313,12 @@ describe("Renderer", () => { await yieldNow(); expect(sync.out.delay.peek()).toBe(Milli.zero); enabled.set(true); + container.set(undefined); + await yieldNow(); + expect(sync.out.delay.peek()).toBe(Milli.zero); + container.set({} as HTMLElement); + await yieldNow(); + expect(sync.out.delay.peek()).toBe(Milli(300)); config.set(undefined); await yieldNow(); expect(sync.out.delay.peek()).toBe(Milli.zero); diff --git a/js/watch/src/text/renderer.ts b/js/watch/src/text/renderer.ts index 0e10efe3900..2e405267c5f 100644 --- a/js/watch/src/text/renderer.ts +++ b/js/watch/src/text/renderer.ts @@ -244,7 +244,8 @@ export class Renderer { }; const target = this.#signals.computed((effect) => { - if (!effect.get(this.in.enabled)) return undefined; + // Captions with nowhere to render hold nothing. + if (!effect.get(this.in.enabled) || !effect.get(this.in.container)) return undefined; const delay = effect.get(this.source.out.config)?.delay; return delay !== undefined ? Milli(delay) : undefined; }); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index c3e590e978d..6f643ff4aa0 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -785,14 +785,18 @@ for (const kind of ["legacy", "cmaf"] as const) { it("releases a failed video subscription and recovers only on a new source", async () => { const close = spyOn(Moq.Track.Subscriber.prototype, "close"); + const reset = spyOn(Track.prototype, "reset"); const playback = await guardedPlayback("legacy", []); try { const before = close.mock.calls.length; + const resets = reset.mock.calls.length; const failure = new DOMException("invalid encoded video", "EncodingError"); playback.fail(failure); await microtasks(); expect(playback.decoder.out.error.peek()).toBe(failure); expect(close.mock.calls.length).toBeGreaterThan(before); + // Video's arrivals stop holding the shared clock. + expect(reset.mock.calls.length).toBeGreaterThan(resets); playback.enabled.set(false); await microtasks(); playback.enabled.set(true); @@ -804,6 +808,7 @@ it("releases a failed video subscription and recovers only on a new source", asy } finally { playback.close(); close.mockRestore(); + reset.mockRestore(); } }); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index ca5b369d7a2..d26a1dbfc63 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -323,6 +323,8 @@ export class Decoder { pending.close(); pending = undefined; this.#pending.set(undefined); + // With no active rendition left, nothing holds video's timeline. + if (!this.#active.peek()) this.#clock.reset(); }); effect.set(this.#pending, { track, catalog: broadcast.out.catalog }); @@ -371,6 +373,8 @@ export class Decoder { if (!error) return; this.#recordError(active, error); this.#active.set(undefined); + // A pending replacement carries video's timeline on; its parked frames must stay. + if (!this.#pending.peek()) this.#clock.reset(); this.#clearCurrentFrame(); }); diff --git a/quest/m2/audio-bufferbloat-av.md b/quest/m2/audio-bufferbloat-av.md index f939068fda4..12caea622a3 100644 --- a/quest/m2/audio-bufferbloat-av.md +++ b/quest/m2/audio-bufferbloat-av.md @@ -4,16 +4,15 @@ A nightly test sends audio and video together through a bufferbloated link and proves audio holds up better than video: audio stays within its underrun -and skip budgets while video may freeze or skip, and every video frame that -renders is in sync with audio. Today no test publishes both through the -shaper, and the audio-quality profiles leave rate limits off. +and skip budgets while video may freeze, skip, or lag audio. Today no test +publishes both through the shaper, and the audio-quality profiles leave rate +limits off. ## Plan Decided in planning (2026-10-20): -- Behavior asserted is today's design: continuity plus sync. Audio is sent - first (players subscribe to audio at a higher priority than video) and its +- Behavior asserted is audio continuity. Audio is sent first (players subscribe to audio at a higher priority than video) and its ring plays through a video stall. In `"auto"` delay the shared target is the deepest track's, so late video raises audio latency too; that is expected, not a failure. Making audio latency independent of video would be a player change, not this @@ -21,10 +20,11 @@ Decided in planning (2026-10-20): - Extend `test/audio-quality` rather than routing media QA through the shaper: add rows whose publisher sends the media QA pattern (frame counter plus stepped tone), and reuse the tap worklet, budgets, captures, and - replays. Add media QA's sync probe to grade rendered frames against the - same sync bound `just test media` holds the browser to. Video freezes and - skips are reported, not gated: the point is that audio does not pay for - them. + replays. Add media QA's sync probe to report each rendered frame's gap to + audio. Video freezes, skips, and lag are reported, not gated: the point is + that audio does not pay for them. Since 2026-11-05 the shared clock never + follows audio, so congested video lags it by design, and steady-state lip + sync belongs to [the Watch stretch](/quest/m1/audio-jitter-target/watch.md). - Folded in: the native `pulse` rows become enforced. They report only because an intermittent two-second device stall voids about one AAC native row in seven (see `test/audio-quality/README.md`). Reproduce and fix that From a95ed5d0fc2d3d09449c38361695909902c41f25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 11:07:53 -0800 Subject: [PATCH 9/9] style(msf): format catalog test Co-Authored-By: Claude Opus 5.5 --- js/msf/src/catalog.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index ab97095fae1..d352acda37e 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -247,7 +247,9 @@ test("rejects a negative delay", () => { expect(() => decode(raw)).toThrow(); expect(() => encode({ - tracks: [{ name: "video0", packaging: "loc", isLive: true, role: "video", codec: "avc1.640028", delay: -1 }], + tracks: [ + { name: "video0", packaging: "loc", isLive: true, role: "video", codec: "avc1.640028", delay: -1 }, + ], }), ).toThrow(); });