diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 478d49ea0e..1fdab9340b 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -31,7 +31,7 @@ subset drops. What a peer actually observes against this implementation: - **Strict SETUP and parameters.** Repeated unknown `SETUP` options, GREASE included, are accepted if well-formed; a repeated known option is rejected. On draft 17 and later, a `GROUP_ORDER` other than Ascending (1) or Descending (2) closes the session. - **One credential per session**, carried in `SETUP` and forwarded to the [auth server](/bin/relay/auth#the-contract) unverified. A token attached to an individual request is ignored. An alias reference, a second token, or a parameter the negotiated draft does not define closes the session. - **Refused, not fatal.** A legal request this stack does not serve is rejected on its own and the session stays up: `FORWARD=0`, range filters, `TRACK_STATUS`, `SUBSCRIBE_TRACKS`, and the fetch forms above. On draft 19 and later, and in Rust on drafts 14 through 16, a subscription update may change only priority; any other update ends that subscription. -- **Datagrams** are a single normal object at object 0, forwarded without renumbering. Anything else is dropped. JavaScript does not carry datagrams on moq-transport. +- **Datagrams** are a single normal object at object 0, forwarded without renumbering. Anything else is dropped, and a malformed one closes the session. Rust and JavaScript both carry them on every draft. - **Priority.** Higher is served first. The IETF default of 128 is this stack's 127, and a track that never sets one is 127. - **Size.** An object extension block larger than 64 KiB ends that subgroup stream. The session stays up. This cap is ours, not the draft's. diff --git a/js/net/src/lite/datagram_stream.test.ts b/js/net/src/datagram_stream.test.ts similarity index 100% rename from js/net/src/lite/datagram_stream.test.ts rename to js/net/src/datagram_stream.test.ts diff --git a/js/net/src/lite/datagram_stream.ts b/js/net/src/datagram_stream.ts similarity index 100% rename from js/net/src/lite/datagram_stream.ts rename to js/net/src/datagram_stream.ts diff --git a/js/net/src/ietf/aliases.ts b/js/net/src/ietf/aliases.ts index 00ded5c486..871a96719b 100644 --- a/js/net/src/ietf/aliases.ts +++ b/js/net/src/ietf/aliases.ts @@ -118,6 +118,11 @@ export class TrackAliases { } } + /** The alias's binding right now, without waiting for one. */ + peek(alias: bigint): T | undefined { + return this.#active.get(alias)?.value; + } + /** * Establishes an alias and releases any data streams waiting for it. * diff --git a/js/net/src/ietf/connection.ts b/js/net/src/ietf/connection.ts index 87dcc30e9d..6cf0994973 100644 --- a/js/net/src/ietf/connection.ts +++ b/js/net/src/ietf/connection.ts @@ -213,6 +213,7 @@ export class Connection implements Established { if (this.#closed) return; this.#closed = true; + this.#publisher.close(); // Before the session, whose own close would send a clean code first. try { @@ -232,11 +233,16 @@ export class Connection implements Established { async #run(early: Reader[]): Promise { try { - // Both run together. runPublishNamespaces is a no-op when the peer asked to be + // All run together. runPublishNamespaces is a no-op when the peer asked to be // solicited; otherwise it pushes PUBLISH_NAMESPACE. On draft-16 and later a // SUBSCRIBE_NAMESPACE stream is filled either way. A NAMESPACE is discovery, // not a second route. - await Promise.all([this.#runBidis(), this.#runUnis(early), this.#publisher.runPublishNamespaces()]); + await Promise.all([ + this.#runBidis(), + this.#runUnis(early), + this.#runDatagrams(), + this.#publisher.runPublishNamespaces(), + ]); } catch (err) { if (!this.#closed) { console.error("fatal error running connection", err); @@ -389,6 +395,17 @@ export class Connection implements Established { } } + // A malformed OBJECT_DATAGRAM is the peer breaking the protocol, like an invalid stream type. + async #runDatagrams() { + try { + await this.#subscriber.runDatagrams(); + } catch (err: unknown) { + if (!(err instanceof ProtocolViolation)) throw err; + console.warn("malformed datagram", err); + this.#violated(err); + } + } + /** * Handles unidirectional streams for media delivery (groups). */ diff --git a/js/net/src/ietf/datagram.test.ts b/js/net/src/ietf/datagram.test.ts new file mode 100644 index 0000000000..a97131f80f --- /dev/null +++ b/js/net/src/ietf/datagram.test.ts @@ -0,0 +1,112 @@ +import { expect, test } from "bun:test"; +import { ProtocolViolation } from "../error.ts"; +import { encodeVarint } from "../stream.ts"; +import { ObjectDatagram } from "./datagram.ts"; +import { type IetfVersion, Version } from "./version.ts"; + +const ALL: IetfVersion[] = [ + Version.DRAFT_14, + Version.DRAFT_15, + Version.DRAFT_16, + Version.DRAFT_17, + Version.DRAFT_18, + Version.DRAFT_19, + Version.DRAFT_20, + Version.DRAFT_21, + Version.DRAFT_22, +]; + +const LEGACY: IetfVersion[] = [Version.DRAFT_14, Version.DRAFT_15, Version.DRAFT_16]; + +function single(publisherPriority?: number): ObjectDatagram { + return new ObjectDatagram({ + trackAlias: 3n, + groupId: 42, + publisherPriority, + endOfGroup: true, + properties: Uint8Array.of(0x10, 0x05), + body: { payload: new TextEncoder().encode("hello") }, + }); +} + +async function rejects(bytes: Uint8Array, version: IetfVersion) { + expect(await ObjectDatagram.decode(bytes, version).catch((err: unknown) => err)).toBeInstanceOf(ProtocolViolation); +} + +test("roundtrips on every draft", async () => { + for (const version of ALL) { + const datagram = single(7); + const bytes = datagram.encode(version); + expect(bytes[0]).toBe(0x07); // properties, end of group, object 0 + expect(await ObjectDatagram.decode(bytes, version)).toEqual(datagram); + } +}); + +test("a default priority needs draft-15", async () => { + expect(() => single().encode(Version.DRAFT_14)).toThrow(); + await rejects(Uint8Array.of(0x0c, 0x01, 0x02), Version.DRAFT_14); + + const bytes = single().encode(Version.DRAFT_16); + expect(bytes[0]).toBe(0x0f); + expect(await ObjectDatagram.decode(bytes, Version.DRAFT_16)).toEqual(single()); +}); + +test("an explicit object id and a status", async () => { + const datagram = new ObjectDatagram({ + trackAlias: 1n, + groupId: 2, + objectId: 0, + publisherPriority: 128, + endOfGroup: false, + body: { status: 0 }, + }); + for (const version of ALL) { + const bytes = datagram.encode(version); + expect(bytes[0]).toBe(0x20); + expect(await ObjectDatagram.decode(bytes, version)).toEqual(datagram); + } +}); + +test("rejects invalid types", async () => { + for (const version of ALL) { + // A status that ends the group, the reserved bit, a bit past the defined ones, and a + // Type too wide for any of them. + for (const kind of [0x22, 0x10, 0x40, 2 ** 40]) { + const type = encodeVarint(kind, version); + const bytes = new Uint8Array(type.byteLength + 4); + bytes.set(type); + bytes.set([0x01, 0x02, 0x03, 0x04], type.byteLength); + await rejects(bytes, version); + } + } +}); + +test("a status with properties needs a Normal Object past draft-16", async () => { + const datagram = new ObjectDatagram({ + trackAlias: 1n, + groupId: 2, + objectId: 0, + publisherPriority: 0, + endOfGroup: false, + properties: Uint8Array.of(0x10, 0x05), + body: { status: 3 }, + }); + for (const version of ALL) { + if (LEGACY.includes(version)) { + expect(await ObjectDatagram.decode(datagram.encode(version), version)).toEqual(datagram); + } else { + expect(() => datagram.encode(version)).toThrow(); + // The same bytes a legacy draft writes, which these drafts refuse. + await rejects(datagram.encode(Version.DRAFT_16), version); + } + } +}); + +test("rejects an empty properties block, a truncated header, and bytes after a status", async () => { + // PROPERTIES and ZERO_OBJECT_ID, alias 1, group 2, priority 0, empty block. + await rejects(Uint8Array.of(0x05, 0x01, 0x02, 0x00, 0x00, 0x78), Version.DRAFT_16); + // The group ID is missing. + await rejects(Uint8Array.of(0x04, 0x01), Version.DRAFT_16); + // A Normal status, then a stray byte. + await rejects(Uint8Array.of(0x24, 0x01, 0x02, 0x00, 0x00, 0x00), Version.DRAFT_16); +}); diff --git a/js/net/src/ietf/datagram.ts b/js/net/src/ietf/datagram.ts new file mode 100644 index 0000000000..509e320963 --- /dev/null +++ b/js/net/src/ietf/datagram.ts @@ -0,0 +1,189 @@ +/** + * OBJECT_DATAGRAM: one Object carried in a QUIC datagram (draft-14 section 10.3.1 through + * draft-20 section 11.3.1). + * + * The model counterpart is {@link Datagram}, a single-frame group, so only an Object at ID 0 + * maps onto it. The Type is a set of flags on every draft; draft-14 lacks the + * DEFAULT_PRIORITY bit and a status with an omitted Object ID. Mirrors the Rust + * `ietf::ObjectDatagram`. + * + * @module + */ +import type { Datagram } from "../datagram.ts"; +import { error, ProtocolViolation, reason } from "../error.ts"; +import { type Cursor, encodeVarint, Reader } from "../stream.ts"; +import type { Timescale } from "../time.ts"; +import { encodeObjectExtensions } from "./object.ts"; +import { type IetfVersion, Version } from "./version.ts"; + +// The bits of an OBJECT_DATAGRAM Type. +const PROPERTIES = 0x01; +const END_OF_GROUP = 0x02; +const ZERO_OBJECT_ID = 0x04; +const DEFAULT_PRIORITY = 0x08; +const STATUS = 0x20; +// Every defined bit. Anything else, including the reserved 0x10, is invalid. +const ALL = PROPERTIES | END_OF_GROUP | ZERO_OBJECT_ID | DEFAULT_PRIORITY | STATUS; + +/** Whether `kind` is a Type this draft defines. */ +function valid(kind: number, version: IetfVersion): boolean { + if (version === Version.DRAFT_14) return kind <= 0x07 || kind === 0x20 || kind === 0x21; + // A status cannot also end the group. + return (kind & ~ALL) === 0 && !((kind & STATUS) !== 0 && (kind & END_OF_GROUP) !== 0); +} + +/** Drafts 14-16 let any status carry Properties; later ones only a Normal Object. */ +function legacy(version: IetfVersion): boolean { + return version === Version.DRAFT_14 || version === Version.DRAFT_15 || version === Version.DRAFT_16; +} + +/** What follows an OBJECT_DATAGRAM's header: the payload, or the status of an Object without one. */ +export type DatagramBody = { payload: Uint8Array } | { status: number }; + +/** A decoded OBJECT_DATAGRAM. */ +export class ObjectDatagram { + trackAlias: bigint; + groupId: number; + /** The Object ID, or `undefined` when the ZERO_OBJECT_ID bit omits it (Object 0). */ + objectId?: number; + /** The Publisher Priority, or `undefined` to inherit the subscription's (draft-15+). */ + publisherPriority?: number; + /** No Object past this one exists in the group. */ + endOfGroup: boolean; + /** The Object Properties block without its length prefix, which carries the Timestamp. */ + properties?: Uint8Array; + body: DatagramBody; + + constructor(props: { + trackAlias: bigint; + groupId: number; + objectId?: number; + publisherPriority?: number; + endOfGroup: boolean; + properties?: Uint8Array; + body: DatagramBody; + }) { + this.trackAlias = props.trackAlias; + this.groupId = props.groupId; + this.objectId = props.objectId; + this.publisherPriority = props.publisherPriority; + this.endOfGroup = props.endOfGroup; + this.properties = props.properties; + this.body = props.body; + } + + /** Encode one QUIC datagram's bytes. Throws on a combination this draft cannot carry. */ + encode(version: IetfVersion): Uint8Array { + let kind = 0; + if (this.properties !== undefined) kind |= PROPERTIES; + if (this.endOfGroup) kind |= END_OF_GROUP; + if (this.objectId === undefined) kind |= ZERO_OBJECT_ID; + if (this.publisherPriority === undefined) kind |= DEFAULT_PRIORITY; + if ("status" in this.body) kind |= STATUS; + if (!valid(kind, version)) throw new Error(`OBJECT_DATAGRAM type 0x${kind.toString(16)} is invalid`); + + const parts = [encodeVarint(kind, version), encodeVarint(this.trackAlias, version)]; + parts.push(encodeVarint(this.groupId, version)); + if (this.objectId !== undefined) parts.push(encodeVarint(this.objectId, version)); + if (this.publisherPriority !== undefined) parts.push(Uint8Array.of(this.publisherPriority)); + if (this.properties !== undefined) { + // A present but empty block is a protocol violation for the peer. + if (this.properties.byteLength === 0) throw new Error("OBJECT_DATAGRAM properties are empty"); + parts.push(encodeVarint(this.properties.byteLength, version), this.properties); + } + if ("status" in this.body) { + if (!legacy(version) && this.body.status !== 0 && this.properties !== undefined) { + throw new Error("only a Normal OBJECT_DATAGRAM may carry properties"); + } + parts.push(encodeVarint(this.body.status, version)); + } else { + // Runs to the datagram boundary: written raw, no length prefix. + parts.push(this.body.payload); + } + + const out = new Uint8Array(parts.reduce((total, part) => total + part.byteLength, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; + } + + /** + * Decode one QUIC datagram's bytes. Anything malformed is the peer breaking the protocol, + * so every failure is a {@link ProtocolViolation}. + */ + static async decode(data: Uint8Array, version: IetfVersion): Promise { + try { + return await new Reader(undefined, data, version).decode((c) => decodeFields(c, version)); + } catch (err: unknown) { + if (err instanceof ProtocolViolation) throw err; + throw new ProtocolViolation(`malformed OBJECT_DATAGRAM: ${reason(error(err))}`, { cause: err }); + } + } +} + +/** + * Encode a datagram the way a publisher sends it: Object 0 of a group that has no other, so it + * ends its group, stamped in `timescale` units when the subscription declared one. + */ +export async function encodeDatagram( + datagram: Datagram, + options: { trackAlias: bigint; publisherPriority: number; timescale?: Timescale }, + version: IetfVersion, +): Promise { + const properties = + options.timescale !== undefined + ? await encodeObjectExtensions(datagram.timestamp, options.timescale, version) + : undefined; + return new ObjectDatagram({ + trackAlias: options.trackAlias, + groupId: datagram.sequence, + publisherPriority: options.publisherPriority, + endOfGroup: true, + properties: properties?.byteLength ? properties : undefined, + body: { payload: datagram.payload }, + }).encode(version); +} + +function decodeFields(c: Cursor, version: IetfVersion): ObjectDatagram { + // Full width, so an out-of-range Type is refused rather than overflowing. + const raw = c.u62(); + const kind = raw <= 0xffn ? Number(raw) : -1; + if (kind < 0 || !valid(kind, version)) { + throw new ProtocolViolation(`invalid OBJECT_DATAGRAM type 0x${raw.toString(16)}`); + } + + const trackAlias = c.u62(); + const groupId = c.u53(); + const objectId = kind & ZERO_OBJECT_ID ? undefined : c.u53(); + const publisherPriority = kind & DEFAULT_PRIORITY ? undefined : c.u8(); + let properties: Uint8Array | undefined; + if (kind & PROPERTIES) { + properties = c.read(c.u53()); + if (properties.byteLength === 0) throw new ProtocolViolation("OBJECT_DATAGRAM properties are empty"); + } + + let body: DatagramBody; + if (kind & STATUS) { + const status = c.u53(); + if (!legacy(version) && status !== 0 && properties !== undefined) { + throw new ProtocolViolation("only a Normal OBJECT_DATAGRAM may carry properties"); + } + if (c.remaining > 0) throw new ProtocolViolation(`OBJECT_DATAGRAM has ${c.remaining} trailing bytes`); + body = { status }; + } else { + body = { payload: c.read(c.remaining) }; + } + + return new ObjectDatagram({ + trackAlias, + groupId, + objectId, + publisherPriority, + endOfGroup: (kind & END_OF_GROUP) !== 0, + properties, + body, + }); +} diff --git a/js/net/src/ietf/index.ts b/js/net/src/ietf/index.ts index 97df0698fc..f475cadc0b 100644 --- a/js/net/src/ietf/index.ts +++ b/js/net/src/ietf/index.ts @@ -1,6 +1,7 @@ export * from "./adapter.ts"; export * as Cluster from "./cluster.ts"; export * from "./connection.ts"; +export * from "./datagram.ts"; export * from "./fetch.ts"; export * from "./goaway.ts"; export * from "./hidden.ts"; diff --git a/js/net/src/ietf/object.ts b/js/net/src/ietf/object.ts index dcbcb96c89..84ade41653 100644 --- a/js/net/src/ietf/object.ts +++ b/js/net/src/ietf/object.ts @@ -68,8 +68,12 @@ async function encodeObjectTime( await w.u62(BigInt(value)); } -// Without units (`timescale` undefined) there is no Timestamp to write, even if the frame has one. -async function encodeObjectExtensions( +/** + * Encode an object's Properties block: its Timestamp in `timescale` units, or nothing. + * + * Without units (`timescale` undefined) there is no Timestamp to write, even if the frame has one. + */ +export async function encodeObjectExtensions( timestamp: Timestamp | undefined, timescale: Timescale | undefined, version: IetfVersion, @@ -101,7 +105,8 @@ async function encodeObjectExtensions( return result; } -function decodeObjectTime(c: Cursor, timescale: Timescale): Timestamp | undefined { +/** Decode the Timestamp an object's Properties block carries, consuming the whole block. */ +export function decodeObjectTime(c: Cursor, timescale: Timescale): Timestamp | undefined { let timestamp: bigint | undefined; let prevType = 0n; let first = true; diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index f43de91120..54193f51f0 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -1,6 +1,7 @@ import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import type * as broadcast from "../broadcast.ts"; import { Withdrawal } from "../connection/withdrawal.ts"; +import * as DatagramStream from "../datagram_stream.ts"; import { controlTimeout, error, reason, StreamCode, StreamError } from "../error.ts"; import type * as group from "../group.ts"; import type { Hop, Route } from "../hop.ts"; @@ -15,6 +16,7 @@ import * as Varint from "../varint.ts"; import { type Advertised, type Advertisements, wireOf } from "../wire.ts"; import type { Session } from "./adapter.ts"; import * as Cluster from "./cluster.ts"; +import { encodeDatagram } from "./datagram.ts"; import { type RequestKind, requestReason, toRequestCode } from "./error.ts"; import { type Fetch, FetchError, FetchHeader } from "./fetch.ts"; import * as Filter from "./filter.ts"; @@ -232,6 +234,11 @@ export class Publisher { #quic: WebTransport; #session: Session; #requiresSolicitation: boolean; + + // The one writer for the outbound datagram stream, shared by every subscription since a + // second getWriter would throw. Undefined when the transport carries no datagrams, which + // turns them off: there is no stream fallback. Released in close(). + #datagramWriter?: WritableStreamDefaultWriter; #hidden: boolean; // The origin this session serves, borrowed: it outlives the session, and closing the @@ -281,6 +288,18 @@ export class Publisher { this.#requiresSolicitation = requiresSolicitation; this.#hidden = hidden; this.#advert = Cluster.advertise(cluster); + this.#datagramWriter = DatagramStream.datagramWriter(quic); + } + + /** + * Release the datagram writer's lock so the transport can be torn down. The broadcasts + * belong to the origin, which outlives this session. + * + * @internal + */ + close() { + this.#datagramWriter?.releaseLock(); + this.#datagramWriter = undefined; } /** @@ -532,6 +551,10 @@ export class Publisher { } })(); + // Datagrams count toward no stream, so PUBLISH_DONE never waits on them; closing + // the track below ends the loop. + void this.#runDatagrams({ requestId: msg.requestId, track, timescale, publisherPriority }); + // The fill (when one was requested) runs alongside on its own fetch stream; its // failures reset that stream and never touch the subscription. const filling = @@ -709,6 +732,48 @@ export class Publisher { } } + /** + * Send a track's datagrams as OBJECT_DATAGRAMs, best-effort, like moq-lite. + * + * One that does not fit the transport's limit is dropped; there is no stream fallback. A + * send failure stops the loop but never fails the subscription. + */ + async #runDatagrams(options: { + requestId: bigint; + track: TrackSubscriber; + /** The units datagrams are stamped in, if any; see {@link RunGroup.timescale}. */ + timescale: Timescale | undefined; + publisherPriority: number; + }) { + const writer = this.#datagramWriter; + if (!writer) return; + const { requestId, track, timescale, publisherPriority } = options; + const version = this.#session.version; + + try { + for (;;) { + const datagram = await track.recvDatagram(); + if (!datagram) return; + + const body = await encodeDatagram( + datagram, + { trackAlias: requestId, publisherPriority, timescale }, + version, + ); + const max = DatagramStream.maxDatagramSize(this.#quic); + if (body.byteLength > max) { + console.debug(`dropping oversize datagram: alias=${requestId} size=${body.byteLength} max=${max}`); + continue; + } + + await writer.ready; + await writer.write(body); + } + } catch (err: unknown) { + console.debug(`datagram send stopped: alias=${requestId} error=${reason(error(err))}`); + } + } + /** * Mark the track's end with an END_OF_TRACK object on its own stream, at object 0 of the * group that will never exist. diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index 8d602aa196..e67b56d016 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -6,12 +6,13 @@ import { hooks } from "../internal.ts"; import { createMockTransportPair } from "../mock.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; -import { Timescale } from "../time.ts"; +import { Timescale, Timestamp } from "../time.ts"; import type * as track from "../track.ts"; import { ControlStreamAdapter, NativeSession } from "./adapter.ts"; import type * as Cluster from "./cluster.ts"; import { Connection } from "./connection.ts"; -import { type GroupFlags, Group as GroupMessage } from "./object.ts"; +import { ObjectDatagram } from "./datagram.ts"; +import { encodeObjectExtensions, type GroupFlags, Group as GroupMessage } from "./object.ts"; import type { Properties } from "./properties.ts"; import { PublishNamespace, PublishNamespaceUpdate } from "./publish_namespace.ts"; import { RequestError, RequestOk } from "./request.ts"; @@ -1622,3 +1623,61 @@ test("object extension limit accepts 64 KiB and stops one byte over before readi } } }); + +/** + * An OBJECT_DATAGRAM at object 0 is a datagram at its Group ID; anything the model cannot + * carry is dropped like a lost datagram, and a malformed one ends the session. + */ +test("an object datagram is a datagram group", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const subscriber = new Subscriber({ session: new NativeSession(pair.server, VERSION, true), quic: pair.server }); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + + const peer = await nextStream(pair.client); + if (!peer) throw new Error("the subscriber never opened a subscribe stream"); + expect(await peer.reader.u53()).toBe(Subscribe.id); + const request = await Subscribe.decode(peer.reader, VERSION); + await peer.writer.u53(SubscribeOk.id); + await new SubscribeOk({ + requestId: request.requestId, + trackAlias: ALIAS, + properties: { timescale: Timescale.MILLI }, + }).encode(peer.writer, VERSION); + await track.info(); + + const receiving = subscriber.runDatagrams(); + const writer = pair.client.datagrams.writable.getWriter(); + const send = (fields: Partial[0]>) => + writer.write( + new ObjectDatagram({ + trackAlias: ALIAS, + groupId: 4, + endOfGroup: true, + body: { payload: new TextEncoder().encode("no") }, + ...fields, + }).encode(VERSION), + ); + + // Past object 0, an unbound alias, a status other than Normal, and no Timestamp on a timed track. + await send({ objectId: 1 }); + await send({ trackAlias: ALIAS + 1n }); + await send({ endOfGroup: false, body: { status: 3 } }); + await send({ groupId: 5, objectId: 0 }); + const properties = await encodeObjectExtensions(Timestamp.fromMillis(1234), Timescale.MILLI, VERSION); + await send({ groupId: 9, objectId: 0, publisherPriority: 7, properties, body: { payload: Uint8Array.of(1) } }); + + const datagram = await track.recvDatagram(); + expect(datagram?.sequence).toBe(9); + expect(datagram?.payload).toEqual(Uint8Array.of(1)); + expect(datagram?.timestamp?.as(Timescale.MILLI)).toBe(1234); + + // A status cannot end the group. + await writer.write(Uint8Array.of(0x22, Number(ALIAS), 4, 0, 0)); + expect( + await receiving.then( + () => undefined, + (err: unknown) => err, + ), + ).toBeInstanceOf(ProtocolViolation); + track.close(); +}); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 76f7cacf76..76d775fb26 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -3,6 +3,7 @@ import * as announce from "../announced.ts"; import * as broadcast from "../broadcast.ts"; import type { Drain } from "../connection/goaway.ts"; import { BroadcastCache } from "../consume.ts"; +import * as DatagramStream from "../datagram_stream.ts"; import { closeError, controlTimeout, @@ -17,17 +18,18 @@ import * as netGroup from "../group.ts"; import { Cost, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; import { hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; import * as Path from "../path.ts"; -import { type Cursor, type Reader, type Stream, UnexpectedEnd } from "../stream.ts"; +import { type Cursor, Reader, type Stream, UnexpectedEnd } from "../stream.ts"; import { Tail } from "../tail.ts"; -import { Milli, type Timescale } from "../time.ts"; +import { Milli, type Timescale, type Timestamp } from "../time.ts"; import type * as track from "../track.ts"; import { TimeoutError, withTimeout } from "../util/timeout.ts"; import { overrideBroadcastWire, wireOf } from "../wire.ts"; import type { Session } from "./adapter.ts"; import { DuplicateTrackAlias, RetiredTrackAlias, TrackAliases } from "./aliases.ts"; import * as Cluster from "./cluster.ts"; +import { ObjectDatagram } from "./datagram.ts"; import { requestReason, toRequestCode } from "./error.ts"; -import { Frame, type Group as GroupMessage, hasFirstObjectBit, ObjectIdGap } from "./object.ts"; +import { decodeObjectTime, Frame, type Group as GroupMessage, hasFirstObjectBit, ObjectIdGap } from "./object.ts"; import { fromWire, toWire } from "./priority.ts"; import { type Publish, PublishDone, PublishError, publishDoneClean } from "./publish.ts"; import { @@ -1232,6 +1234,94 @@ export class Subscriber { read(); } } + + /** + * Receive QUIC datagrams, each an OBJECT_DATAGRAM for one of our subscriptions. + * + * Returns at once on a transport without datagrams, and once the datagram stream ends or + * fails. A malformed datagram throws a {@link ProtocolViolation}, which ends the session. + * + * @internal + */ + async runDatagrams(): Promise { + if (!this.#quic || DatagramStream.maxDatagramSize(this.#quic) === 0) return; + const reader = DatagramStream.datagramReader(this.#quic); + if (!reader) return; + + try { + for (;;) { + // The stream errors once the session closes, which ends this loop like any other. + const next = await reader.read().catch(() => undefined); + if (!next || next.done) return; + await this.#recvDatagram(next.value); + } + } finally { + reader.releaseLock(); + } + } + + /** + * Deliver one OBJECT_DATAGRAM as a datagram on its subscription's track: a single-frame + * group at the Group ID. + * + * One the model cannot carry is dropped like any lost datagram: an Object past ID 0 (the + * group would need a second object), a status other than Normal, an alias that is not + * bound yet (the draft lets us drop rather than buffer), or no Timestamp on a track that + * declared a timescale. + */ + async #recvDatagram(data: Uint8Array): Promise { + const version = this.#session.version; + const datagram = await ObjectDatagram.decode(data, version); + const { trackAlias: alias, groupId: sequence } = datagram; + + if ((datagram.objectId ?? 0) !== 0) { + console.debug(`dropping a datagram past object 0: alias=${alias} group=${sequence}`); + return; + } + let payload: Uint8Array; + if ("status" in datagram.body) { + if (datagram.body.status !== 0) { + console.debug( + `dropping a datagram status: alias=${alias} group=${sequence} status=${datagram.body.status}`, + ); + return; + } + payload = new Uint8Array(); + } else { + payload = datagram.body.payload; + } + + const subscription = this.#aliases.peek(alias); + if (!subscription) { + console.debug(`dropping a datagram for an unbound alias: alias=${alias} group=${sequence}`); + return; + } + + // Like a subgroup object: a track that declared no timescale is untimed. + const timescale = this.#timescales.get(alias); + let timestamp: Timestamp | undefined; + if (timescale !== undefined && datagram.properties !== undefined) { + try { + timestamp = await new Reader(undefined, datagram.properties, version).decode((c) => + decodeObjectTime(c, timescale), + ); + } catch (err: unknown) { + throw new ProtocolViolation(`malformed OBJECT_DATAGRAM properties: ${reason(error(err))}`, { + cause: err, + }); + } + } + if (timescale !== undefined && timestamp === undefined) { + console.debug(`dropping an unstamped datagram: alias=${alias} group=${sequence}`); + return; + } + + try { + subscription.track.insertDatagram(sequence, timestamp, payload); + } catch (err: unknown) { + console.debug(`dropping datagram: alias=${alias} group=${sequence} error=${reason(error(err))}`); + } + } } /** diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index 2a0b8edb12..8122b17b43 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -624,38 +624,58 @@ test("integration: lite draft-05 missing datagram writer does not close streams" server.abort(); }); -test("integration: ietf does not deliver datagrams", async () => { - const enc = new TextEncoder(); - const pair = createMockTransportPair(Ietf.ALPN.DRAFT_19); - const origin = new OriginProducer(); - - const [client, server] = await Promise.all([ - connect(url, { transport: pair.client }), - accept(pair.server, url, { publish: origin.consume() }), - ]); - - const broadcast = publish(origin, Path.from("test")); - const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); - - const remote = wireOf(client).consume(Path.from("test")); - const track = remote.track("video").subscribe().ordered(); - const datagrams = remote.track("video").subscribe(); - - producer.insertDatagram(7, Timestamp.fromMillis(0), enc.encode("dgram")); - producer.writeString("group"); - - expect(await track.readString()).toBe("group"); - - const datagram = datagrams.recvDatagram(); - datagram.catch(() => {}); - const outcome = await Promise.race([datagram, sleep(50).then(() => "timeout" as const)]); - expect(outcome).toBe("timeout"); +for (const [name, protocol, version] of [ + ["draft-14", "", Ietf.Version.DRAFT_14], + ["draft-15", Ietf.ALPN.DRAFT_15, undefined], + ["draft-16", Ietf.ALPN.DRAFT_16, undefined], + ["draft-17", Ietf.ALPN.DRAFT_17, undefined], + ["draft-18", Ietf.ALPN.DRAFT_18, undefined], + ["draft-19", Ietf.ALPN.DRAFT_19, undefined], + ["draft-20", Ietf.ALPN.DRAFT_20, undefined], + ["draft-21", Ietf.ALPN.DRAFT_21, undefined], + ["draft-22", Ietf.ALPN.DRAFT_22, undefined], +] as const) { + for (const timed of [true, false]) { + test(`integration: ietf ${name} delivers ${timed ? "timed" : "untimed"} datagrams`, async () => { + const enc = new TextEncoder(); + const dec = new TextDecoder(); + const pair = createMockTransportPair(protocol); + const origin = new OriginProducer(); + + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { version, publish: origin.consume() }), + ]); + + const broadcast = publish(origin, Path.from("test")); + const producer = broadcast.createTrack("video", timed ? { timescale: Timescale.MILLI } : {}); + + const remote = wireOf(client).consume(Path.from("test")); + const track = remote.track("video").subscribe().ordered(); + const datagrams = remote.track("video").subscribe(); + + // A group first, so the subscription is serving and its alias is bound on both ends. + producer.writeFrame({ payload: enc.encode("group"), timestamp: timed ? Timestamp.now() : undefined }); + expect(await track.readString()).toBe("group"); + + producer.insertDatagram(7, timed ? Timestamp.fromMillis(1234) : undefined, enc.encode("dgram")); + const datagram = await withTimeout(datagrams.recvDatagram(), 1000, "no datagram arrived"); + expect(datagram?.sequence).toBe(7); + expect(dec.decode(datagram?.payload)).toBe("dgram"); + // Drafts 14-16 cannot declare TIMESCALE, so their datagrams arrive untimed. + if (timed && !["draft-14", "draft-15", "draft-16"].includes(name)) { + expect(datagram?.timestamp?.as(Timescale.MILLI)).toBe(1234); + } else { + expect(datagram?.timestamp).toBeUndefined(); + } - broadcast.close(); - remote.close(); - client.abort(); - server.abort(); -}); + broadcast.close(); + remote.close(); + client.abort(); + server.abort(); + }); + } +} test("integration: lite draft-05 missing datagram reader does not close streams", async () => { const enc = new TextEncoder(); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 756dcefd87..d5557cd3cf 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1,6 +1,7 @@ import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import type * as broadcast from "../broadcast.ts"; import { Withdrawal } from "../connection/withdrawal.ts"; +import * as DatagramStream from "../datagram_stream.ts"; import { error, NotFound, reason, StreamCode, StreamError } from "../error.ts"; import type * as group from "../group.ts"; import { Cost, type Hop, type Route, routesEqual } from "../hop.ts"; @@ -13,7 +14,6 @@ import type * as track from "../track.ts"; import { type Advertised, type Advertisements, wireOf } from "../wire.ts"; import { AnnounceInit, AnnounceOk, type AnnounceRequest, encodeAnnounceBroadcast } from "./announce.ts"; import { Datagram as DatagramMessage } from "./datagram.ts"; -import * as DatagramStream from "./datagram_stream.ts"; import type { Fetch } from "./fetch.ts"; import { Group as GroupMessage } from "./group.ts"; import { Priority, sendOrder } from "./priority.ts"; diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 75c7337f29..2b2f11e4bb 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -4,6 +4,7 @@ import * as broadcast from "../broadcast.ts"; import type { Drain } from "../connection/goaway.ts"; import type { Probe as ProbeStats } from "../connection/stats.ts"; import { BroadcastCache } from "../consume.ts"; +import * as DatagramStream from "../datagram_stream.ts"; import type * as Epoch from "../epoch.ts"; import { controlTimeout, error, ProtocolViolation, reason, StreamCode, StreamError, sessionCause } from "../error.ts"; import * as netGroup from "../group.ts"; @@ -25,7 +26,6 @@ import { decodeAnnounceBroadcastMaybe, } from "./announce.ts"; import { Datagram as DatagramMessage } from "./datagram.ts"; -import * as DatagramStream from "./datagram_stream.ts"; import { Fetch as FetchMessage } from "./fetch.ts"; import { frameDecoder, type Group as GroupMessage, readFrames } from "./group.ts"; import { sendOrder } from "./priority.ts"; diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 77392b491d..5f1d9275b8 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -930,7 +930,7 @@ export class Producer { * never reuses a number). The payload must fit the negotiated transport datagram size minus * a small header; an oversize payload is dropped at each hop (there is no group fallback), so * keep datagram payloads small (e.g. a single audio frame). Datagrams are never delivered - * over IETF moq-transport or stream-only transports (the WebSocket fallback). A payload over + * over stream-only transports (the WebSocket fallback). A payload over * 65535 bytes (the QUIC datagram frame ceiling) throws. An origin publisher uses this; a * relay preserving upstream numbering uses {@link insertDatagram}. * diff --git a/quest/m1/README.md b/quest/m1/README.md index 0d07d74492..ab753ae383 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -85,7 +85,6 @@ blocks. The quests that gated m0 lines moved under them. - [FETCH_OK properties](/quest/m1/fetch-ok-properties.md) - our FETCH_OK carries the track properties SUBSCRIBE_OK does, as draft 16+ requires - [Fetch without SUBSCRIBE](/quest/m1/ietf-fetch-only.md) - a relay fetches from an IETF upstream without subscribing, finished tracks included, with End of Track always reported - [IETF object gaps](/quest/m1/ietf-object-gaps.md) - a gapped object ID is refused loudly like a subgroup in Rust and JS, and an overflowing one closes the session in Rust -- [JS IETF datagrams](/quest/m1/js-ietf-datagram.md) - `@moq/net` sends and receives datagram groups over moq-transport, like Rust - [JavaScript FETCH](/quest/m1/js-fetch.md) - browser publishers answer IETF FETCH through the JS ranges request surface - [Relay session limits](/quest/m1/relay-session-limits.md) - moq-relay sets per-session request limits, tighter for clients than peers, and the bindings name a refused request - [Churn with held subscriptions](/quest/m1/session-churn-held.md) - opening and closing a request costs the same with 1 or 1,024 held subscriptions diff --git a/quest/m1/js-ietf-datagram.md b/quest/m1/js-ietf-datagram.md deleted file mode 100644 index 0326b949ca..0000000000 --- a/quest/m1/js-ietf-datagram.md +++ /dev/null @@ -1,17 +0,0 @@ -# [M] JavaScript moq-transport datagrams - -## Goal - -`@moq/net` sends and receives datagrams over moq-transport as -`OBJECT_DATAGRAM`, matching Rust: one Object at ID 0 is a single-frame group -whose Group ID is the sequence. A JavaScript publisher's datagrams reach a Rust -subscriber, and a Rust publisher's reach a JavaScript one. - -## Plan - -Port `rs/moq-net/src/ietf/datagram.rs` and the session's send and receive -loops. Decode every draft's Type flags, drop what the model cannot carry the -same way Rust does, and close the session on a malformed Type. - -The integration test `ietf does not deliver datagrams` flips to delivery on -every supported draft, and `just test interop --all` covers both directions. diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 83072be4ff..1b8cee78a0 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -174,3 +174,109 @@ fn lite_varint_interop() { } } } + +#[derive(serde::Deserialize)] +struct DatagramEcho { + echoes: Vec>, + /// The Timestamp each datagram's Properties carry, in milliseconds, if any. + timestamps: Vec>, + /// The OBJECT_DATAGRAM the JS publisher sends for the first datagram's fields. + published: Vec, +} + +/// OBJECT_DATAGRAM on every draft, both ways: JS decodes what Rust encodes, including the +/// Timestamp, and re-encodes it byte for byte; Rust decodes what the JS publisher writes. +#[test] +#[ignore = "requires Bun; run by just test interop"] +fn ietf_datagram_interop() { + use crate::coding::Decode; + use crate::{Timescale, Timestamp}; + + let all = [ + ietf::Version::Draft14, + ietf::Version::Draft15, + ietf::Version::Draft16, + ietf::Version::Draft17, + ietf::Version::Draft18, + ietf::Version::Draft19, + ietf::Version::Draft20, + ietf::Version::Draft21, + ietf::Version::Draft22, + ]; + for (draft, version) in (14..).zip(all) { + let legacy = matches!( + version, + ietf::Version::Draft14 | ietf::Version::Draft15 | ietf::Version::Draft16 + ); + let mut properties = Vec::new(); + let mut w = Encoder::new(&mut properties, version.into()); + ietf::encode_object_time(&mut w, Timestamp::from_millis(1234).unwrap(), Timescale::MILLI, version).unwrap(); + + // What a publisher sends: Object 0, explicit priority, ending its group, stamped. + let mut datagrams = vec![ietf::ObjectDatagram { + track_alias: 3, + group_id: 42, + object_id: None, + publisher_priority: Some(7), + end_of_group: true, + properties: Some(properties.clone()), + body: ietf::DatagramBody::Payload(bytes::Bytes::from_static(b"hello")), + }]; + datagrams.push(ietf::ObjectDatagram { + track_alias: 1, + group_id: 2, + object_id: Some(0), + publisher_priority: Some(128), + end_of_group: false, + properties: None, + body: ietf::DatagramBody::Status(0), + }); + if version != ietf::Version::Draft14 { + datagrams.push(ietf::ObjectDatagram { + track_alias: (1 << 40) + 1, + group_id: (1 << 50) + 1, + object_id: Some(1 << 20), + publisher_priority: None, + end_of_group: true, + properties: None, + body: ietf::DatagramBody::Payload(bytes::Bytes::new()), + }); + } + if legacy { + datagrams.push(ietf::ObjectDatagram { + track_alias: 1, + group_id: 2, + object_id: Some(0), + publisher_priority: Some(0), + end_of_group: false, + properties: Some(properties.clone()), + body: ietf::DatagramBody::Status(3), + }); + } + let encoded: Vec> = datagrams + .iter() + .map(|datagram| datagram.encode_bytes(version).unwrap().to_vec()) + .collect(); + + let input = serde_json::json!({ "draft": draft, "datagrams": encoded }); + let echo: DatagramEcho = + serde_json::from_slice(&bun("ietf-datagram.ts", input)).expect("JS returned its encodings"); + + assert_eq!(echo.echoes, encoded, "{version}: JS re-encoded differently"); + let expected: Vec> = datagrams + .iter() + .map(|datagram| datagram.properties.as_ref().map(|_| 1234)) + .collect(); + assert_eq!(echo.timestamps, expected, "{version}: JS decoded different timestamps"); + + let (published, used) = ietf::ObjectDatagram::decode_slice(&echo.published, version).unwrap(); + assert_eq!(used, echo.published.len(), "{version}: a datagram runs to its end"); + assert_eq!(published, datagrams[0], "{version}: the JS publisher's datagram"); + let Some(properties) = &published.properties else { + panic!("{version}: the JS publisher sent no Timestamp"); + }; + let mut r = Decoder::new(properties, version.into()); + let timestamp = ietf::decode_object_time(&mut r, Timescale::MILLI, version).unwrap(); + assert_eq!(timestamp.map(|t| t.as_millis()), Some(1234), "{version}: Timestamp"); + } +} diff --git a/test/interop/README.md b/test/interop/README.md index 0d87ea48d5..9297b20b5d 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -224,3 +224,8 @@ version dispatch: `lite-varint.ts` decodes Rust's lite-06 (QUIC) and lite-07 GROUP stream with frames, and re-encodes them byte for byte. Past 2^62-1 the range is per version: JS writes lite-07's 64-bit values, which Rust reads back, and JS refuses them on lite-06. + +`ietf_datagram_interop` does the same for moq-transport's `OBJECT_DATAGRAM` on +drafts 14 through 22: `ietf-datagram.ts` decodes Rust's datagrams and their +Timestamps and re-encodes them byte for byte, and Rust decodes the datagram the +JS publisher sends. diff --git a/test/interop/ietf-datagram.ts b/test/interop/ietf-datagram.ts new file mode 100644 index 0000000000..81956218a7 --- /dev/null +++ b/test/interop/ietf-datagram.ts @@ -0,0 +1,34 @@ +/** Decode Rust's OBJECT_DATAGRAMs and echo JS's encoding back, plus what the JS publisher sends. */ +import assert from "node:assert/strict"; +import { encodeDatagram, ObjectDatagram } from "../../js/net/src/ietf/datagram.ts"; +import { decodeObjectTime } from "../../js/net/src/ietf/object.ts"; +import type { IetfVersion } from "../../js/net/src/ietf/version.ts"; +import { Reader } from "../../js/net/src/stream.ts"; +import { Timescale, Timestamp } from "../../js/net/src/time.ts"; + +const input: { draft: number; datagrams: number[][] } = JSON.parse(process.argv[2]); +assert(input.draft >= 14 && input.draft <= 22, `unknown draft ${input.draft}`); +const version = (0xff000000 + input.draft) as IetfVersion; + +const echoes: number[][] = []; +const timestamps: (number | null)[] = []; +for (const bytes of input.datagrams) { + const datagram = await ObjectDatagram.decode(Uint8Array.from(bytes), version); + echoes.push([...datagram.encode(version)]); + + const properties = datagram.properties; + const timestamp = + properties && + (await new Reader(undefined, properties, version).decode((c) => decodeObjectTime(c, Timescale.MILLI))); + timestamps.push(timestamp ? timestamp.as(Timescale.MILLI) : null); +} + +// The fields of Rust's first datagram, sent the way the JS publisher sends a track's datagram. +const published = await encodeDatagram( + { sequence: 42, timestamp: Timestamp.fromMillis(1234), payload: new TextEncoder().encode("hello") }, + { trackAlias: 3n, publisherPriority: 7, timescale: Timescale.MILLI }, + version, +); + +// Stdout is the encoding channel back to Rust. +console.log(JSON.stringify({ echoes, timestamps, published: [...published] })); diff --git a/test/justfile b/test/justfile index 9939a6b639..bb90b53b92 100644 --- a/test/justfile +++ b/test/justfile @@ -33,10 +33,11 @@ harness: # client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. # Both runs first check js/net's varints against moq-net's encoder at every size boundary, -# and on lite-06/07 messages (the filter also matches `lite_varint_interop`). +# and on lite-06/07 messages (the filter also matches `lite_varint_interop`), then +# moq-transport OBJECT_DATAGRAMs both ways on every draft. [positional-arguments] interop *args: - just rs test -p moq-net --lib varint_interop --run-ignored all + just rs test -p moq-net --lib varint_interop ietf_datagram_interop --run-ignored all ./interop/interop.sh "$@" # Negative control: no publisher, every subscriber must time out (proves the