diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 767de121ea0..ab7391cf40e 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -79,19 +79,32 @@ typedef Instance = MoqInstance; /// A route over a prefix: its origin-relative path, required wildcard captures (empty without wildcards), and route metadata. typedef Announce = MoqAnnounce; -/// What an [AnnounceConsumer] yields: [AnnounceEventPresent] or -/// [AnnounceEventAbsent]. +/// What an [AnnounceConsumer] yields: [AnnounceEventPresent], +/// [AnnounceEventUnavailable], [AnnounceEventAvailable], [AnnounceEventAbsent], +/// or [AnnounceEventOffline]. /// /// Another publisher instance taking a path over keeps it present: hold a /// [Following] to see the broadcast change. typedef AnnounceEvent = MoqAnnounceEvent; -/// A route now covers the path, which was absent. +/// A route now covers the path, which was absent; it may arrive unavailable. typedef AnnounceEventPresent = PresentMoqAnnounceEvent; -/// No route covers the path any more; carries its last route. +/// The present path lost every usable path to its origin but may recover one: +/// it stays present while requests for it may fail. +typedef AnnounceEventUnavailable = UnavailableMoqAnnounceEvent; + +/// The present path recovered a usable path: request it again. +typedef AnnounceEventAvailable = AvailableMoqAnnounceEvent; + +/// The origin withdrew the path; carries its last route. typedef AnnounceEventAbsent = AbsentMoqAnnounceEvent; +/// This side let go of the path, which its origin may still serve: the session +/// it arrived on closed, or the network lost every path to it. Carries its last +/// route. +typedef AnnounceEventOffline = OfflineMoqAnnounceEvent; + /// The write side of a broadcast: publish tracks into it. typedef BroadcastProducer = MoqBroadcastProducer; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index be0743c7198..77c91be2a0b 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -222,7 +222,7 @@ final class Moq { MoqRoute route(String path, {RouteOptions? options}) => session.publish().route(path: path, options: options ?? RouteOptions()); - /// List the paths matching [options] as each becomes present or absent; + /// List the paths matching [options] as each becomes present, absent, or offline, or changes availability; /// prefixes stay relative to the origin. /// /// Listen to `announced(...).updates()` for a [Stream] of [AnnounceEvent] diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 38d53560f1b..3cca3b647d9 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -3011,7 +3011,12 @@ class FfiConverterMoqOriginConfig { class MoqRouteInfo { final int cost; final MoqInstance instance; - MoqRouteInfo({required this.cost, required this.instance}); + final bool available; + MoqRouteInfo({ + required this.cost, + required this.instance, + required this.available, + }); } class FfiConverterMoqRouteInfo { @@ -3031,8 +3036,13 @@ class FfiConverterMoqRouteInfo { ); final instance = instance_lifted.value; new_offset += instance_lifted.bytesRead; + final available_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final available = available_lifted.value; + new_offset += available_lifted.bytesRead; return LiftRetVal( - MoqRouteInfo(cost: cost, instance: instance), + MoqRouteInfo(cost: cost, instance: instance, available: available), new_offset - buf.offsetInBytes, ); } @@ -3041,6 +3051,7 @@ class FfiConverterMoqRouteInfo { final total_length = FfiConverterUInt64.allocationSize(value.cost) + FfiConverterMoqInstance.allocationSize(value.instance) + + FfiConverterBool.allocationSize(value.available) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -3057,12 +3068,17 @@ class FfiConverterMoqRouteInfo { value.instance, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterBool.write( + value.available, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } static int allocationSize(MoqRouteInfo value) { return FfiConverterUInt64.allocationSize(value.cost) + FfiConverterMoqInstance.allocationSize(value.instance) + + FfiConverterBool.allocationSize(value.available) + 0; } } @@ -8410,11 +8426,29 @@ class FfiConverterMoqAnnounceEvent { lifted.bytesRead - subview.offsetInBytes + 4, ); case 2: + final lifted = UnavailableMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + case 3: + final lifted = AvailableMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + case 4: final lifted = AbsentMoqAnnounceEvent.read(subview); return LiftRetVal( lifted.value, lifted.bytesRead - subview.offsetInBytes + 4, ); + case 5: + final lifted = OfflineMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); default: throw UniffiInternalError( UniffiInternalError.unexpectedEnumCase, @@ -8474,6 +8508,82 @@ class PresentMoqAnnounceEvent extends MoqAnnounceEvent { } } +class UnavailableMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + UnavailableMoqAnnounceEvent(MoqAnnounce this.announce); + UnavailableMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(UnavailableMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 2); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +class AvailableMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + AvailableMoqAnnounceEvent(MoqAnnounce this.announce); + AvailableMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(AvailableMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 3); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + class AbsentMoqAnnounceEvent extends MoqAnnounceEvent { final MoqAnnounce announce; AbsentMoqAnnounceEvent(MoqAnnounce this.announce); @@ -8502,7 +8612,45 @@ class AbsentMoqAnnounceEvent extends MoqAnnounceEvent { @override int write(Uint8List buf) { - buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 2); + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 4); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +class OfflineMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + OfflineMoqAnnounceEvent(MoqAnnounce this.announce); + OfflineMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(OfflineMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 5); int new_offset = buf.offsetInBytes + 4; new_offset += FfiConverterMoqAnnounce.write( announce, diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 3a401cbfb4f..558ceb34d04 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -137,9 +137,12 @@ goes offline; JavaScript `origin.follow(path)`, whose `active` is undefined between instances; the moq-ffi bindings `follow(path)`, whose `available()` returns null once the path goes offline; and C `moq_origin_follow`, whose callback reports `MOQ_FOLLOW_EVENT_OFFLINE`. -An announcement listing (`announced()`) only reports a path becoming present or -absent: an instance change keeps it present, so a reader that needs one holds a -follower. `@moq/watch`, `moq play`, and `moqsrc` follow this way, and +An announcement listing (`announced()`) reports a path becoming present, +unavailable while the network holds its route without a path to it, available +again once it recovers one, and finally absent once the origin withdraws it or +offline once this side lets go of it, which says nothing of whether the origin +still serves it. An instance change keeps it present, so a reader that needs +one holds a follower. `@moq/watch`, `moq play`, and `moqsrc` follow this way, and `moq export ts --stitch` can splice a replacement as a marked program switch. A request failing is never the signal to restart; the follower handing out a new broadcast is. Nor is a track's error kind: a track withdrawn diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 52ad1805195..8f3a4e95c1c 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -70,7 +70,7 @@ for event, err := range announced.All(ctx) { // handed each new run, with Available returning nil while the path is offline. present, ok := event.(moq.AnnounceEventPresent) if !ok { - continue // AnnounceEventAbsent + continue // absent, offline, or a change in availability } ann := present.Announce // Prefix stays origin-relative; Captures is a list, empty without wildcards. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index c509aa5d7e2..369105923ba 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -58,7 +58,8 @@ Moq.connect("https://relay.example.com", ClientConfig(tls = ClientTls(roots = li // each new run, or null once it goes offline. val announce = when (event) { is AnnounceEventPresent -> event.announce - is AnnounceEventAbsent -> return@collect + // Absent, offline, or a change in availability. + else -> return@collect } // Prefixes stay origin-relative; captures is a list of wildcard matches, empty without wildcards. println(announce.captures) diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 50a3b436545..51ca45f9b3c 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -50,7 +50,7 @@ async def main(): # A path stays present across publisher restarts: hold client.follow(path) to # be handed each new run, or None once it goes offline. if not isinstance(event, moq.AnnounceEventPresent): - continue # AnnounceEventAbsent + continue # absent, offline, or a change in availability announcement = event.announce print(announcement.captures) # list of wildcard matches, empty without wildcards broadcast = await client.request(announcement.prefix) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 59ed9d11f9b..2925131f324 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -19,7 +19,7 @@ What you use it for, beyond what the concept page already describes: - **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()` and `cost()`; an application sets only `RouteOptions`. The hop chain a relay uses for loop detection stays internal. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. - **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). -- **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. +- **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths as present, then unavailable or available as a held route loses and recovers its path, and finally absent once the origin withdraws one or offline once this side lets go of it, which says nothing of whether the origin still serves it. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-webrtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. - **Relay by opting into proxying.** On moq-lite-07 (`moq-lite-07-wip`, opt-in) a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)` to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` is a server, a client, or both: its first dial announces a hidden client endpoint at `/.client/`, and its first accept a hidden server endpoint at `/.server/`, both used only to route sessions, so grants decide which roles it may take. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(server)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(server, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 08f00708ecd..f026f67e54a 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -451,6 +451,7 @@ The publisher sends a ROUTE_START before any claim under it, and SYNCED only onc A floor names a Revision and certifies that the stream's claims for the route, across the whole requested prefix, are the origin's claims at that Revision as the request presents them (see [Announce](#announce)). It certifies only what its own request shows: a stream whose ANNOUNCE_REQUEST did not set `Hidden` certifies nothing about hidden claims. A publisher that does not know the route's claims across the whole requested prefix at one Revision, such as a relay that knows them only beneath part of it, still closes each batch, with a floor that certifies nothing. +A subscriber that may receive claims only beneath one prefix, because its [scope](#auth-stream) or the publisher's confines the stream there, SHOULD request that prefix rather than one above it, so the publisher can certify the request. When that prefix changes, it SHOULD open a request for the new one and end the old one once the new one is SYNCED. The subscriber applies a batch whole, once its floor arrives. A receiver that hears one origin's claims on several streams merges them claim by claim, each at the prefix its stream presents it: of the streams whose floor certifies the claim, the one with the highest Revision decides whether it exists, and a claim no floor certifies exists while some stream carries it. @@ -2233,6 +2234,7 @@ The `Message Length` describes the payload size on the wire. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. - A route presents only its most specific claim covering the requested prefix, moved to the next covering claim when it is withdrawn rather than ended. +- A subscriber whose scope confines a stream to one prefix SHOULD request that prefix, and replace the request when the prefix changes. - A route goes on a stream only while it has a claim the request presents, and ends after the batch withdrawing its last claim. It starts on that stream again only once the subscriber echoed the ROUTE_END. - Added `Node ID` to TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS: the origin node the first hop selected, followed by every later hop. - Replaced Warm and Cold Route Cost with the origin's Announce Cost, unchanged in transit. ANNOUNCE_UPDATE carries only the Announce Cost. diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 50ef6ae8710..173e0bb1c48 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -55,6 +55,8 @@ for event, err := range announced.All(ctx) { // Prefix stays origin-relative; Captures reports wildcard matches. fmt.Println("broadcast online", event.Announce.Prefix) case moq.AnnounceEventAbsent: + fmt.Println("broadcast ended", event.Announce.Prefix) + case moq.AnnounceEventOffline: fmt.Println("broadcast offline", event.Announce.Prefix) } } diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index c616cf86936..4b0ac44da28 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -270,7 +270,7 @@ type AnnounceOptions struct { } // Announced lists the paths under a literal prefix matching an optional pattern filter, as each -// becomes present or absent. To see another publisher instance take a path over, use Follow. +// becomes present, absent, or offline, or changes availability. To see another publisher instance take a path over, use Follow. func (o *OriginConsumer) Announced(options AnnounceOptions) (*AnnounceConsumer, error) { inner, err := o.inner.Announced(ffi.MoqAnnounceConfig{ Prefix: options.Prefix, @@ -309,12 +309,12 @@ func (o *OriginConsumer) Request(ctx context.Context, path string, instance *Ins return &BroadcastConsumer{inner: inner}, nil } -// AnnounceConsumer is a listing of paths, each present or absent. +// AnnounceConsumer is a listing of paths, each present, absent, or offline. type AnnounceConsumer struct { inner *ffi.MoqAnnounceConsumer } -// Next returns the next path becoming present or absent, or (nil, nil) when the stream ends. +// Next returns the next change to a path, or (nil, nil) when the stream ends. func (a *AnnounceConsumer) Next(ctx context.Context) (AnnounceEvent, error) { res, err := bridge.Call(ctx, nil, a.inner.Next) if err != nil || res == nil { diff --git a/go/wrapper/types.go b/go/wrapper/types.go index f12ff58d20e..df7661cc642 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -39,13 +39,22 @@ type ( // resolve a path with [OriginConsumer.Request]. Captures is a required slice, // empty when the filter has no wildcards. Announce = ffi.MoqAnnounce - // AnnounceEvent is what an AnnounceConsumer yields: AnnounceEventPresent or AnnounceEventAbsent. + // AnnounceEvent is what an AnnounceConsumer yields: AnnounceEventPresent, AnnounceEventUnavailable, + // AnnounceEventAvailable, AnnounceEventAbsent, or AnnounceEventOffline. // Another publisher instance taking a path over keeps it present; see it with [OriginConsumer.Follow]. AnnounceEvent = ffi.MoqAnnounceEvent - // AnnounceEventPresent reports a route now covering a path that was absent. + // AnnounceEventPresent reports a route now covering a path that was absent; it may arrive unavailable. AnnounceEventPresent = ffi.MoqAnnounceEventPresent - // AnnounceEventAbsent reports that no route covers a path any more, carrying its last route. + // AnnounceEventUnavailable reports that a present path lost every usable path to its origin but may + // recover one: it stays present while requests for it may fail. + AnnounceEventUnavailable = ffi.MoqAnnounceEventUnavailable + // AnnounceEventAvailable reports that a present path recovered a usable path: request it again. + AnnounceEventAvailable = ffi.MoqAnnounceEventAvailable + // AnnounceEventAbsent reports that the origin withdrew a path, carrying its last route. AnnounceEventAbsent = ffi.MoqAnnounceEventAbsent + // AnnounceEventOffline reports that this side let go of a path, carrying its last route: the session + // it arrived on closed, or the network lost every path to it, so its origin may still serve it. + AnnounceEventOffline = ffi.MoqAnnounceEventOffline // Range is a span of a track to deliver: from Start up to End, or onward when End is nil. // A relative start is the first frame of its group, a relative end includes its whole // group, and an absolute end is exclusive. diff --git a/js/moq-boy/src/element.tsx b/js/moq-boy/src/element.tsx index 6220c3f9530..698b063dd01 100644 --- a/js/moq-boy/src/element.tsx +++ b/js/moq-boy/src/element.tsx @@ -224,7 +224,8 @@ export default class MoqBoy extends HTMLElement { if (!capture?.isLiteral) continue; const id = capture.text; - if (entry.kind !== "absent" && !this.#sessions.has(id)) { + const gone = entry.kind === "absent" || entry.kind === "offline"; + if (entry.kind === "present" && !this.#sessions.has(id)) { const config: GameConfig = { sessionId: id, connection: this.connection, @@ -236,7 +237,7 @@ export default class MoqBoy extends HTMLElement { const game = new Game(config); this.#sessions.set(id, game); this.games.set(new Map(this.#sessions)); - } else if (entry.kind === "absent") { + } else if (gone) { const game = this.#sessions.get(id); if (game) { game.close(); diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 899ecfa7c4c..907647f4e2f 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -15,7 +15,7 @@ export namespace Announce { get synced(): Signals.Getter; } export type Event = { - kind: "present" | "absent"; + kind: "present" | "unavailable" | "available" | "absent" | "offline"; } & Announce.Announce; export interface Options { covering?: boolean; @@ -523,6 +523,7 @@ export namespace Origin { withdraw(): void; } export interface RouteInfo { + readonly available: boolean; readonly cost: Origin.Cost; readonly instance?: Instance.Valid; } diff --git a/js/net/examples/discovery.ts b/js/net/examples/discovery.ts index eb070025ff1..5b239a6a544 100644 --- a/js/net/examples/discovery.ts +++ b/js/net/examples/discovery.ts @@ -11,7 +11,7 @@ async function main() { // Discover broadcasts announced by the server for await (const announcement of announced) { - if (announcement.kind === "absent" || requests.has(announcement.prefix)) continue; + if (announcement.kind !== "present" || requests.has(announcement.prefix)) continue; console.log("New stream available:", announcement.prefix); // One request per path follows its publisher restarts until closed. diff --git a/js/net/src/announced.test-util.ts b/js/net/src/announced.test-util.ts index 36056ca9d5f..32de92725fe 100644 --- a/js/net/src/announced.test-util.ts +++ b/js/net/src/announced.test-util.ts @@ -15,7 +15,7 @@ export async function announcedView( for await (const event of stream) value.update((previous) => { const next = new Map(previous); - if (event.kind === "end") next.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") next.delete(event.prefix); else next.set(event.prefix, event.route); return next; }); diff --git a/js/net/src/announced.test.ts b/js/net/src/announced.test.ts index 31f0e24fc56..6cdeb8aaca3 100644 --- a/js/net/src/announced.test.ts +++ b/js/net/src/announced.test.ts @@ -63,9 +63,39 @@ test("a listing shows a route's facts, never its epoch, hops, credential, or dra kind: "present", prefix: p("a"), captures: undefined, - route: { instance, cost: 5n }, + route: { instance, cost: 5n, available: true }, }); - expect(Object.keys(event?.route ?? {}).sort()).toEqual(["cost", "instance"]); + expect(Object.keys(event?.route ?? {}).sort()).toEqual(["available", "cost", "instance"]); +}); + +test("a listing reports availability flips and tells an offline path from an absent one", async () => { + const producer = new Announce.Producer(); + const listing = Announce.list(producer.consume()); + const route = WireRoute.normalize({ instance: Instance.random() }); + const held = { ...route, available: false }; + + producer.append({ prefix: p("a"), captures: undefined, kind: "start", route: held }); + // A re-price that leaves availability as it was is not reported. + producer.append({ prefix: p("a"), captures: undefined, kind: "update", route: { ...held, cost: 3n } }); + producer.append({ prefix: p("a"), captures: undefined, kind: "update", route }); + producer.append({ prefix: p("a"), captures: undefined, kind: "restart", route: held }); + producer.append({ prefix: p("a"), captures: undefined, kind: "offline", route: held }); + producer.append({ prefix: p("b"), captures: undefined, kind: "start", route }); + producer.append({ prefix: p("b"), captures: undefined, kind: "end", route }); + + const seen = []; + for (let i = 0; i < 6; i++) { + const event = await listing.next(); + seen.push(`${event?.kind} ${event?.prefix} ${event?.route.available}`); + } + expect(seen).toEqual([ + "present a false", + "available a true", + "unavailable a false", + "offline a false", + "present b true", + "absent b true", + ]); }); test("the consumer is an async iterable of the same events", async () => { diff --git a/js/net/src/announced.ts b/js/net/src/announced.ts index f188c6ab9e6..1189aadca64 100644 --- a/js/net/src/announced.ts +++ b/js/net/src/announced.ts @@ -37,26 +37,35 @@ export interface Announce { export type Routed = Omit & { route: WireRoute }; /** - * What an announcement listing yields: a path becoming present or absent. + * What an announcement listing yields: a path becoming present, absent, or offline, or its + * availability changing. * - * `present`: a route now covers the path, which was absent. `absent`: no route covers it any - * more, carrying its last advertised route. Another instance taking a path over keeps it - * present: follow it with the origin's `follow(path)` to see the broadcast change. + * `present`: a route now covers the path, which was absent; it may arrive unavailable (see + * {@link RouteInfo.available}). `unavailable`: the present path lost every usable path to its + * origin but may recover one, so it stays present while requests for it may fail. + * `available`: it recovered one: request it again. `absent`: the origin withdrew the path, and + * no route covers it any more. `offline`: this side let go of the path, and no route covers it + * any more: the session it arrived on closed, or the network lost every path to it, so its + * origin may still serve it. Both carry its last advertised route. + * + * Another instance taking a path over keeps it present: follow it with the origin's + * `follow(path)` to see the broadcast change. * * @public */ -export type Event = { kind: "present" | "absent" } & Announce; +export type Event = { kind: "present" | "unavailable" | "available" | "absent" | "offline" } & Announce; /** * Every change to a route, for code that forwards or routes them. * - * `start`: a route now covers the prefix. `update`: the route covering it changed hops or cost, - * in place. `restart`: another instance now serves the prefix. `end`: no route covers it any - * more. + * `start`: a route now covers the prefix. `update`: the route covering it changed hops, cost, + * or availability, in place. `restart`: another instance now serves the prefix. `end`: the + * origin withdrew it, and no route covers it any more. `offline`: this side let go of it, + * and no route covers it any more. * * @internal */ -export type RouteEvent = { kind: "start" | "update" | "restart" | "end" } & Routed; +export type RouteEvent = { kind: "start" | "update" | "restart" | "end" | "offline" } & Routed; /** * Options for an announcement stream. @@ -274,7 +283,8 @@ export class Routes { let makeConsumer: (routes: Routes) => Consumer; /** - * The paths announced under a scope, each reported as it becomes present or absent. + * The paths announced under a scope, each reported as it becomes present, absent, or offline, + * and as its availability changes. * * Created internally: obtain one from an origin or the connection's `announced(scope)`. * @@ -282,6 +292,8 @@ let makeConsumer: (routes: Routes) => Consumer; */ export class Consumer { #routes: Routes; + // The present paths last reported unavailable, bounded by the present set. + #unavailable = new Set(); private constructor(routes: Routes) { this.#routes = routes; @@ -318,14 +330,30 @@ export class Consumer { } } - /** Returns the next path that became present or absent, or undefined once the listing or this handle closes. */ + /** Returns the next change to a path, or undefined once the listing or this handle closes. */ async next(): Promise { for (;;) { const event = await this.#routes.next(); if (!event) return undefined; - const { kind, prefix, captures, route } = event; - if (kind === "start") return { kind: "present", prefix, captures, route: routeInfo(route) }; - if (kind === "end") return { kind: "absent", prefix, captures, route: routeInfo(route) }; + const { kind, prefix, captures } = event; + const route = routeInfo(event.route); + switch (kind) { + case "start": + if (!route.available) this.#unavailable.add(prefix); + return { kind: "present", prefix, captures, route }; + case "update": + case "restart": { + const was = !this.#unavailable.has(prefix); + if (was === route.available) continue; + if (route.available) this.#unavailable.delete(prefix); + else this.#unavailable.add(prefix); + return { kind: route.available ? "available" : "unavailable", prefix, captures, route }; + } + case "end": + case "offline": + this.#unavailable.delete(prefix); + return { kind: kind === "end" ? "absent" : "offline", prefix, captures, route }; + } } } @@ -339,7 +367,8 @@ export class Consumer { } /** - * The listing over a route stream: only paths becoming present or absent. + * The listing over a route stream: only paths becoming present, absent, or offline, and their + * availability changing. * * @internal */ diff --git a/js/net/src/connection/close.test.ts b/js/net/src/connection/close.test.ts index b592496faad..8be3d80e421 100644 --- a/js/net/src/connection/close.test.ts +++ b/js/net/src/connection/close.test.ts @@ -79,7 +79,9 @@ for (const protocol of [Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18]) { const closing = server.close(); await fin.reached; expect(closed).toBe(false); - expect((await announced.next())?.kind).toBe("end"); + // lite-07 withdraws its claims; moq-transport only ends the stream, which + // withdraws nothing without a NAMESPACE_DONE. + expect((await announced.next())?.kind).toBe(protocol === Lite.ALPN_07_WIP ? "end" : "offline"); if (outcome === "timeout") { // bun's `rejects` blocks on a pending promise, so catch first and advance. const timedOut = closing.catch((err: unknown) => err); diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index ad0aedf6252..92cfd2b2790 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -383,7 +383,7 @@ test("an exclusive session closes every request beneath a broader one before it expect(wireOf(origin).routes(Path.from("a/b/x"))).toBe(false); start(session.stream("a/**"), "a/b/x"); await settle(); - expect(events).toEqual(["start a/b/x", "end a/b/x", "start a/b/x"]); + expect(events).toEqual(["start a/b/x", "offline a/b/x", "start a/b/x"]); expect(origin.discovery.peek()).toBe(true); // The narrower watch rides the broader request from here on. diff --git a/js/net/src/connection/forward.ts b/js/net/src/connection/forward.ts index 6feb6c72180..43909987e5c 100644 --- a/js/net/src/connection/forward.ts +++ b/js/net/src/connection/forward.ts @@ -91,8 +91,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { const report = (interest: Interest, event: Announce.RouteEvent) => { const { prefix, route } = event; - if (event.kind === "end") { - unreport(interest, prefix); + if (event.kind === "end" || event.kind === "offline") { + unreport(interest, prefix, event.kind === "end"); return; } interest.reported.add(prefix); @@ -132,7 +132,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { } }; - const unreport = (interest: Interest, prefix: Path.Valid) => { + /** Stop `interest` reporting `prefix`, which its peer `ended` rather than lost. */ + const unreport = (interest: Interest, prefix: Path.Valid, ended = false) => { interest.reported.delete(prefix); const existing = routes.get(prefix); if (!existing) return; @@ -142,7 +143,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { const next = existing.reporters.values().next(); if (next.done) { routes.delete(prefix); - existing.handle.close(); + if (ended) wireOf(existing.handle).end(); + else existing.handle.close(); } else if (driving) { // The next request takes over driving. Without an instance nothing proves it reports // the stream the handle serves, so the route restarts rather than stitch two together. diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 2d5edc6d2c6..4b66c61c1f1 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -220,7 +220,7 @@ function present(routes: Announce.Routes): Getter> { for await (const event of routes) { paths.update((previous) => { const next = new Set(previous); - if (event.kind === "end") next.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") next.delete(event.prefix); else next.add(event.prefix); return next; }); @@ -1135,7 +1135,7 @@ test("a watcher syncs with the server's initial set, and again after a reconnect const seen = new Set(); void (async () => { for await (const event of watching) { - if (event.kind === "end") seen.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") seen.delete(event.prefix); else seen.add(event.prefix); } })(); @@ -1185,7 +1185,7 @@ test("a pump owes the retractions of the upstream it replaces", async () => { const second = new Announce.Producer(); second.synced(); upstream.set(second); - expect(await parked).toMatchObject({ prefix: Path.from("a"), kind: "end" }); + expect(await parked).toMatchObject({ prefix: Path.from("a"), kind: "offline" }); await tick(); expect(consumer.synced.peek()).toBe(false); void consumer.next(); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 64552681f42..a4625c7c37b 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -971,7 +971,7 @@ export class Reload { } /** - * Pump `upstream` into `producer` for the rest of `effect`'s run: its events, then an `end` + * Pump `upstream` into `producer` for the rest of `effect`'s run: its events, then an `offline` * for each route still active once it closes or the run ends, so nothing clings to a source * that went away. Synced follows the upstream's, and owes those retractions too. * @@ -986,7 +986,8 @@ export function pumpAnnounced(effect: Effect, upstream: Announce.Routes, produce if (producer.closed.peek() !== undefined || active.size === 0) return; // Behind first, so a reader parked on the stream takes these as owed, not as news. producer.behind(); - for (const entry of active.values()) producer.append({ ...entry, kind: "end" }); + // The source went away, which says nothing of whether the origin ended them. + for (const entry of active.values()) producer.append({ ...entry, kind: "offline" }); active.clear(); }; @@ -1006,7 +1007,7 @@ export function pumpAnnounced(effect: Effect, upstream: Announce.Routes, produce for (;;) { const entry = await effect.race(upstream.next()); if (!entry || !live || producer.closed.peek() !== undefined) break; - if (entry.kind === "end") active.delete(entry.prefix); + if (entry.kind === "end" || entry.kind === "offline") active.delete(entry.prefix); else active.set(entry.prefix, entry); producer.append(entry); } diff --git a/js/net/src/hop.ts b/js/net/src/hop.ts index aab560e6b4e..940e7579c31 100644 --- a/js/net/src/hop.ts +++ b/js/net/src/hop.ts @@ -104,6 +104,13 @@ export interface RouteInfo { readonly instance?: Instance.Valid; /** What pulling content via this route costs; lower wins. */ readonly cost: Cost; + /** + * Whether a usable path reaches the route's origin. An unavailable route is held: the + * network lost every path to it but may recover one, so the announcement stays. It ranks + * with a draining route, below every available one, and requests through it may fail until + * it recovers: request again once it turns available. + */ + readonly available: boolean; } /** @@ -116,6 +123,7 @@ export function routeInfo(route: WireRoute): RouteInfo { return { instance: route.instance, cost: route.cost, + available: route.available ?? true, }; } @@ -155,6 +163,8 @@ export interface WireRoute { * preference state, never a saturated cost. Local: never forwarded, and readers never see it. */ readonly draining?: boolean; + /** See {@link RouteInfo.available}; unset means available. */ + readonly available?: boolean; } /** Constructors and helpers for {@link WireRoute}. @internal */ @@ -175,6 +185,7 @@ export const WireRoute = { cost: cost > max ? max : cost, auth: route.auth, draining: route.draining, + available: route.available, }; }, }; @@ -192,7 +203,8 @@ export function isAnonymous(route: WireRoute): boolean { } /** - * Whether two routes name the same instance, epoch, hop chain, cost, auth, and drain state. + * Whether two routes name the same instance, epoch, hop chain, cost, auth, drain state, and + * availability. * * @internal */ @@ -205,6 +217,7 @@ export function routesEqual(a: WireRoute | undefined, b: WireRoute | undefined): a.cost === b.cost && a.auth === b.auth && (a.draining ?? false) === (b.draining ?? false) && + (a.available ?? true) === (b.available ?? true) && a.hops.length === b.hops.length && a.hops.every((hop, i) => hop === b.hops[i]) ); diff --git a/js/net/src/ietf/peer-violations.test.ts b/js/net/src/ietf/peer-violations.test.ts index 27b4e41a37a..28539a57aab 100644 --- a/js/net/src/ietf/peer-violations.test.ts +++ b/js/net/src/ietf/peer-violations.test.ts @@ -188,7 +188,7 @@ for (const [version, alpn] of versions) { await entry(stream, "a"); expect((await announced.next())?.kind).toBe("start"); await entry(stream, "a", true); - expect((await announced.next())?.kind).toBe("end"); + expect((await announced.next())?.kind).toBe("offline"); await entry(stream, "a"); expect((await announced.next())?.kind).toBe("start"); if (ending === "done") { diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index d3d5ce7f1ae..4c3ac4ffe05 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -281,10 +281,10 @@ test("a subscription that dies releases what it advertised", async () => { expect(await doomed.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "start" }); expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "start" }); - // The stream that advertised it goes away without a NAMESPACE_DONE. + // The stream that advertised it goes away without a NAMESPACE_DONE: lost, not withdrawn. streamA.writer.close(); - expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "end" }); + expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "offline" }); }); /** @@ -358,7 +358,7 @@ test("an inline NAMESPACE that starts looping back is retracted", async () => { expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "start" }); await inlineNamespace(subscription, Path.from("theirs"), { hops: [SELF, PEER], cost: 0n }); - expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "offline" }); }); /** @@ -513,7 +513,7 @@ test("a PUBLISH_NAMESPACE update that starts looping back is detached", async () VERSION, ); - expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "offline" }); expect(await peer.reader.u53()).toBe(RequestOk.id); await RequestOk.decode(peer.reader, VERSION); @@ -1697,7 +1697,7 @@ test("a request overlapping one on its way out waits for the peer to end it", as expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "start" }); narrow.close(); const broad = subscriber.routes(Path.Pattern.subtree(Path.from("a"))); - expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "end" }); + expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "offline" }); // Draft 22 cancels with a reset, and the broader request waits on the peer's half. expect(stream.reader.done()).rejects.toMatchObject({ code: StreamCode.Cancel }); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index ee5420b10f3..5fb2518a254 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -446,7 +446,7 @@ export class Subscriber { /** * Drop advertisement `id` for a path, retracting it only once the last one goes. */ - #detachAnnounce(path: Path.Valid, id: number) { + #detachAnnounce(path: Path.Valid, id: number, ended: boolean) { const existing = this.#announced.get(path); const index = existing?.adverts.findIndex((advert) => advert.id === id) ?? -1; if (existing === undefined || index === -1) return; @@ -468,7 +468,13 @@ export class Subscriber { // A consumer that closed stays registered until its request is off the wire. if (!sees(filter, path) || consumer.closed.peek() !== undefined) continue; const scope = filter.scope; - consumer.append({ prefix: path, captures: scopeCaptures(scope, path), kind: "end", route: existing.route }); + // Only the peer's withdrawal ends the route; losing its stream or path lets it go. + consumer.append({ + prefix: path, + captures: scopeCaptures(scope, path), + kind: ended ? "end" : "offline", + route: existing.route, + }); } } @@ -505,13 +511,14 @@ export class Subscriber { let released = false; // The stream owns every advertisement it carried, so give them back however it ends: - // as the request starts leaving, a decode error, or the peer resetting it. Without + // as the request starts leaving, a decode error, or the peer resetting it. Closing the + // stream withdraws nothing, which NAMESPACE_DONE is for, so they go offline. Without // this each namespace keeps its count and the source never detaches, which would pin // the path for the session even after the other source withdrew. const release = () => { released = true; for (const [path, held] of live) { - if (held.id !== undefined) this.#detachAnnounce(path, held.id); + if (held.id !== undefined) this.#detachAnnounce(path, held.id, false); } live.clear(); }; @@ -633,7 +640,7 @@ export class Subscriber { // over: the path is gone even though the message says active. if (this.#reflected(params.cluster)) { console.debug(`dropping reflected namespace: broadcast=${path}`); - if (held?.id !== undefined) this.#detachAnnounce(path, held.id); + if (held?.id !== undefined) this.#detachAnnounce(path, held.id, false); live.set(path, {}); continue; } @@ -652,7 +659,7 @@ export class Subscriber { const held = live.get(path); live.delete(path); - if (held?.id !== undefined) this.#detachAnnounce(path, held.id); + if (held?.id !== undefined) this.#detachAnnounce(path, held.id, true); } else { throw new Error( `unexpected message on subscribe_namespace stream: 0x${msgType.toString(16)}`, @@ -1735,6 +1742,9 @@ export class Subscriber { // stream ends. this.#announceSlots.acquire(); let attached: number | undefined; + // Set once the peer ends the request, by FIN or reset, which withdraws the + // advertisement; a violation or a refused update here loses it instead. + let withdrawn = false; try { // Send OK first. This must complete before notifying consumers, @@ -1755,6 +1765,7 @@ export class Subscriber { if (framed === undefined) { // FIN cancels only on some drafts; on the rest the request lasts until reset. await request.ended; + withdrawn = true; break; } @@ -1802,7 +1813,7 @@ export class Subscriber { if (this.#reflected(held)) { if (attached !== undefined) { console.debug(`publish_namespace now loops back, detaching: broadcast=${path}`); - this.#detachAnnounce(path, attached); + this.#detachAnnounce(path, attached, false); attached = undefined; } } else if (attached === undefined) { @@ -1823,7 +1834,7 @@ export class Subscriber { // Give back exactly what was taken: a request that never got its OK out never // referenced the path, and retracting there would drop someone else's count. if (attached !== undefined) { - this.#detachAnnounce(path, attached); + this.#detachAnnounce(path, attached, withdrawn); } } } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 10a73768995..336dccf698b 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -1309,12 +1309,13 @@ export class Epochs { } /** - * Prefer a route that is not draining (a peer that sent GOAWAY ranks below every other), then - * identified publishers, then lower static cost. + * Prefer a route that is neither draining (a peer that sent GOAWAY) nor unavailable (held), + * which rank below every other, then identified publishers, then lower static cost. */ export function compareRoutes(a: WireRoute, b: WireRoute): number { + const last = (route: WireRoute) => Number((route.draining ?? false) || route.available === false); return ( - Number(a.draining ?? false) - Number(b.draining ?? false) || + last(a) - last(b) || Number(isAnonymous(a)) - Number(isAnonymous(b)) || (a.cost < b.cost ? -1 : a.cost > b.cost ? 1 : 0) ); diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 0940c9c6ade..6a8535450ce 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -444,7 +444,7 @@ test("a remote entry resolves by path and retracts on dispose", async () => { expect(await announced.next()).toMatchObject({ prefix: path, kind: "start", route: WireRoute.default }); dispose(); - expect(await announced.next()).toMatchObject({ prefix: path, kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: path, kind: "offline" }); expect(wireOf(consumer).routes(path)).toBe(false); announced.close(); @@ -452,6 +452,24 @@ test("a remote entry resolves by path and retracts on dispose", async () => { origin.close(); }); +test("a received route its announcer ends ends, and one its session loses goes offline", async () => { + const origin = new Producer(); + const announced = origin.consume().routes(); + + const ended = wireOf(origin).receive(Path.from("ended"), WireRoute.default); + const lost = wireOf(origin).receive(Path.from("lost"), WireRoute.default); + expect(await announced.next()).toMatchObject({ prefix: Path.from("ended"), kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("lost"), kind: "start" }); + + wireOf(ended).end(); + expect(await announced.next()).toMatchObject({ prefix: Path.from("ended"), kind: "end" }); + lost.close(); + expect(await announced.next()).toMatchObject({ prefix: Path.from("lost"), kind: "offline" }); + + announced.close(); + origin.close(); +}); + test("announced keeps a broader covering route at its prefix", async () => { const origin = new Producer(); const consumer = origin.consume(); @@ -465,7 +483,7 @@ test("announced keeps a broader covering route at its prefix", async () => { expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "start" }); dispose(); - expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "offline" }); announced.close(); upstream.close(); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 870a017ac02..40888b5e479 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -288,11 +288,18 @@ interface RouteEntry { readonly route: Signal; readonly originated: boolean; readonly server?: ServeState; + /** + * Set as a received route leaves because its session lost it, rather than its announcer + * ending it: readers then see it go offline, not end. + */ + lost: boolean; } /** One advertisement at a prefix. `exact` marks an announced local broadcast, which is only its own path. */ interface Candidate extends Advertised { readonly exact: boolean; + /** Whether the route left because its session lost it; see {@link RouteEntry.lost}. */ + readonly lost?: () => boolean; } /** @@ -495,6 +502,8 @@ interface Handoff { interface Presented extends Advertised { readonly captures: Path.Pattern[] | undefined; + /** See {@link Candidate.lost}. */ + readonly lost?: () => boolean; } /** A table mutation invalidates the shared route snapshot before its async notification. */ @@ -577,6 +586,7 @@ class OriginState { route: entry.route.peek(), claim: entry.claim, exact: false, + lost: () => entry.lost, }); } if (list.length > 0) out.set(path, list); @@ -1546,6 +1556,7 @@ export class Producer implements Table { announced, change: (update) => change(WireRoute.normalize({ ...info, ...update })), close: release, + end: release, info: () => (announced() ? this.#state.advertisedLocal.peek()?.get(path) : undefined), update: (next) => { const named = WireRoute.normalize(next); @@ -1632,6 +1643,7 @@ export class Producer implements Table { route: new Signal(route), originated, server, + lost: false, }; server.onReject = (path, err) => this.#state.refuse(path, entry, err); @@ -1680,6 +1692,11 @@ export class Producer implements Table { this.#state.servers.delete(server); remove(() => server.close()); }); + // A received route its handle closes was lost with its session, unless its announcer ended it. + const close = (lost: boolean) => { + if (!ended) entry.lost = lost; + release(); + }; // A route made after its origin (or this handle) closed is born retracted. if (this.#borrowed?.closed || !this.#state.routes.peek()) { ended = true; @@ -1733,7 +1750,8 @@ export class Producer implements Table { }, announced: () => inserted && this.#state.routes.peek() !== undefined, change: (changed) => update(WireRoute.normalize({ ...entry.route.peek(), ...changed })), - close: release, + close: () => close(!originated), + end: () => close(false), info: () => (inserted && this.#state.routes.peek() ? entry.route.peek() : undefined), update: (next) => { // A route naming no instance keeps the one it serves. @@ -2523,6 +2541,7 @@ export class Consumer implements Reader { // forwards to, see the route as it is. route: entry.route.draining === undefined ? entry.route : { ...entry.route, draining: undefined }, captures: scopeCaptures(scope.pattern, path), + lost: entry.lost, }); } return next; @@ -2553,7 +2572,7 @@ export class Consumer implements Reader { producer.append({ prefix: path, captures: snap.captures, - kind: "end", + kind: snap.lost?.() ? "offline" : "end", route: snap.route, }); } @@ -2721,6 +2740,8 @@ interface RouteControl { announced(): boolean; change(update: Partial>): void; close(): void; + /** Close as its announcer ending it; {@link close} on a received route means its session lost it. */ + end(): void; info(): WireRoute | undefined; update(info: Partial): void; } @@ -2732,6 +2753,7 @@ function registerControl(handle: Route | Claim, control: RouteControl): void { return control.info(); }, update: (info) => control.update(info), + end: () => control.end(), }); } diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index ff81466452c..c9213747bb9 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -87,6 +87,12 @@ export interface Route { * Announces a hidden route. */ update(info: Partial): void; + /** + * Close the route as its announcer withdrawing it, so readers see it end. Closing the handle + * of a route a session received instead means the session lost it, and readers see it go + * offline. + */ + end(): void; } /** The protocol-facing operations behind an origin consumer. */ diff --git a/js/room/src/room.test.ts b/js/room/src/room.test.ts index 5f1938b1201..f0db998052c 100644 --- a/js/room/src/room.test.ts +++ b/js/room/src/room.test.ts @@ -21,7 +21,7 @@ test("room restores the announce prefix and reconciles local identity changes", prefix: Net.Path.from("room-a/bob/camera.hang"), captures: [Net.Path.Pattern.literal(Net.Path.from("bob/camera.hang"))], kind: "present", - route: { cost: 0n }, + route: { cost: 0n, available: true }, }; const stream = { next: async () => { diff --git a/js/room/src/room.ts b/js/room/src/room.ts index 700d4df4cd3..8176805a5c3 100644 --- a/js/room/src/room.ts +++ b/js/room/src/room.ts @@ -95,9 +95,10 @@ export class Room { const local = this.identity.peek(); if (local && parsed.identity === local) continue; - if (update.kind !== "absent") { + // A held path keeps its participant; only presence changes membership. + if (update.kind === "present") { this.#add(parsed.identity, parsed.kind, Moq.Path.from(covered)); - } else { + } else if (update.kind === "absent" || update.kind === "offline") { this.#remove(parsed.identity, parsed.kind); } } diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index a71ab564d54..a237ab2a67c 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -60,8 +60,8 @@ function watchCovered( for (;;) { const entry = await announced.next(); if (!entry) break; - if (entry.kind === "absent") active.delete(entry.prefix); - else active.add(entry.prefix); + if (entry.kind === "absent" || entry.kind === "offline") active.delete(entry.prefix); + else if (entry.kind === "present") active.add(entry.prefix); covered.set([...active].some((prefix) => Path.hasPrefix(prefix, path))); } } catch { diff --git a/js/watch/src/echo.test.ts b/js/watch/src/echo.test.ts index 2f272604e8d..33167a6d988 100644 --- a/js/watch/src/echo.test.ts +++ b/js/watch/src/echo.test.ts @@ -31,8 +31,8 @@ function announced(origin: Origin.Producer): Set { for (;;) { const entry = await listing.next(); if (!entry) break; - if (entry.kind === "absent") paths.delete(entry.prefix); - else paths.add(entry.prefix); + if (entry.kind === "absent" || entry.kind === "offline") paths.delete(entry.prefix); + else if (entry.kind === "present") paths.add(entry.prefix); } })(); return paths; diff --git a/kt/README.md b/kt/README.md index ab6061e4308..d38d5befeda 100644 --- a/kt/README.md +++ b/kt/README.md @@ -35,7 +35,8 @@ Moq.connect("https://relay.example.com").use { moq -> // restarts; `moq.follow(path)` hands out each new run, or null once it goes offline. val announce = when (event) { is AnnounceEventPresent -> event.announce - is AnnounceEventAbsent -> return@collect + // Absent, offline, or a change in availability. + else -> return@collect } // Prefix stays origin-relative; captures reports what * matched. println("got broadcast ${announce.prefix}") diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 57fe666279b..948c7424528 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -95,7 +95,8 @@ typealias Following = uniffi.moq.MoqFollowing typealias Announce = uniffi.moq.MoqAnnounce /** - * What an [AnnounceConsumer] yields: [AnnounceEventPresent] or [AnnounceEventAbsent]. + * What an [AnnounceConsumer] yields: [AnnounceEventPresent], [AnnounceEventUnavailable], + * [AnnounceEventAvailable], [AnnounceEventAbsent], or [AnnounceEventOffline]. * * Another publisher instance taking a path over keeps it present: hold a [Following] to see the * broadcast change. @@ -104,12 +105,27 @@ typealias AnnounceEvent = uniffi.moq.MoqAnnounceEvent // Kotlin cannot reach a sealed class's subtypes through its typealias, so each // variant gets its own. -/** A route now covers the path, which was absent. */ +/** A route now covers the path, which was absent; it may arrive unavailable. */ typealias AnnounceEventPresent = uniffi.moq.MoqAnnounceEvent.Present -/** No route covers the path any more; carries its last route. */ +/** + * The present path lost every usable path to its origin but may recover one: it stays present while + * requests for it may fail. + */ +typealias AnnounceEventUnavailable = uniffi.moq.MoqAnnounceEvent.Unavailable + +/** The present path recovered a usable path: request it again. */ +typealias AnnounceEventAvailable = uniffi.moq.MoqAnnounceEvent.Available + +/** The origin withdrew the path; carries its last route. */ typealias AnnounceEventAbsent = uniffi.moq.MoqAnnounceEvent.Absent +/** + * This side let go of the path, which its origin may still serve: the session it arrived on closed, + * or the network lost every path to it. Carries its last route. + */ +typealias AnnounceEventOffline = uniffi.moq.MoqAnnounceEvent.Offline + // Broadcast / track / group producers and consumers. /** The write side of a broadcast: publish tracks into it. */ typealias BroadcastProducer = uniffi.moq.MoqBroadcastProducer diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt index 1a007569ab2..9809bd31f14 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt @@ -27,10 +27,10 @@ internal constructor( session.publish().route(path, options) /** - * List the paths matching [config] as each becomes present or absent; prefixes stay relative to - * the origin. Collect `announced(config).updates()` for a [Flow] of [AnnounceEvent] that - * cancels the handle when collection ends. A new publisher instance keeps its path present, so - * use [follow] to see it. + * List the paths matching [config] as each becomes present, absent, or offline, or changes + * availability; prefixes stay relative to the origin. Collect `announced(config).updates()` for + * a [Flow] of [AnnounceEvent] that cancels the handle when collection ends. A new publisher + * instance keeps its path present, so use [follow] to see it. */ fun announced(config: AnnounceConfig = AnnounceConfig()): AnnounceConsumer = session.consume().announced(config) diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index c2c46e08b34..218369e4015 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -30,7 +30,7 @@ async def main(): async with moq.connect("https://cdn.moq.dev/anon") as client: async for event in client.announced(): if not isinstance(event, moq.AnnounceEventPresent): - continue # AnnounceEventAbsent + continue # absent, offline, or a change in availability # A route covers a prefix and carries no broadcast, so resolve the path. broadcast = await client.request(event.announce.prefix) catalog = await moq.media.catalog(broadcast) @@ -295,7 +295,7 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect - `.announce()`, `.withdraw()`, `.set_cost(cost)`. Show, hide, or re-price it in place. - `await .requested() → BroadcastRequest | None`, or async iteration. Call `.serve(consumer)` to answer with held content under the claim's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the claim's instance until the job ends and then moves the claim to a fresh one (unless either names an instance: a named claim lends none, and a named job keeps its own), or `.reject(code)` to fail the requester. - **`OriginConsumer`**. Discover broadcasts. - - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent` or `AnnounceEventAbsent`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards + - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent`, `AnnounceEventUnavailable`, `AnnounceEventAvailable`, `AnnounceEventAbsent`, or `AnnounceEventOffline`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards - `.follow(path, instance=None) → Following` (awaitable, waits until something serves the path, then returns each stream that takes over, or `None` once the path goes offline; an `instance` follows only that stream) - `.request(path, instance=None) → BroadcastConsumer` (awaitable; owns discovery and resolves one stream, or raises once synced without a route; an `instance` refuses any other stream) diff --git a/py/moq-rs/examples/announced.py b/py/moq-rs/examples/announced.py index 77dc6617c28..451cfef48af 100644 --- a/py/moq-rs/examples/announced.py +++ b/py/moq-rs/examples/announced.py @@ -16,7 +16,7 @@ async def run(url: str, prefix: str, insecure: bool) -> None: async for event in client.announced(prefix): if isinstance(event, moq.AnnounceEventPresent): print(f" + {event.announce.prefix}") - elif isinstance(event, moq.AnnounceEventAbsent): + elif isinstance(event, (moq.AnnounceEventAbsent, moq.AnnounceEventOffline)): print(f" - {event.announce.prefix}") diff --git a/py/moq-rs/moq/__init__.py b/py/moq-rs/moq/__init__.py index 88e2cbac170..997c926ca91 100644 --- a/py/moq-rs/moq/__init__.py +++ b/py/moq-rs/moq/__init__.py @@ -20,7 +20,10 @@ AnnounceConsumer, AnnounceEvent, AnnounceEventAbsent, + AnnounceEventAvailable, + AnnounceEventOffline, AnnounceEventPresent, + AnnounceEventUnavailable, BroadcastRequest, Claim, Following, @@ -104,7 +107,10 @@ "AnnounceConsumer", "AnnounceEvent", "AnnounceEventAbsent", + "AnnounceEventAvailable", + "AnnounceEventOffline", "AnnounceEventPresent", + "AnnounceEventUnavailable", "AudioCodec", "AudioEncoderInput", "AudioSampleFormat", diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index 10e3bf1bb05..73aec83d882 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -33,18 +33,31 @@ """ AnnounceEvent = MoqAnnounceEvent -"""What :class:`AnnounceConsumer` yields: :data:`AnnounceEventPresent` or -:data:`AnnounceEventAbsent`, each carrying an :data:`Announce` as ``announce``. +"""What :class:`AnnounceConsumer` yields: :data:`AnnounceEventPresent`, +:data:`AnnounceEventUnavailable`, :data:`AnnounceEventAvailable`, +:data:`AnnounceEventAbsent`, or :data:`AnnounceEventOffline`, each carrying an +:data:`Announce` as ``announce``. Another publisher instance taking a path over keeps it present; hold :meth:`OriginConsumer.follow` to see the broadcast change. """ AnnounceEventPresent = MoqAnnounceEvent.PRESENT -"""A route now covers the path, which was absent.""" +"""A route now covers the path, which was absent; it may arrive unavailable.""" + +AnnounceEventUnavailable = MoqAnnounceEvent.UNAVAILABLE +"""The present path lost every usable path to its origin but may recover one: +it stays present while requests for it may fail.""" + +AnnounceEventAvailable = MoqAnnounceEvent.AVAILABLE +"""The present path recovered a usable path: request it again.""" AnnounceEventAbsent = MoqAnnounceEvent.ABSENT -"""No route covers the path any more; carries its last route.""" +"""The origin withdrew the path; carries its last route.""" + +AnnounceEventOffline = MoqAnnounceEvent.OFFLINE +"""This side let go of the path, which its origin may still serve: the session it +arrived on closed, or the network lost every path to it. Carries its last route.""" class AnnounceConsumer(_Stream[MoqAnnounceEvent]): diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 26e66f7f560..ac0b1214c4d 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -28,6 +28,8 @@ claim whose withdrawal is already known. 13. ✅ Decided 2026-11-05 (maintainer): **ANNOUNCE_FLOOR encodes `Revision + 1`, with 0 meaning the floor certifies nothing,** the same offset as `Max Age`. Rust decodes it to `Option`. It replaces a Revision of 2^64-1 as the marker: a huge sentinel is a bad pattern and costs 9 bytes as a lite-07 varint. Rejected: a separate message; a boolean field. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. 15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293) implements it. +16. ✅ Decided 2026-11-09 (maintainer, /quest-unblock on [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)), asked how a token-scoped lite-07 session certifies withdrawals so a clean finish ENDs across a relay: ✅ **the subscriber narrows its ANNOUNCE_REQUEST to its scope's single region** (where its limit and its subscribe grant meet), and reopens the request when that region changes so a widening renewal isn't lost. Floors keep their exact whole-prefix meaning, with no wire change and one SHOULD in the lite-07 draft. A multi-region scope still certifies nothing. Rejected: floors carrying regions on the wire, which reverses item 10; landing with the finish lanes accepting offline. +17. ✅ Decided 2026-11-09 (maintainer, /quest-unblock on [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)), asked how item 16's reopen counts the session's announce cap while the old request keeps serving until the new one is SYNCED: ✅ **the old request gets its own budget for the handover**, moving what it holds onto a separate counter with the same cap, so the session's count belongs wholly to the new request. A peer can make the session hold at most twice the cap, and only during one handover. Rejected: dropping the old request at once on every region change, which flickers claims offline; holding a client's first request until its credential is answered and dropping the old one on later renewals. The route layer's core landed in @@ -537,7 +539,6 @@ benchmarks. ## Required -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks - [Firefox tabs prove their route key](/quest/m2/cluster-routing/routes/browser-proof.md) - a current Firefox tab opens a lite-07 session with a route-key proof the relay verifies, once neqo's exporter follows the draft diff --git a/quest/m2/cluster-routing/routes/availability.md b/quest/m2/cluster-routing/routes/availability.md deleted file mode 100644 index 746565f2b58..00000000000 --- a/quest/m2/cluster-routing/routes/availability.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] Announcement availability - -## Goal - -An announcement carries availability, and letting go is a distinct offline -event, separate from the origin's END, in Rust and in `js/net`'s -announcement API (decision 3 of the epic's handoff proof). A held route -shows its claims unavailable; a let-go shows them offline; only the origin -ends them. Bindings mirror Rust. - -## Plan - -- Decided 2026-10-16: an announcement carries availability, and letting go - is a distinct offline event, separate from the origin's END. Rejected: - presence alone, which gives a reader nothing to re-request on when a held - route recovers; let-go reported as END, as today; and availability with - let-go kept as END, which hides whether the origin ended or the network - lost it. -- Build on [Listing and follow](https://github.com/moq-dev/moq/pull/7678)'s - listing and its offline report rather than adding event kinds beside it. -- JS gets the API shape here, fed by what `js/net` knows today; the - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) feeds it from the wire. - -## Related - -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index efc779abf98..210a0bd9699 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -16,7 +16,7 @@ Decided 2026-10-27 (maintainer), from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s `origin.ts` a detached request stays on its own route instead of following the cheapest one; fix it with the route-selection rework here. -- Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). +- Feed the availability API (`RouteInfo.available` and the `offline` route kind) from the route layer. - Rust/JS interop covers the route messages, the floor marker, and the echo. - Encode lite-07's Route ID (ANNOUNCE_START) and Node ID (every TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS request, 0 when the publisher selects the node) as Rust does, and pin `js/net/src/lite/announce.test.ts`'s goldens to `lite::compress`'s `GOLDEN` and `JS_LITERAL`. - Leave the temporary Lite-06 Chain Setup Parameter unsent: JS translates nothing, so it keeps no chain, and a Rust peer sends the chain only to a peer that offers it. @@ -41,10 +41,6 @@ the cheapest one; fix it with the route-selection rework here. The local drain/datagram recipes and `just test media --media-cases replacement` remain available to validate the port. -## Required - -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - the API this feeds - ## Related - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the contract diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index f11c2f36698..c174a8970a4 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -63,6 +63,19 @@ async fn run_subscribe(consumer: moq_net::origin::Consumer) -> anyhow::Result<() old.abort(); } } + moq_net::announce::Event::Offline(announce) => { + tracing::info!(path = %announce.prefix, "broadcast offline"); + if let Some(old) = playing.take() { + old.abort(); + } + } + // The follow resumes on its own once the path is reachable again. + moq_net::announce::Event::Unavailable(announce) => { + tracing::info!(path = %announce.prefix, "broadcast unreachable"); + } + moq_net::announce::Event::Available(announce) => { + tracing::info!(path = %announce.prefix, "broadcast reachable again"); + } } } anyhow::bail!("origin closed") diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 9095bafc38d..5a413ac6b8e 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -365,7 +365,7 @@ async fn subscribe( Some( moq_net::announce::Event::Present(update), ) => update, - Some(moq_net::announce::Event::Absent(_)) => continue, + Some(_) => continue, None => break, }; let path = update.prefix.to_string(); @@ -391,7 +391,7 @@ async fn subscribe( while selected < want { let update = match announced.next().await { Some(moq_net::announce::Event::Present(update)) => update, - Some(moq_net::announce::Event::Absent(_)) => continue, + Some(_) => continue, None => break, }; let path = update.prefix.to_string(); diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 4103cdd477c..d9d3a550257 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1027,6 +1027,8 @@ pub struct moq_route_info { /// [moq_origin_announced_free], and never passed to [moq_instance_free]. Routes with one /// instance are one continuous stream. pub instance: u32, + /// Whether a usable path reaches the route's origin; requests may fail while it is not. + pub available: bool, } /// Parse [moq_route_options], treating NULL as the default. @@ -1100,11 +1102,20 @@ pub enum moq_announce_kind { /// A route now covers the path, which was absent. Another instance taking it over keeps it /// present: follow it with [moq_origin_follow] to see the broadcast change. MOQ_ANNOUNCE_KIND_PRESENT = 0, - /// No route covers the path any more. + /// The origin withdrew the path: no route covers it any more. MOQ_ANNOUNCE_KIND_ABSENT = 1, -} - -/// An announce event from an origin: a path becoming present or absent. + /// This origin let go of the path, and no route covers it any more: the session it arrived + /// on closed, or the network lost every path to it. Its origin may still serve it. + MOQ_ANNOUNCE_KIND_OFFLINE = 2, + /// The present path lost every usable path to its origin but may recover one: it stays + /// present while requests for it may fail. + MOQ_ANNOUNCE_KIND_UNAVAILABLE = 3, + /// The present path recovered a usable path: request it again. + MOQ_ANNOUNCE_KIND_AVAILABLE = 4, +} + +/// An announce event from an origin: a path becoming present, absent, or offline, or its +/// availability changing. #[repr(C)] pub struct moq_announce_event { /// The covered prefix, relative to the origin, NOT NULL terminated @@ -1121,8 +1132,8 @@ pub struct moq_announce_event { /// Which event this is. pub kind: moq_announce_kind, - /// The route serving the prefix. On `MOQ_ANNOUNCE_KIND_ABSENT` it carries the last - /// advertised route. + /// The route serving the prefix. On `MOQ_ANNOUNCE_KIND_ABSENT` and + /// `MOQ_ANNOUNCE_KIND_OFFLINE` it carries the last advertised route. pub route: moq_route_info, } diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index 8ae994ae652..c134fc5e91b 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -89,6 +89,7 @@ struct AnnouncedRecord { capture_views: Vec, kind: moq_announce_kind, cost: u64, + available: bool, /// The route's instance handle, freed with the record. instance: Id, } @@ -103,6 +104,9 @@ impl AnnouncedRecord { let (update, kind) = match event { Event::Present(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_PRESENT), Event::Absent(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_ABSENT), + Event::Offline(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_OFFLINE), + Event::Unavailable(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_UNAVAILABLE), + Event::Available(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_AVAILABLE), }; let captures: Vec<_> = update .captures @@ -124,6 +128,7 @@ impl AnnouncedRecord { capture_views, kind, cost: update.route.cost().value(), + available: update.route.available(), instance: instances.insert(update.route.instance().clone())?, }) } @@ -244,6 +249,7 @@ impl Origin { route: moq_route_info { cost: announced.cost, instance: announced.instance.into(), + available: announced.available, }, }; Ok(()) diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index e3d53477727..3677f8df996 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -221,7 +221,11 @@ fn announce_info(announced: u32) -> moq_announce_event { captures: std::ptr::null(), captures_len: 0, kind: moq_announce_kind::MOQ_ANNOUNCE_KIND_PRESENT, - route: moq_route_info { cost: 0, instance: 0 }, + route: moq_route_info { + cost: 0, + instance: 0, + available: false, + }, }; assert_eq!(unsafe { moq_origin_announced_info(announced, &mut info) }, 0); info diff --git a/rs/moq-cli/src/announced.rs b/rs/moq-cli/src/announced.rs index 8ec9b04b934..217d519241b 100644 --- a/rs/moq-cli/src/announced.rs +++ b/rs/moq-cli/src/announced.rs @@ -92,7 +92,9 @@ async fn follow( let (announce, active) = match event { Event::Present(announce) => (announce, true), - Event::Absent(announce) => (announce, false), + Event::Absent(announce) | Event::Offline(announce) => (announce, false), + // The set lists presence; a held path stays listed. + Event::Available(_) | Event::Unavailable(_) => continue, }; let path = announce.prefix.as_str(); diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 3f5699d786a..2e1ef13647e 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -91,12 +91,15 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp old.abort(); } } - moq_net::announce::Event::Absent(announce) => { + moq_net::announce::Event::Absent(announce) | moq_net::announce::Event::Offline(announce) => { if let Some(follower) = followers.remove(&announce.prefix) { follower.abort(); } } - moq_net::announce::Event::Present(_) => {} + // A follower rides out its path being held. + moq_net::announce::Event::Present(_) + | moq_net::announce::Event::Available(_) + | moq_net::announce::Event::Unavailable(_) => {} } continue; } diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 959cfe175ed..79d7d066a3f 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -489,7 +489,9 @@ async fn write_echo( let event = event.context("announcements ended")?; let (announce, active) = match event { moq_net::announce::Event::Present(announce) => (announce, true), - moq_net::announce::Event::Absent(announce) => (announce, false), + moq_net::announce::Event::Absent(announce) | moq_net::announce::Event::Offline(announce) => (announce, false), + // A reader rides out its path being held. + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, }; let path = announce.prefix; if !path.has_prefix(&prefix) || !hang::echo::is_echo(&path) { diff --git a/rs/moq-cli/tests/integration/instances.rs b/rs/moq-cli/tests/integration/instances.rs index f487bd14a5f..c3188c9a51d 100644 --- a/rs/moq-cli/tests/integration/instances.rs +++ b/rs/moq-cli/tests/integration/instances.rs @@ -111,7 +111,7 @@ async fn changed(follow: &mut origin::Routes) -> bool { .. } => return true, origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, .. } => { assert!(matches!( @@ -445,7 +445,7 @@ async fn a_player_reads_through_a_claim_on_its_first_request() { loop { match routes.next().await.unwrap() { origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce: end, } if end.prefix.as_str() == "live/media" => break, _ => {} diff --git a/rs/moq-cli/tests/integration/rtsp.rs b/rs/moq-cli/tests/integration/rtsp.rs index 3512a4c5e35..49b21f5c656 100644 --- a/rs/moq-cli/tests/integration/rtsp.rs +++ b/rs/moq-cli/tests/integration/rtsp.rs @@ -69,7 +69,10 @@ async fn next_broadcast( .. } => break, moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Update, + kind: + moq_net::origin::RouteKind::End + | moq_net::origin::RouteKind::Offline + | moq_net::origin::RouteKind::Update, .. } => {} } diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 898b424121f..b3febfd5573 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -79,7 +79,8 @@ impl TryFrom for moq_net::origin::RouteOptions { } } -/// The read-only facts on a received route: a static cost and the stream's instance. +/// The read-only facts on a received route: a static cost, the stream's instance, and +/// whether a usable path reaches it. /// /// Carried by `MoqAnnounceConsumer` events. A route claims capability, not /// inventory: by convention a publisher creates each broadcast's exact route, @@ -91,6 +92,8 @@ pub struct MoqRouteInfo { pub cost: u64, /// The stream the route serves: routes with one instance are one continuous stream. pub instance: Arc, + /// Whether a usable path reaches the route's origin; requests may fail while it is not. + pub available: bool, } impl From for MoqRouteInfo { @@ -98,6 +101,7 @@ impl From for MoqRouteInfo { Self { cost: route.cost().value(), instance: MoqInstance::wrap(route.instance().clone()), + available: route.available(), } } } @@ -201,7 +205,8 @@ impl From for MoqAnnounce { } } -/// What a [`MoqAnnounceConsumer`] yields: a path becoming present or absent. +/// What a [`MoqAnnounceConsumer`] yields: a path becoming present, absent, or offline, or +/// its availability changing. /// /// Another publisher instance taking a path over keeps it present: follow it with /// `MoqOriginConsumer::follow` to see the broadcast change. @@ -212,11 +217,30 @@ pub enum MoqAnnounceEvent { /// The announcement that made the path present. announce: MoqAnnounce, }, - /// No route covers the path any more. Carries its last advertised route. + /// The present path lost every usable path to its origin but may recover one: it stays + /// present while requests for it may fail. + Unavailable { + /// The announcement, now unavailable. + announce: MoqAnnounce, + }, + /// The present path recovered a usable path: request it again. + Available { + /// The announcement, available again. + announce: MoqAnnounce, + }, + /// The origin withdrew the path: no route covers it any more. Carries its last + /// advertised route. Absent { /// The announcement that went. announce: MoqAnnounce, }, + /// This origin let go of the path, and no route covers it any more: the session it + /// arrived on closed, or the network lost every path to it. Its origin may still serve + /// it. Carries its last advertised route. + Offline { + /// The announcement that went. + announce: MoqAnnounce, + }, } impl From for MoqAnnounceEvent { @@ -226,9 +250,18 @@ impl From for MoqAnnounceEvent { Event::Present(announce) => Self::Present { announce: announce.into(), }, + Event::Unavailable(announce) => Self::Unavailable { + announce: announce.into(), + }, + Event::Available(announce) => Self::Available { + announce: announce.into(), + }, Event::Absent(announce) => Self::Absent { announce: announce.into(), }, + Event::Offline(announce) => Self::Offline { + announce: announce.into(), + }, } } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 01b61f92971..20ceac7d8b3 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -6266,7 +6266,7 @@ mod session_tests { matches!( event, moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::Offline, .. } ) diff --git a/rs/moq-net/benches/announce.rs b/rs/moq-net/benches/announce.rs index 38b4dd58a6c..38c97c65700 100644 --- a/rs/moq-net/benches/announce.rs +++ b/rs/moq-net/benches/announce.rs @@ -39,7 +39,7 @@ fn scanned(events: Vec) -> Vec { announce: update, } => (update, true), origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce: update, } => (update, false), }; diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 72b2fe7500b..09128722f8f 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -591,6 +591,35 @@ impl Permit { Some(crate::model::interest_prefixes(limit)) } + /// The one prefix the grant and the limit together confine this direction to, or + /// `None` when they confine it to several or to none. A request for it, rather than + /// one above it, is one the peer's floors can certify. + #[cfg(feature = "async")] + pub(crate) fn region(&self) -> Option { + let heads = |patterns: &Option| patterns.as_ref().map(crate::model::interest_prefixes); + let regions = match (heads(&self.granted), heads(&self.limit)) { + (None, None) => return None, + (Some(regions), None) | (None, Some(regions)) => regions, + (Some(granted), Some(limit)) => { + let mut meet: Vec = (granted.iter()) + .flat_map(|a| limit.iter().map(move |b| (a, b))) + .filter_map(|(a, b)| match () { + _ if a.has_prefix(b) => Some(a.clone()), + _ if b.has_prefix(a) => Some(b.clone()), + _ => None, + }) + .collect(); + meet.sort(); + meet.dedup(); + meet + } + }; + match regions.as_slice() { + [region] => Some(region.clone()), + _ => None, + } + } + /// Whether both set the same limit on the peer, whatever either was granted. #[cfg(feature = "async")] pub(crate) fn same_limit(&self, other: &Self) -> bool { @@ -2755,7 +2784,7 @@ impl Enforce { self.live.insert(gate.path); } crate::origin::RouteEvent { - kind: crate::origin::RouteKind::End, + kind: crate::origin::RouteKind::End | crate::origin::RouteKind::Offline, announce: update, } => { self.live.remove(&update.prefix); @@ -3029,6 +3058,24 @@ mod tests { assert_eq!(everything.regions(), None, "a root grant confines nothing"); } + /// The region is where the grant and the limit meet, and only when that is one prefix. + #[cfg(feature = "async")] + #[test] + fn the_region_is_where_grant_and_limit_meet() { + let permit = |granted: Option<&[&str]>, limit: Option<&[&str]>| Permit { + granted: granted.map(patterns), + limit: limit.map(patterns), + }; + let path = crate::Path::from_static; + assert_eq!(permit(None, None).region(), None); + assert_eq!(permit(Some(&["a"]), None).region(), Some(path("a"))); + assert_eq!(permit(None, Some(&["a/b"])).region(), Some(path("a/b"))); + assert_eq!(permit(Some(&["a"]), Some(&["a/b", "c"])).region(), Some(path("a/b"))); + assert_eq!(permit(Some(&[""]), Some(&["a"])).region(), Some(path("a"))); + assert_eq!(permit(Some(&["a", "b"]), None).region(), None, "two regions"); + assert_eq!(permit(Some(&["a"]), Some(&["b"])).region(), None, "they never meet"); + } + /// A limit replaces the last one, narrower or wider, and applies to a version /// without AUTH too. #[test] diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 5be484e23ce..234f37f9229 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -4148,7 +4148,7 @@ impl Publisher { initial.insert(update.prefix.clone(), update); } crate::origin::RouteEvent { - kind: crate::origin::RouteKind::End, + kind: crate::origin::RouteKind::End | crate::origin::RouteKind::Offline, announce: update, } => { initial.remove(&update.prefix); @@ -5921,7 +5921,7 @@ impl ForwardNamespaces { announce: update, } => (update, true, true), crate::origin::RouteEvent { - kind: crate::origin::RouteKind::End, + kind: crate::origin::RouteKind::End | crate::origin::RouteKind::Offline, announce: update, } => (update, false, false), }))); diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 08c13f00eb5..4623fa4c084 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1694,7 +1694,7 @@ impl Subscriber { // The advertisement itself stays live on the wire. tracing::debug!(%path, "dropping reflected namespace"); if let Some(id) = held.id.take() { - let _ = self.stop_announce(path, id, false); + let _ = self.stop_announce(path, id, false, false); } continue; }; @@ -1725,7 +1725,7 @@ impl Subscriber { let path = prefix.join(&msg.suffix)?; tracing::debug!(%path, "namespace_done"); if let Some(id) = live.remove(&path).and_then(|held| held.id) { - let _ = self.stop_announce(path, id, false); + let _ = self.stop_announce(path, id, false, true); } } _ => { @@ -1833,7 +1833,7 @@ impl NamespaceSubscription { // detaches. for (path, advert) in std::mem::take(&mut request.live) { if let Some(id) = advert.id { - let _ = self.this.stop_announce(path, id, false); + let _ = self.this.stop_announce(path, id, false, false); } } @@ -2072,10 +2072,17 @@ impl Subscriber { } /// Retract an advertisement's route ahead of a withdrawal that waits on the peer, so a peer - /// that stops reading cannot keep it routed. `attached` makes it once only. - fn release_announce(&mut self, path: &Path, attached: &mut Option, token: bool) -> Result<(), Error> { + /// that stops reading cannot keep it routed. `attached` makes it once only. `ended` when + /// the peer withdrew it, rather than the stream failing. + fn release_announce( + &mut self, + path: &Path, + attached: &mut Option, + token: bool, + ended: bool, + ) -> Result<(), Error> { if let Some(id) = attached.take() { - self.stop_announce(path.clone(), id, token)?; + self.stop_announce(path.clone(), id, token, ended)?; } Ok(()) } @@ -2286,7 +2293,10 @@ impl ServePublishNamespace { // cannot keep it routed. A stream that detached mid-flight (a reflected update) has // nothing left to release. let token = live.grant.is_some(); - self.this.release_announce(&live.path, &mut live.attached, token)?; + // Only the peer's own withdrawal ends the route; a failed stream loses it. + let ended = matches!(lived, Ok(Ok(Withdraw::Done))); + self.this + .release_announce(&live.path, &mut live.attached, token, ended)?; let withdraw = match lived { Ok(withdraw) => withdraw?, @@ -2769,7 +2779,7 @@ impl PublishNamespaceUpdates { None => { if let Some(id) = attached.take() { tracing::debug!(%path, "publish_namespace now loops back; detaching"); - let _ = this.stop_announce(path.clone(), id, token_grant.is_some()); + let _ = this.stop_announce(path.clone(), id, token_grant.is_some(), false); } Ok(()) } @@ -2989,7 +2999,7 @@ impl Subscriber { /// Release advertisement `id` of `path`, closing its sources when it was the last. /// `token` says its own request token authorized it. - fn stop_announce(&mut self, path: Path, id: u64, token: bool) -> Result<(), Error> { + fn stop_announce(&mut self, path: Path, id: u64, token: bool, ended: bool) -> Result<(), Error> { let mut state = self.state.lock(); match state.broadcasts.entry(path.clone()) { @@ -3003,8 +3013,12 @@ impl Subscriber { if state.adverts.is_empty() { tracing::debug!(route = %crate::path::Rooted(self.origin.root(), &path), "unannounced"); // Dropping the entry retracts the route (its announcement drops) and - // closes its sources (its token closes). - entry.remove(); + // closes its sources (its token closes). Readers see it end only when + // the peer withdrew it; otherwise it goes offline. + let state = entry.remove(); + if ended && let Some(dynamic) = state.dynamic { + dynamic.ended(); + } return Ok(()); } // The newest went while an older one stands: that one is what remains. @@ -10650,8 +10664,8 @@ mod tests { )] ); - subscriber.stop_announce(a.clone(), first, false).unwrap(); - subscriber.stop_announce(a, repeat, false).unwrap(); + subscriber.stop_announce(a.clone(), first, false, true).unwrap(); + subscriber.stop_announce(a, repeat, false, true).unwrap(); subscriber.start_announce(b, last, Admit::Session(None)).unwrap(); } @@ -10837,7 +10851,7 @@ mod tests { use futures::FutureExt; match consumer.path_routes(path).ok()?.next().now_or_never()?? { crate::origin::RouteEvent { - kind: crate::origin::RouteKind::End, + kind: crate::origin::RouteKind::End | crate::origin::RouteKind::Offline, .. } => None, crate::origin::RouteEvent { @@ -11102,7 +11116,7 @@ mod tests { .unwrap(); settle().await; - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!( routed_now(&consumer, "room/host").is_none(), "an explicit NAMESPACE_DONE must retract the route", @@ -11251,7 +11265,7 @@ mod tests { settle().await; // One advertisement's stream dies: the other still holds the source. - subscriber.stop_announce(path.clone(), ids[0], false).unwrap(); + subscriber.stop_announce(path.clone(), ids[0], false, true).unwrap(); settle().await; assert!( routed_now(&consumer, "room/host").is_some(), @@ -11259,7 +11273,7 @@ mod tests { ); // The last owner retracts: the broadcast closes with it. - subscriber.stop_announce(path, ids[1], false).unwrap(); + subscriber.stop_announce(path, ids[1], false, true).unwrap(); settle().await; assert!( routed_now(&consumer, "room/host").is_none(), @@ -11362,7 +11376,7 @@ mod tests { // One advertisement, so one unannounce detaches it. If the update had bumped the // refcount, this would leave the route stranded. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11414,7 +11428,7 @@ mod tests { ); // One advertisement, so one unannounce detaches it. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11447,9 +11461,9 @@ mod tests { assert!(routed_now(&consumer, "room/host").is_some()); // Two advertisements, so it takes two unannounces to retract. - subscriber.stop_announce(path.clone(), first, false).unwrap(); + subscriber.stop_announce(path.clone(), first, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_some()); - subscriber.stop_announce(path, second, false).unwrap(); + subscriber.stop_announce(path, second, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11480,9 +11494,9 @@ mod tests { assert!(routed_now(&consumer, "room/host").is_some()); // Two advertisements, so it takes two unannounces to retract. - subscriber.stop_announce(path.clone(), first, false).unwrap(); + subscriber.stop_announce(path.clone(), first, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_some()); - subscriber.stop_announce(path, second, false).unwrap(); + subscriber.stop_announce(path, second, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11521,7 +11535,7 @@ mod tests { ); // That supersedes the advertisement it repeats, so the old route is retired. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!( routed_now(&consumer, "room/host").is_none(), "the superseded route must not stay attached" @@ -12131,7 +12145,7 @@ mod tests { // What the stream's exit path does with whatever it still holds. for (path, id) in live { - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); } assert!(routed_now(&consumer, "room/a").is_none(), "room/a leaked a refcount"); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 8ea1c511903..81259aaf509 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1238,7 +1238,7 @@ impl AnnounceRun { current.insert(update.prefix, update.route); } origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce: update, } => { current.remove(&update.prefix); @@ -1612,7 +1612,7 @@ impl AnnounceRun { announce: update, }) => (update, true, true), Some(origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce: update, }) => (update, false, false), None => { diff --git a/rs/moq-net/src/lite/replay.rs b/rs/moq-net/src/lite/replay.rs index 060b6a16fa1..3e284b3f395 100644 --- a/rs/moq-net/src/lite/replay.rs +++ b/rs/moq-net/src/lite/replay.rs @@ -15,7 +15,7 @@ pub(super) fn initial(events: impl IntoIterator) -> V announce: update, } => (update, true), origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce: update, } => (update, false), }; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 1a0a4e3fbd8..f7b0475482f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1011,6 +1011,8 @@ pub(super) struct SubscriberDriver { announces: kio::Tasks, /// Each open prefix's machine in `announces`. prefixes: HashMap, + /// Lite-07: the subscribe permit's epoch the requests were last narrowed at. + auth_epoch: u64, uni: UniAccept, /// PROBE feedback; finishes quietly when unsupported or given up on. bandwidth: Option, @@ -1033,6 +1035,7 @@ impl SubscriberDriver { discovery: crate::discovery::Discovery::new(&subscriber.origin, lazy, subscriber.runtime.clone()), announces: kio::Tasks::new(), prefixes: HashMap::new(), + auth_epoch: 0, cleanup: SubscriptionCleanup(subscriber.subscribes.clone(), subscriber.datagrams.clone()), uni: UniAccept::new(subscriber.clone(), early), bandwidth: Some(RecvBandwidth::new(subscriber.clone())), @@ -1080,6 +1083,25 @@ impl SubscriberDriver { } pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + // A new permit may narrow a request to another region: it reopens there, so a + // renewal that widens the scope is not lost. + if self.subscriber.version.has_routes() { + let auth = &self.subscriber.auth; + let mut changed = false; + while auth + .poll_permit(crate::auth::Direction::Subscribe, &mut self.auth_epoch, waiter) + .is_ready() + { + changed = true; + } + if changed { + for key in self.prefixes.values_mut() { + if let Some(machine) = self.announces.remove(*key) { + *key = self.announces.insert(machine.reopen()); + } + } + } + } loop { while let Poll::Ready(change) = self.discovery.poll(waiter) { match change { @@ -2103,12 +2125,20 @@ struct AnnouncePrefix { /// Its place in line for the peer's stream credit while its stream opens. open: crate::transport::open::Open, subscriber: Subscriber, + /// The prefix discovery wants, which keys the machine. prefix: Path, + /// The prefix on the wire: lite-07 narrows `prefix` to the region the session's + /// subscribe permit confines it to, so the peer's floors can certify it. + request: Path, state: PrefixState, /// Holds the origin's announce consumers under the prefix unsynced /// until the peer's initial set is in. Dropped at the boundary the version /// marks, or with the stream. snapshot: Option, + /// Lite-07: the request this one replaces, after the region changed. It keeps + /// serving until this one is synced, so the node lets go only of what the new + /// request no longer carries. + replacing: Option>, } impl kio::Task for AnnouncePrefix { @@ -2119,6 +2149,15 @@ impl kio::Task for AnnouncePrefix { } } +/// The prefix to request for `prefix`: the region the permit confines it to when +/// that lies beneath it, since a floor certifies only a request its scope covers whole. +fn narrow(prefix: &Path, region: Option) -> Path { + match region { + Some(region) if region.has_prefix(prefix) => region, + _ => prefix.clone(), + } +} + /// The announce-decode loop's state. struct PrefixRun { responder_origin: Option, @@ -2139,6 +2178,8 @@ struct PrefixRun { initial: u64, /// Lite-07: the route layer the stream feeds in place of `announced`. routed: Option, + /// Lite-07: SYNCED arrived. + synced: bool, } #[kio::machine(self.state: PrefixState)] @@ -2146,18 +2187,73 @@ impl AnnouncePrefix { /// `snapshot` was taken when the prefix was first wanted, so a consumer of the /// origin never reads as synced before the request even goes out. fn new(subscriber: Subscriber, prefix: Path, snapshot: crate::model::Snapshot) -> Self { + let request = Self::narrowed(&subscriber, &prefix); Self { open: Default::default(), subscriber, prefix, + request, state: PrefixState::Start, snapshot: Some(snapshot), + replacing: None, + } + } + + /// The prefix to put on the wire for `prefix`, as the session's permit stands now. + fn narrowed(subscriber: &Subscriber, prefix: &Path) -> Path { + match subscriber.version.has_routes() { + true => narrow( + prefix, + subscriber.auth.permit(crate::auth::Direction::Subscribe).region(), + ), + false => prefix.clone(), + } + } + + /// A request for the region the permit narrows the prefix to now, replacing this + /// one, or this one while its request still is that region. The replacement takes + /// over the initial set's snapshot if this one was still waiting on it. + fn reopen(mut self) -> Self { + let request = Self::narrowed(&self.subscriber, &self.prefix); + if request == self.request { + return self; + } + // A replacement never waits on another: the one before is let go at once. + self.replacing = None; + let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); + // One that never reached its loop has fed the node nothing to hand over. One that + // did moves its announce slots onto a budget of its own, since the replacement takes + // slots for the same claims: the replacement gets the session's whole cap. + let replacing = match &mut self.state { + PrefixState::Run(step) => { + if let Some(routed) = &mut step.run.routed { + routed.hand_over(); + } + Some(Box::new(self)) + } + _ => None, + }; + Self { + open: Default::default(), + subscriber, + prefix, + request, + state: PrefixState::Start, + snapshot, + replacing, } } /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + if let Some(replacing) = &mut self.replacing + && let Poll::Ready(res) = replacing.poll(waiter) + { + // The peer finished the old request: nothing left to hand over. + self.replacing = None; + res?; + } self.poll_run(waiter) .map_err(|err| lite::error::control(&mut self.subscriber.session.clone(), err)) } @@ -2180,7 +2276,7 @@ impl AnnouncePrefix { // the origin, and each local reader opts in on its own // (`origin::Consumer::with_hidden`). stream.writer.buffer(&lite::AnnounceRequest { - prefix: self.prefix.clone(), + prefix: self.request.clone(), exclude_hop: self.subscriber.self_origin, hidden: true, })?; @@ -2233,10 +2329,16 @@ impl AnnouncePrefix { decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, initial, + synced: false, routed: match version.has_routes() { true => { let permit = self.subscriber.auth.permit(crate::auth::Direction::Subscribe); - Some(routed::Routed::new(&self.subscriber, &self.prefix, link_cost, &permit)?) + Some(routed::Routed::new( + &self.subscriber, + &self.request, + link_cost, + &permit, + )?) } false => None, }, @@ -2258,7 +2360,7 @@ impl AnnouncePrefix { })?; if let Some(init) = init { for suffix in init.suffixes { - let path = self.prefix.join(&suffix)?; + let path = self.request.join(&suffix)?; // Lite01/02 don't carry hop information; the broadcast starts with // an empty chain and an unpriced link. Stats are attributed in the // model when this enters the origin via `route`. @@ -2313,9 +2415,12 @@ impl AnnouncePrefix { if let Some(routed) = &mut run.routed { routed.outputs(&self.subscriber); } - if self.snapshot.take().is_none() { + if std::mem::replace(&mut run.synced, true) { return Poll::Ready(Err(Error::ProtocolViolation)); } + self.snapshot = None; + // This request now carries what the one it replaces did. + self.replacing = None; continue; } if run.initial > 0 { @@ -2326,13 +2431,13 @@ impl AnnouncePrefix { _ => return Poll::Ready(Err(Error::ProtocolViolation)), } } - self.subscriber.handle_announce(&self.prefix, announce, run)?; + self.subscriber.handle_announce(&self.request, announce, run)?; // Released only once the last counted route is in the origin. if run.initial == 0 && !self.subscriber.version.has_synced() { self.snapshot = None; } } - Poll::Ready(Ok(None)) if run.initial > 0 || self.snapshot.is_some() => { + Poll::Ready(Ok(None)) if run.initial > 0 || (version.has_synced() && !run.synced) => { return Poll::Ready(Err(Error::ProtocolViolation)); } Poll::Ready(Ok(None)) => { @@ -8743,7 +8848,7 @@ mod tests { ) .unwrap(); assert!(!attached, "a restart outside the limit was attached"); - cursor.expect_next_ended("room/host").await; + cursor.expect_next_offline("room/host").await; } /// An announce stream that dies without an explicit `ended` retracts the route @@ -8790,7 +8895,7 @@ mod tests { // The stream ends without retracting anything: the map dies with it and the // route retracts. drop(announced); - cursor.expect_next_ended("room/host").await; + cursor.expect_next_offline("room/host").await; // An explicit retraction retracts it the same way. let hops = crate::Hops::try_from(vec![crate::Hop::new(7).unwrap()]).unwrap(); diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index d1b55c5bd6e..46436177c4d 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -304,6 +304,18 @@ impl Routed { Ok(()) } + /// Move what the stream holds onto a budget of its own with the session's cap, while a + /// request replacing it takes the same claims over on the session's. The session's + /// count is then the replacement's alone, and a peer flooding this stream fails it + /// on its own budget: at most twice the cap, during one handover. + pub(super) fn hand_over(&mut self) { + let slots = self.slots.split(); + for slot in self.held.iter_mut().chain(&mut self.echoing) { + *slot = slots.hold(); + } + self.slots = slots; + } + /// The peer sent a GOAWAY: its routes rank below every other. Idempotent. pub(super) fn goaway(&mut self, subscriber: &Subscriber) { if self.drained { @@ -358,9 +370,8 @@ impl Routed { drop(std::mem::replace(&mut self.stream, stream)); self.previous = previous; self.outputs(subscriber); - for (_, served) in std::mem::take(&mut self.previous) { - served.route.dynamic.withdrawn(); - } + // What the new stream no longer serves is let go here, not ended by its origin. + self.previous.clear(); Ok(()) } @@ -400,8 +411,11 @@ impl Routed { claim, served, } => self.serve(subscriber, id, named, claim, served), - node::Output::Unserve { id, claim } => { - if let Some(served) = self.served.remove(&(id, claim)) { + node::Output::Unserve { id, claim, ended } => { + // Dropped, a claim the node let go of goes offline rather than ending. + if let Some(served) = self.served.remove(&(id, claim)) + && ended + { served.route.dynamic.withdrawn(); } } @@ -440,15 +454,16 @@ impl Routed { } return; } - // A route held without a path to it serves only as a last resort, like a draining - // peer's, at its real cost. + // A route held without a path to it is unavailable: it serves only as a last + // resort, like a draining peer's, at its real cost. let route = crate::origin::RouteInfo::new() .with_instance(served.claim.instance.clone(), served.claim.epoch) .with_hops(hops) .with_cost(served.claim.cost.saturating_add(served.metric)) .with_via(subscriber.via(None)) .with_node(Some(id)) - .with_draining(self.drained || !served.available); + .with_draining(self.drained) + .with_available(served.available); if let Some(entry) = self.previous.remove(&key) { self.served.insert(key.clone(), entry); } @@ -788,6 +803,93 @@ mod tests { assert_eq!(subscriber.announces.available(), 2); } + /// The route and claims a peer sends one stream, its Route ID 0. + fn feed( + routed: &mut super::Routed, + decoder: &mut AnnounceDecoder, + paths: &[&'static str], + ) -> Result<(), crate::Error> { + let prefix = Path::empty(); + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + routed.receive(&prefix, route, decoder)?; + for path in paths { + routed.receive(&prefix, start(path), decoder)?; + } + Ok(()) + } + + /// A request handed over keeps what it holds on a budget of its own, so the claims + /// arriving on both it and its replacement during the handover count once against the + /// session's cap: a session that fits its claims stays open. + #[kio::test] + async fn a_handover_counts_the_replacement_alone() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + // One route and three claims. + subscriber.announces = crate::session::Slots::new(4); + let prefix = Path::empty(); + let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut old_decoder = AnnounceDecoder::default(); + feed(&mut old, &mut old_decoder, &["x", "y"]).unwrap(); + + old.hand_over(); + assert_eq!( + subscriber.announces.available(), + 4, + "the session's count is the replacement's" + ); + let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut new_decoder = AnnounceDecoder::default(); + feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); + + // A claim the peer starts during the handover reaches both requests. + old.receive(&prefix, start("z"), &mut old_decoder) + .expect("the old request fits it on its own budget"); + new.receive(&prefix, start("z"), &mut new_decoder) + .expect("the session fits it"); + assert_eq!(subscriber.announces.available(), 0); + + drop(old); + assert_eq!( + subscriber.announces.available(), + 0, + "the old request held none of the session's" + ); + drop(new); + assert_eq!(subscriber.announces.available(), 4); + } + + /// A peer flooding a request handed over fails it on its own budget, the session's cap, + /// so the session holds at most twice the cap during one handover. + #[kio::test] + async fn a_handover_caps_the_old_request() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + subscriber.announces = crate::session::Slots::new(3); + let prefix = Path::empty(); + let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut old_decoder = AnnounceDecoder::default(); + feed(&mut old, &mut old_decoder, &["x"]).unwrap(); + + old.hand_over(); + let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut new_decoder = AnnounceDecoder::default(); + feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); + assert_eq!(subscriber.announces.available(), 0); + + old.receive(&prefix, start("y"), &mut old_decoder) + .expect("the old request's budget has room for one more"); + assert!( + old.receive(&prefix, start("z"), &mut old_decoder).is_err(), + "the old request is capped on its own budget" + ); + assert!( + new.receive(&prefix, start("z"), &mut new_decoder).is_err(), + "the replacement is capped on the session's" + ); + } + /// A downstream peer raising its ROUTE_REQUEST seqno while this stream's peer /// leaves its replies unread holds one request for the identity, sent once the /// writer flushes and naming the highest seqno. diff --git a/rs/moq-net/src/model/announced.rs b/rs/moq-net/src/model/announced.rs index a1b195b5496..beb550c6253 100644 --- a/rs/moq-net/src/model/announced.rs +++ b/rs/moq-net/src/model/announced.rs @@ -1,5 +1,6 @@ //! The paths announced under a scope, each present or absent. +use std::collections::HashSet; use std::task::Poll; use super::origin_impl::{Announce, RouteEvent, RouteKind, Routes}; @@ -8,42 +9,86 @@ use crate::{Error, Path}; /// What an announcement listing reports for a path. #[derive(Clone, Debug)] pub enum AnnounceEvent { - /// A route now covers the path, which was absent. + /// A route now covers the path, which was absent. It may arrive unavailable; see + /// [`RouteInfo::available`](crate::origin::RouteInfo::available). Present(Announce), - /// No route covers the path any more. Carries its last advertised route. + /// The present path's route lost every usable path to its origin but may recover + /// one, so the path stays present while requests for it may fail. + Unavailable(Announce), + /// The present path's route recovered a usable path: request it again. + Available(Announce), + /// The origin withdrew the path: no route covers it any more. Carries its last + /// advertised route. Absent(Announce), + /// This origin let go of the path, and no route covers it any more: the session it + /// arrived on closed, or the network lost every path to it. Its origin may still + /// serve it. Carries its last advertised route. + Offline(Announce), } /// The paths announced in a scope, each reported as it becomes present or absent. /// /// A path that another instance takes over stays present, and a route's metadata -/// changing is not reported: hold a [`follow`](crate::origin::Consumer::follow) to -/// see the broadcast change. Created by +/// changing is not reported, except its availability: hold a +/// [`follow`](crate::origin::Consumer::follow) to see the broadcast change. Created by /// [`Consumer::announced`](crate::origin::Consumer::announced). Drop to unregister. pub struct AnnounceConsumer { routes: Routes, + /// The present paths last reported unavailable, bounded by the present set. + unavailable: HashSet, } impl AnnounceConsumer { pub(super) fn new(routes: Routes) -> Self { - Self { routes } + Self { + routes, + unavailable: HashSet::new(), + } } /// What a route change means for the listing, if anything. - fn list(event: RouteEvent) -> Option { + fn list(&mut self, event: RouteEvent) -> Option { match event { RouteEvent { kind: RouteKind::Start, announce, - } => Some(AnnounceEvent::Present(announce)), + } => { + if !announce.route.available() { + self.unavailable.insert(announce.prefix.clone()); + } + Some(AnnounceEvent::Present(announce)) + } + RouteEvent { + kind: RouteKind::Update | RouteKind::Restart, + announce, + } => { + let was = !self.unavailable.contains(&announce.prefix); + match (was, announce.route.available()) { + (true, false) => { + self.unavailable.insert(announce.prefix.clone()); + Some(AnnounceEvent::Unavailable(announce)) + } + (false, true) => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Available(announce)) + } + _ => None, + } + } RouteEvent { kind: RouteKind::End, announce, - } => Some(AnnounceEvent::Absent(announce)), + } => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Absent(announce)) + } RouteEvent { - kind: RouteKind::Update | RouteKind::Restart, - .. - } => None, + kind: RouteKind::Offline, + announce, + } => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Offline(announce)) + } } } @@ -64,7 +109,7 @@ impl AnnounceConsumer { let Some(event) = std::task::ready!(self.routes.poll_next(waiter)) else { return Poll::Ready(None); }; - if let Some(event) = Self::list(event) { + if let Some(event) = self.list(event) { return Poll::Ready(Some(event)); } } @@ -76,7 +121,8 @@ impl AnnounceConsumer { /// Use [`Self::is_closed`] to check if the listing is closed. pub fn try_next(&mut self) -> Option { loop { - if let Some(event) = Self::list(self.routes.try_next()?) { + let event = self.routes.try_next()?; + if let Some(event) = self.list(event) { return Some(event); } } @@ -122,7 +168,7 @@ impl futures::Stream for AnnounceConsumer { else { return Poll::Ready(None); }; - if let Some(event) = Self::list(event) { + if let Some(event) = this.list(event) { return Poll::Ready(Some(event)); } } @@ -149,6 +195,9 @@ mod tests { match event { AnnounceEvent::Present(announce) => format!("present {}", announce.prefix), AnnounceEvent::Absent(announce) => format!("absent {}", announce.prefix), + AnnounceEvent::Offline(announce) => format!("offline {}", announce.prefix), + AnnounceEvent::Available(announce) => format!("available {}", announce.prefix), + AnnounceEvent::Unavailable(announce) => format!("unavailable {}", announce.prefix), } } @@ -205,4 +254,38 @@ mod tests { drop(second); assert_eq!(listed(&mut announced).await, "absent live"); } + + /// A held route keeps its path present, reported unavailable and available as it flips; a + /// session losing a route takes its path offline, while a peer's withdrawal ends it. + #[kio::test] + async fn availability_flips_and_offline_is_not_absent() { + let origin = Hop::new(1).unwrap().produce(); + let mut announced = origin.consume().announced(); + let instance = Instance::random(); + let held = RouteInfo::new() + .with_instance(instance.clone(), 0) + .with_available(false); + + let lost = origin.forward("lost", held.clone()).unwrap(); + assert_eq!(listed(&mut announced).await, "present lost"); + lost.update(held.clone().with_cost(Cost::new(3))).unwrap(); + quiet(&mut announced).await; + lost.update(held.clone().with_available(true)).unwrap(); + assert_eq!(listed(&mut announced).await, "available lost"); + lost.update(held.clone()).unwrap(); + assert_eq!(listed(&mut announced).await, "unavailable lost"); + drop(lost); + assert_eq!(listed(&mut announced).await, "offline lost"); + + let ended = origin.forward("ended", held.with_available(true)).unwrap(); + assert_eq!(listed(&mut announced).await, "present ended"); + ended.withdrawn(); + assert_eq!(listed(&mut announced).await, "absent ended"); + + // A route published here ends with its handle. + let local = origin.route("local", route(&instance)).unwrap(); + assert_eq!(listed(&mut announced).await, "present local"); + drop(local); + assert_eq!(listed(&mut announced).await, "absent local"); + } } diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index c4ed0ae6fa4..79188e30c6e 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -99,7 +99,7 @@ impl Follow { self.idle = matches!( event, RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, .. } ); @@ -135,18 +135,15 @@ impl Follow { /// Apply one route's event, returning what it means for the path, if anything. fn fold(&mut self, event: RouteEvent) -> Option { - let (RouteEvent { - kind: RouteKind::Start | RouteKind::Update, - announce, - } - | RouteEvent { - kind: RouteKind::Restart | RouteKind::End, - announce, - }) = &event; - if !self.path.has_prefix(&announce.prefix) { + let prefix = event.announce.prefix.clone(); + if !self.path.has_prefix(&prefix) { return None; } - let prefix = announce.prefix.clone(); + // A route that went reports its own way of going: ended by its origin, or let go. + let gone = match event.kind { + RouteKind::Offline => RouteKind::Offline, + _ => RouteKind::End, + }; let before = self .serving() .map(|standing| (standing.announce.clone(), standing.seen)); @@ -158,7 +155,8 @@ impl Follow { .map(|index| self.covering.swap_remove(index).since); let (announce, since, restart) = match event { RouteEvent { - kind: RouteKind::End, .. + kind: RouteKind::End | RouteKind::Offline, + .. } => (None, None, false), RouteEvent { kind: RouteKind::Start, @@ -191,7 +189,7 @@ impl Follow { announce: after, }), (Some((before, _)), None) => Some(RouteEvent { - kind: RouteKind::End, + kind: gone, announce: before, }), // The serving route ended, and the one left may have started after it did. @@ -199,7 +197,7 @@ impl Follow { if before.prefix == prefix && before.prefix != after.prefix && since > self.dry.max(seen) => { Some(RouteEvent { - kind: RouteKind::End, + kind: gone, announce: before, }) } @@ -428,7 +426,8 @@ impl Following { // An end is a gap that ended every request on the old route, so whatever serves the // path next is handed out, even the same instance coming back. RouteEvent { - kind: RouteKind::End, .. + kind: RouteKind::End | RouteKind::Offline, + .. } => { self.unseen = false; self.end(); @@ -482,6 +481,10 @@ mod tests { kind: RouteKind::End, announce, } => ("end", announce.prefix.to_string()), + RouteEvent { + kind: RouteKind::Offline, + announce, + } => ("offline", announce.prefix.to_string()), } } @@ -534,7 +537,7 @@ mod tests { assert_eq!(followed(&mut follow).await, ("update", "pool".into())); drop(pool); - assert_eq!(followed(&mut follow).await, ("end", "pool".into())); + assert_eq!(followed(&mut follow).await, ("offline", "pool".into())); } /// A follower that joins late starts on the route serving the path, not on whichever diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 559a50abcd7..7e48dfca0ee 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -501,14 +501,17 @@ pub struct RouteInfo { /// peer that names no node. pub(crate) node: Option, - /// The session it arrived on is going away (the peer sent a GOAWAY), or the route - /// layer holds the route without a path to it: it ranks below every other eligible - /// route, whatever its preference or cost, yet still serves when nothing else does. + /// The session it arrived on is going away (the peer sent a GOAWAY): it ranks below + /// every other eligible route, whatever its preference or cost, yet still serves when + /// nothing else does. /// /// A preference state, never a saturated metric. Local, like [`Self::via`]: never /// forwarded, so a relay advertises a draining route at its real cost and its /// readers never see the flag. pub(crate) draining: bool, + + /// Whether a usable path reaches the route's origin; see [`Self::available`]. + pub(crate) available: bool, } impl RouteInfo { @@ -525,6 +528,7 @@ impl RouteInfo { source: Source::Local, node: None, draining: false, + available: true, } } } @@ -562,6 +566,14 @@ impl RouteInfo { self.cost } + /// Whether a usable path reaches the route's origin. An unavailable route is held: + /// the network lost every path to it but may recover one, so the announcement stays. + /// It ranks with a draining route, below every available one, and requests through + /// it may fail until it recovers: request again once it turns available. + pub fn available(&self) -> bool { + self.available + } + /// Serve a stable `instance` at `epoch`; see [`RouteOptions::with_instance`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn with_instance(mut self, instance: Instance, epoch: u64) -> Self { @@ -616,6 +628,13 @@ impl RouteInfo { self } + /// Mark whether a usable path reaches the route's origin; see [`Self::available`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_available(mut self, available: bool) -> Self { + self.available = available; + self + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -769,9 +788,9 @@ impl<'a> Epochs<'a> { } /// Ordering key for an eligible route entry resolving `path` (see [`Epochs`]). -/// Lower wins: first a route that is not draining (see [`RouteInfo::draining`]), so a -/// peer that sent a GOAWAY ranks below every other eligible route and serves only as a -/// last resort. Then the higher link [`Preference`], so metrics from different operators +/// Lower wins: first a route that is neither draining (see [`RouteInfo::draining`]) nor +/// unavailable (see [`RouteInfo::available`]), so a peer that sent a GOAWAY or a held +/// route ranks below every other eligible route and serves only as a last resort. Then the higher link [`Preference`], so metrics from different operators /// are never compared. Then an identified chain (no 0) outranks an anonymous one regardless /// of cost, then the cheapest cost, then a broadcast published on this origin (it serves /// what is here, not a claim that has to ask), then the shortest hop chain, then a hash @@ -797,7 +816,7 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { None => (entry.is_anonymous(), entry.hops.len()), }; ( - entry.draining, + entry.draining || !entry.available, Reverse(entry.preference), anonymous, entry.cost, @@ -828,8 +847,8 @@ type RouteOrder = ( Reverse, ); -/// The `(hops, cost, source, instance, epoch, auth, node)` metadata an announce cursor delivers -/// alongside a prefix. +/// The `(hops, cost, source, instance, epoch, auth, node, available)` metadata an announce +/// cursor delivers alongside a prefix. type RouteMeta = ( Hops, Cost, @@ -838,6 +857,7 @@ type RouteMeta = ( u64, Option, Option, + bool, ); /// The earlier of two optional deadlines. @@ -895,8 +915,31 @@ type AnnounceMeta = (RouteMeta, Option>); enum PendingUpdate { Announce(AnnounceMeta), Restart(AnnounceMeta), - Unannounce(AnnounceMeta), - UnannounceAnnounce { old: AnnounceMeta, new: AnnounceMeta }, + Unannounce(AnnounceMeta, Gone), + UnannounceAnnounce { + old: AnnounceMeta, + gone: Gone, + new: AnnounceMeta, + }, +} + +/// How a route left a cursor: the origin ended it, or this node lost it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Gone { + /// Its origin withdrew it, as far as this node knows: [`RouteKind::End`]. + End, + /// This node let go of it, such as the session it arrived on closing: + /// [`RouteKind::Offline`]. + Offline, +} + +impl Gone { + fn kind(self) -> RouteKind { + match self { + Self::End => RouteKind::End, + Self::Offline => RouteKind::Offline, + } + } } // An update is taken only while collapsing it under the cursor state lock. @@ -966,7 +1009,7 @@ impl OriginConsumerState { pending.owed = !self.delivered.contains(prefix) || matches!( pending.update, - Some(PendingUpdate::Unannounce(_) | PendingUpdate::UnannounceAnnounce { .. }) + Some(PendingUpdate::Unannounce(..) | PendingUpdate::UnannounceAnnounce { .. }) ); self.owed += usize::from(pending.owed); } @@ -1030,8 +1073,8 @@ impl OriginConsumerState { // The consumer still owes the restart; this only moves its metadata. Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), // Consumer needs to observe the retraction before this announce. - Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { - PendingUpdate::UnannounceAnnounce { old, new: meta } + Some(PendingUpdate::Unannounce(old, gone) | PendingUpdate::UnannounceAnnounce { old, gone, .. }) => { + PendingUpdate::UnannounceAnnounce { old, gone, new: meta } } }; self.put(prefix, new); @@ -1042,8 +1085,8 @@ impl OriginConsumerState { fn apply_restart(&mut self, prefix: Path, meta: RouteMeta, captures: Option>) { let meta = (meta, captures); let new = match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { - Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { - PendingUpdate::UnannounceAnnounce { old, new: meta } + Some(PendingUpdate::Unannounce(old, gone) | PendingUpdate::UnannounceAnnounce { old, gone, .. }) => { + PendingUpdate::UnannounceAnnounce { old, gone, new: meta } } Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), _ if self.delivered.contains(&prefix) => PendingUpdate::Restart(meta), @@ -1053,7 +1096,7 @@ impl OriginConsumerState { self.put(prefix, new); } - fn apply_unannounce(&mut self, prefix: Path, last: RouteMeta, captures: Option>) { + fn apply_unannounce(&mut self, prefix: Path, last: RouteMeta, captures: Option>, gone: Gone) { let last = (last, captures); match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { // The pending announce was never delivered and neither was any earlier @@ -1064,13 +1107,13 @@ impl OriginConsumerState { } // Either nothing is pending or the pending announce was a metadata // update on a delivered route; the consumer still owes a retraction. - None | Some(PendingUpdate::Announce(_) | PendingUpdate::Restart(..) | PendingUpdate::Unannounce(_)) => { - self.put(prefix, PendingUpdate::Unannounce(last)); + None | Some(PendingUpdate::Announce(_) | PendingUpdate::Restart(..) | PendingUpdate::Unannounce(..)) => { + self.put(prefix, PendingUpdate::Unannounce(last, gone)); } // The embedded announce cancels with this retraction; the consumer still - // needs the leading one. + // needs the leading one, reported as the later one went. Some(PendingUpdate::UnannounceAnnounce { old, .. }) => { - self.put(prefix, PendingUpdate::Unannounce(old)); + self.put(prefix, PendingUpdate::Unannounce(old, gone)); } } } @@ -1187,16 +1230,16 @@ impl OriginConsumerState { self.delivered.insert(prefix.clone()); (meta, RouteKind::Restart) } - PendingUpdate::Unannounce(meta) => { + PendingUpdate::Unannounce(meta, gone) => { self.delivered.remove(&prefix); - (meta, RouteKind::End) + (meta, gone.kind()) } - PendingUpdate::UnannounceAnnounce { old, new } => { + PendingUpdate::UnannounceAnnounce { old, gone, new } => { // Deliver the retraction now; leave the trailing announce pending so // the next take returns it for the same prefix. self.delivered.remove(&prefix); self.insert(prefix.clone(), PendingUpdate::Announce(new), owed); - (old, RouteKind::End) + (old, gone.kind()) } }; let announce = Announce { @@ -1212,6 +1255,7 @@ impl OriginConsumerState { auth: meta.5, node: meta.6, draining: false, + available: meta.7, }, }; (RouteEvent { kind, announce }, owed) @@ -1252,6 +1296,8 @@ struct RouteEntry { /// Whether the route ranks last; see [`RouteInfo::draining`]. Local selection state, /// so never part of [`Self::meta`]. draining: bool, + /// Whether a usable path reaches the route's origin; see [`RouteInfo::available`]. + available: bool, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a [`Claim`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -1319,12 +1365,13 @@ impl RouteEntry { self.epoch, self.auth.clone(), self.node, + self.available, ) } /// Whether [`Self::meta`] would equal `meta`, without building it. fn delivers(&self, meta: &RouteMeta) -> bool { - let (hops, cost, source, instance, epoch, auth, node) = meta; + let (hops, cost, source, instance, epoch, auth, node, available) = meta; *hops == self.hops && *cost == self.cost && *source == self.entered() @@ -1332,6 +1379,7 @@ impl RouteEntry { && *epoch == self.epoch && *auth == self.auth && *node == self.node + && *available == self.available } /// Whether a request for `path` can be served through this entry. A served @@ -1661,13 +1709,14 @@ impl TableCursor { && self.named.as_ref().is_none_or(|(mount, _)| mount.names(&entry.prefix)) } - /// Deliver a changed winner at this presented prefix. - fn update(&mut self, presented: &Path, best: Option<&RouteEntry>) { + /// Deliver a changed winner at this presented prefix, reporting a route left with + /// none as `gone`. + fn update(&mut self, presented: &Path, best: Option<&RouteEntry>, gone: Gone) { let Some(entry) = best else { if let Some(last) = self.current.remove(presented) && let Ok(mut state) = self.state.write() { - state.apply_unannounce(self.under.rejoin(presented), last.meta, last.captures); + state.apply_unannounce(self.under.rejoin(presented), last.meta, last.captures, gone); } return; }; @@ -1701,7 +1750,12 @@ impl TableCursor { // identity explicitly so consumers keyed on it remove it. Some(previous) if previous.captures != captures => { if let Ok(mut state) = self.state.write() { - state.apply_unannounce(self.under.rejoin(presented), previous.meta, previous.captures); + state.apply_unannounce( + self.under.rejoin(presented), + previous.meta, + previous.captures, + Gone::End, + ); state.apply_announce(self.under.rejoin(presented), meta, captures); } } @@ -2044,11 +2098,17 @@ pub struct RouteEvent { pub enum RouteKind { /// A route now covers the prefix; the cursor had none there. Start, - /// The route covering the prefix changed hops, cost or epoch, with the same instance; - /// it is delivered in place. + /// The route covering the prefix changed hops, cost, epoch or + /// [availability](RouteInfo::available), with the same instance; it is delivered in + /// place. Update, - /// No route covers the prefix any more. + /// The origin withdrew the route, as far as this origin knows, and no other covers + /// the prefix. End, + /// This origin let go of the route, and no other covers the prefix: the session it + /// arrived on closed, or the route layer lost every path to it. Its origin may still + /// serve it; only an [`End`](Self::End) says it stopped. + Offline, /// Another [`Instance`] now serves the prefix: another stream, such as a publisher /// that restarted, a redundant ingest switching, or a pool failing over. /// @@ -2451,6 +2511,7 @@ impl Producer { state: Arc::new(RouteState { announcement: entry, control: kio::Lock::new(control), + forwarded: false, _keepalive: self.tasks.keepalive(), }), broadcast: source.clone(), @@ -2669,10 +2730,13 @@ impl Producer { None, )?; let control = RouteControl::new(options, &announcement, None); + // A session's route outlives nothing it was given: dropping it means the session lost it. + let forwarded = publisher.is_none(); Ok(Claim { state: Arc::new(RouteState { announcement, control: kio::Lock::new(control), + forwarded, _keepalive: self.tasks.keepalive(), }), server: serve, @@ -2908,6 +2972,7 @@ impl Announcing { link: self.link, node: route.node, draining: route.draining, + available: route.available, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -2917,7 +2982,7 @@ impl Announcing { lends: serving.lends, lent: None, }); - shared.sync_route(prefix, claim); + shared.sync_route(prefix, claim, Gone::End); entries.push((prefix.clone(), id)); } drop(shared); @@ -3004,6 +3069,9 @@ struct RouteState { /// The advertisement, retracted on drop. announcement: AnnounceProducer, control: kio::Lock, + /// Whether a session landed it for its peer, so dropping it means the session lost + /// the route ([`RouteKind::Offline`]), not that its origin ended it. + forwarded: bool, /// A route is lifecycle work: the origin's driver keeps running for as long as one /// lives, even once every producer handle is gone. _keepalive: Keepalive, @@ -3061,7 +3129,7 @@ impl RouteState { // first or finds the entries gone, never advertises after it. let mut control = self.control.lock(); let request = control.request.take(); - self.announcement.retract(false); + self.announcement.retract(Retract::Ended); drop(control); // Dropped outside every lock: refusing it wakes the requesters. drop(request); @@ -3183,7 +3251,7 @@ impl Route { // A failed answer can never announce. Retracted under the control lock, as // `ended` does, so a clone's racing announce finds the entries gone rather than // advertising through the ordinary branch now that the request is taken. - self.state.announcement.retract(false); + self.state.announcement.retract(Retract::Ended); drop(control); // The claim moved past the instance the job read: its requesters wait on for // the handler to prepare the job again under the new one. @@ -3385,10 +3453,18 @@ impl Claim { } /// Retire an explicit peer withdrawal, hiding derived paths only once its - /// last live advertisement at the prefix is gone. A lost session just drops. + /// last live advertisement at the prefix is gone. Readers see it end. A lost + /// session just drops, and readers see it go offline. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn withdrawn(self) { - self.state.announcement.retract(true); + self.state.announcement.retract(Retract::Withdrawn); + } + + /// Retire a route its announcer ended without a hop chain to stale, such as a + /// moq-transport PUBLISH_NAMESPACE_DONE: readers see it end, not go offline. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn ended(self) { + self.state.announcement.retract(Retract::Ended); } } @@ -3396,10 +3472,24 @@ impl Drop for RouteState { fn drop(&mut self) { // Retract before a job's unanswered request drops, so its requesters are refused // rather than asked again through this route. - self.announcement.retract(false); + self.announcement.retract(match self.forwarded { + true => Retract::Lost, + false => Retract::Ended, + }); } } +/// Why an advertisement leaves the table. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Retract { + /// The session it arrived on lost it: readers see it go offline. + Lost, + /// Its announcer ended it: readers see it end. + Ended, + /// A peer withdrew it, which also stales the routes derived through that peer. + Withdrawn, +} + /// The write half of an advertisement: a live claim that paths under a /// [`Pattern`] can be served. /// @@ -3498,9 +3588,10 @@ impl AnnounceProducer { entry.via = route.via; entry.node = route.node; entry.draining = route.draining; + entry.available = route.available; entry.advertised = true; let claim = entry.claim.clone(); - shared.sync_route(prefix, &claim); + shared.sync_route(prefix, &claim, Gone::End); if let Some(((lender, lender_id), instance)) = spent { shared.advance(&lender, lender_id, &instance); } @@ -3558,6 +3649,7 @@ impl AnnounceProducer { source: entry.entered(), node: entry.node, draining: entry.draining, + available: entry.available, }) } @@ -3585,7 +3677,7 @@ impl AnnounceProducer { } let claim = entry.claim.clone(); let spent = entry.lent.take().map(|lender| (lender, entry.instance.clone())); - shared.sync_route(prefix, &claim); + shared.sync_route(prefix, &claim, Gone::End); if let Some(((lender, lender_id), instance)) = spent { shared.advance(&lender, lender_id, &instance); } @@ -3603,7 +3695,7 @@ impl AnnounceProducer { /// Retract the route now: remove its table entries and reject anything still /// waiting on its queue. Idempotent, and what dropping the advertisement does. - fn retract(&self, withdrawn: bool) { + fn retract(&self, how: Retract) { let mut shared = self.shared.lock(); let guard = self.guard.lock().take(); let mut servers = Vec::new(); @@ -3616,7 +3708,7 @@ impl AnnounceProducer { // prefix. Remove this session's route before testing the remaining // claims, under the same lock, so overlapping withdrawals cannot // invalidate a newer advertisement or miss the last withdrawal. - if withdrawn + if how == Retract::Withdrawn && let Some(&peer) = entry.hops.iter().last() && peer != Hop::UNKNOWN && !shared @@ -3627,7 +3719,11 @@ impl AnnounceProducer { shared.withdrawn.entry(prefix.clone()).or_default().insert(peer); shared.restale(prefix); } - shared.sync_route(&entry.prefix, &entry.claim); + let gone = match how { + Retract::Lost => Gone::Offline, + Retract::Ended | Retract::Withdrawn => Gone::End, + }; + shared.sync_route(&entry.prefix, &entry.claim, gone); if entry.advertised && let Some((lender, lender_id)) = &entry.lent { @@ -3659,7 +3755,7 @@ impl AnnounceProducer { impl Drop for AnnounceProducer { fn drop(&mut self) { - self.retract(false); + self.retract(Retract::Ended); } } @@ -3733,7 +3829,7 @@ impl OwnClaims { announce, } => announce, RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, announce, } => { self.best.remove(&announce.prefix); @@ -3764,7 +3860,7 @@ impl OwnClaims { fn local(&mut self, event: RouteEvent) { match event { RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, announce, } => { self.local.remove(&announce.prefix); @@ -6061,7 +6157,7 @@ impl OriginState { } } if let Some(claim) = changed { - self.sync_route(prefix, &claim); + self.sync_route(prefix, &claim, Gone::End); } } @@ -6109,7 +6205,7 @@ impl OriginState { /// every cursor it can present on. Called after an entry covering `prefix` /// was added, updated, or removed. `claim` is `prefix`'s [`prefix_claim`], /// held by the entry that changed. - fn sync_route(&mut self, prefix: &Path, claim: &Pattern) { + fn sync_route(&mut self, prefix: &Path, claim: &Pattern, gone: Gone) { // Split borrows: the recompute reads `routes` while mutating a cursor. let routes = &self.routes; let mut candidates = None; @@ -6120,7 +6216,7 @@ impl OriginState { if let Some(presented) = cursor.presented(prefix, claim) { if presented.is_empty() { // A cursor root can collapse several covering prefixes into one. - Self::sync_cursor(routes, cursor, &presented); + Self::sync_cursor(routes, cursor, &presented, gone); } else { // Every non-root presentation refers to this exact prefix. Rank // its routes once, then take each cursor's first eligible visible entry. @@ -6138,7 +6234,7 @@ impl OriginState { .filter(|entry| cursor.visible(entry)); let epochs = Epochs::of(visible.clone()); let best = visible.clone().find(|entry| !epochs.superseded(entry)); - cursor.update(&presented, best); + cursor.update(&presented, best, gone); } } } @@ -6256,7 +6352,7 @@ impl OriginState { /// Recompute the best visible route presenting at `presented` (relative) for /// one cursor and deliver the change, if any. - fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path) { + fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path, gone: Gone) { // The entries presenting here are the ones announced at the absolute // prefix, or, for the cursor's own root, at the root and every prefix // above it (all of which present as the empty path). Among them, the @@ -6270,7 +6366,7 @@ impl OriginState { } }; - cursor.update(presented, best); + cursor.update(presented, best, gone); } /// Register a cursor and replay the current best route per presented prefix. @@ -6304,7 +6400,7 @@ impl OriginState { } } for p in &presented { - Self::sync_cursor(&self.routes, &mut cursor, p); + Self::sync_cursor(&self.routes, &mut cursor, p, Gone::End); } for head in &cursor.heads { self.routes.add_cursor(head, id); @@ -6444,7 +6540,7 @@ impl OriginState { entry.instance = Instance::random(); entry.epoch = 0; let claim = entry.claim.clone(); - self.sync_route(prefix, &claim); + self.sync_route(prefix, &claim, Gone::End); } /// Whether `instance` still serves `path` as a request would resolve it now: the winning @@ -6873,7 +6969,8 @@ impl Requesting { loop { match discovery.poll_next(waiter) { Poll::Ready(Some(RouteEvent { - kind: RouteKind::End, .. + kind: RouteKind::End | RouteKind::Offline, + .. })) => self.routed = false, Poll::Ready(Some(RouteEvent { kind: RouteKind::Start | RouteKind::Restart | RouteKind::Update, @@ -8038,6 +8135,21 @@ impl Routes { } } + #[cfg(test)] + pub(crate) async fn expect_next_offline(&mut self, expected: impl TryInto) { + match self.expect_next().await { + RouteEvent { + kind: RouteKind::Offline, + announce, + } => assert_eq!( + announce.prefix, + expected.try_into().expect("expected path"), + "wrong prefix" + ), + other => panic!("should go offline: got {other:?}"), + } + } + fn new( root: Path, cursors: Vec, @@ -8093,7 +8205,7 @@ impl Routes { /// expire: later ones for the prefix are updates, which it never holds. fn quarantine(&mut self, event: &RouteEvent, now: Instant) { let RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, announce: update, } = event else { @@ -8183,7 +8295,7 @@ impl Routes { .or_insert_with(|| scope.announce()); } RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, announce, } => { self.guards.remove(&announce.prefix); @@ -8331,14 +8443,14 @@ impl Routes { for (presented, CursorRoute { meta, captures, .. }) in &cursor.current { let prefix = cursor.under.rejoin(presented); if revoked.contains(&prefix) { - state.apply_unannounce(prefix, meta.clone(), captures.clone()); + state.apply_unannounce(prefix, meta.clone(), captures.clone(), Gone::End); continue; } if advertised.contains(&prefix) { if state.pending.get(&prefix).is_some_and(|pending| { matches!( pending.update, - Some(PendingUpdate::Unannounce(_) | PendingUpdate::UnannounceAnnounce { .. }) + Some(PendingUpdate::Unannounce(..) | PendingUpdate::UnannounceAnnounce { .. }) ) }) { state.apply_announce(prefix, meta.clone(), captures.clone()); @@ -11172,7 +11284,7 @@ mod tests { let replacement = broadcast::Info::new().produce(); queued(&echo).await.serve(&replacement); resolved.assert_not_closed(); - view.expect_next_ended("room").await; + view.expect_next_offline("room").await; } /// A route that ends abruptly releases its open groups, and the next route may @@ -11586,7 +11698,7 @@ mod tests { assert_eq!(announced.expect_next_restarted("live").await.cost, Cost::new(3)); drop(first); - announced.expect_next_ended("live").await; + announced.expect_next_offline("live").await; announced.assert_next_wait(); } @@ -11711,8 +11823,8 @@ mod tests { drop(video_route); assert_eq!(all.expect_next_active("room/live").await.cost, Cost::new(9)); assert_eq!(rooted.expect_next_active("").await.cost, Cost::new(9)); - excluded.expect_next_ended("room/live").await; - video.expect_next_ended("room/live").await; + excluded.expect_next_offline("room/live").await; + video.expect_next_offline("room/live").await; } #[kio::test] @@ -11791,7 +11903,7 @@ mod tests { assert!(matches!( next(), Some(Some(RouteEvent { - kind: RouteKind::End, + kind: RouteKind::Offline, .. })) )); @@ -11805,7 +11917,7 @@ mod tests { announced.expect_next_active("live").await; drop(server); - announced.expect_next_ended("live").await; + announced.expect_next_offline("live").await; } #[kio::test] @@ -12100,10 +12212,10 @@ mod tests { drop(route); match announced.try_next() { Some(RouteEvent { - kind: RouteKind::End, + kind: RouteKind::Offline, announce: end, }) => assert_eq!(end.prefix, Path::new(&format!("room/{i}")).unwrap()), - other => panic!("should be a retraction: got {other:?}"), + other => panic!("should go offline: got {other:?}"), } } @@ -12153,7 +12265,7 @@ mod tests { // The last route goes: retracted at once. drop(farther); - announced.expect_next_ended("room").await; + announced.expect_next_offline("room").await; } /// After a retraction, a worse path to the same publisher waits out the @@ -12177,7 +12289,7 @@ mod tests { let near = peer.forward("named", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("named").await; drop(near); - announced.expect_next_ended("named").await; + announced.expect_next_offline("named").await; let start = kio::clock::now(); let _stale = peer.forward("named", route(&[9, 3, 4], 4)).unwrap(); match announced.next().await.unwrap() { @@ -12192,7 +12304,7 @@ mod tests { let near = peer.forward("anonymous", route(&[0, 2], 2)).unwrap(); announced.expect_next_active("anonymous").await; drop(near); - announced.expect_next_ended("anonymous").await; + announced.expect_next_offline("anonymous").await; let _other = peer.forward("anonymous", route(&[0, 3, 4], 4)).unwrap(); announced.expect_next_active("anonymous").await; @@ -12203,7 +12315,7 @@ mod tests { let churn = peer.forward("churn", route(&chain, 2)).unwrap(); announced.expect_next_active("churn").await; drop(churn); - announced.expect_next_ended("churn").await; + announced.expect_next_offline("churn").await; } assert_eq!(announced.anchoring.len(), queued + 1); @@ -12212,7 +12324,7 @@ mod tests { let gone = peer.forward("gone", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("gone").await; drop(gone); - announced.expect_next_ended("gone").await; + announced.expect_next_offline("gone").await; assert!(!announced.retracted.is_empty()); for _ in 0..2 { announced.assert_next_wait(); @@ -12363,7 +12475,7 @@ mod tests { .unwrap(); assert_eq!(local.expect_next_active("both").await.source(), Source::Local); drop(ingest); - local.expect_next_ended("both").await; + local.expect_next_offline("both").await; // Resolution agrees with the cursor: a peer-only path is unroutable here, // while the full view queues the request on the peer's route. @@ -12420,7 +12532,7 @@ mod tests { .unwrap(); assert_eq!(toward_core.expect_next_active("both").await.source(), Source::Peer(8)); drop(mesh); - toward_core.expect_next_ended("both").await; + toward_core.expect_next_offline("both").await; // Resolution agrees with the cursor: an upstream-only path is unroutable // toward an upstream, while the full view queues it on the core's route. @@ -13157,7 +13269,7 @@ mod tests { // Nothing serves an uncovered path once the handler is gone. drop(dynamic); - announced.expect_next_ended("").await; + announced.expect_next_offline("").await; let err = consumer .request("something/else", None) .now_or_never() diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 55d57f21f26..c17fc75ddbc 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -85,8 +85,9 @@ pub(crate) enum Output { claim: Path, served: Served, }, - /// Stop serving `claim` of `id` through this stream. - Unserve { id: Id, claim: Path }, + /// Stop serving `claim` of `id` through this stream. `ended` when the claim's origin + /// withdrew it, as far as this node knows; otherwise the node let go of it here. + Unserve { id: Id, claim: Path, ended: bool }, } /// The node every lite-07 session of one origin shares. @@ -232,7 +233,7 @@ impl Outbox { } // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. - Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim, .. } => { let key = (id, claim.clone()); if let Some(&seq) = self.serving.get(&key) && let Output::Serve { served, .. } = &output @@ -267,7 +268,7 @@ impl Outbox { } None => { let (_, output) = self.serves.pop_first()?; - if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim } = &output { + if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim, .. } = &output { self.serving.remove(&(*id, claim.clone())); } return Some(output); @@ -388,6 +389,7 @@ impl Home { Output::Unserve { id, claim: claim.clone(), + ended: false, }, ); } @@ -656,13 +658,17 @@ impl State { self.router.echo(key, id); } } - // The shown claims are read in place; the events only say which changed. - self.router.events(); + // The shown claims are read in place; the events only say which changed, and + // whether a claim that went was ended by its origin or let go here. + let mut ended = BTreeMap::new(); + for event in self.router.events() { + ended.insert((event.id, event.claim), event.kind == super::Kind::End); + } let changed = self.router.changed(); let declared = self.router.declared(); for (id, claims) in changed { if Some(id) != declared { - self.home(id, claims); + self.home(id, claims, &ended); } } } @@ -671,7 +677,7 @@ impl State { /// route to it that can carry the claim, keeping it where it is, unavailable, while /// there is none. Only `claims`, whose shown state changed, are served anew, unless /// the routes changed too. - fn home(&mut self, id: Id, claims: BTreeSet) { + fn home(&mut self, id: Id, claims: BTreeSet, ended: &BTreeMap<(Id, Path), bool>) { let routes: Routes = (self.router.routes(id).into_iter()) .map(|(key, metric)| (key, metric, self.router.name(key, id))) .collect(); @@ -756,6 +762,7 @@ impl State { let Some(home) = self.homes.remove(&(id, claim.clone())) else { continue; }; + let ended = ended.get(&(id, claim.clone())).copied().unwrap_or(false); for key in home.keys.into_keys() { push( &self.queues, @@ -763,6 +770,7 @@ impl State { Output::Unserve { id, claim: claim.clone(), + ended, }, ); } @@ -922,7 +930,8 @@ mod tests { served(outputs(&receive)), [Output::Unserve { id: 1, - claim: p("live/x") + claim: p("live/x"), + ended: true }] ); } @@ -962,7 +971,8 @@ mod tests { served(outputs(&far)), [Output::Unserve { id: 1, - claim: p("live/x") + claim: p("live/x"), + ended: false }] ); assert!(served(outputs(&near)).is_empty(), "the remaining route serves on"); diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 707111ab18c..94935f63329 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -1097,6 +1097,11 @@ impl Slots { } } + /// A count of its own under the same cap, starting empty and reported nowhere. + pub(crate) fn split(&self) -> Self { + Self::new(self.max) + } + /// Report how many are held to `stats`, as `cap`'s peak. pub(crate) fn with_stats(mut self, stats: &crate::stats::Session, cap: crate::stats::Cap) -> Self { stats.track_held(cap, &self.live); diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index bc9eadc8c2c..3e12f3569d8 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -121,6 +121,22 @@ async fn lite_07_opens_an_auth_stream_per_side() { .expect("timed out"); } +/// Floors certify withdrawals only on lite-07. +#[kio::test] +async fn lite_07_a_scoped_withdrawal_ends() { + a_scoped_withdrawal_ends(LITE_07).await +} + +#[kio::test] +async fn lite_07_a_scope_change_reopens_without_letting_go() { + a_scope_change_reopens_without_letting_go(LITE_07).await +} + +#[kio::test] +async fn lite_07_a_scope_change_at_the_announce_cap_keeps_the_session() { + a_scope_change_at_the_announce_cap_keeps_the_session(LITE_07).await +} + #[kio::test] async fn lite_05_has_no_grant() { older_versions_have_no_grant("moq-lite-05").await @@ -416,7 +432,7 @@ async fn wait_announced(origin: &origin::Consumer, path: &str, active: bool) { live.insert(update.prefix.to_string()); } moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: update, } => { live.remove(update.prefix.as_str()); @@ -451,6 +467,8 @@ struct Options { client_token: Option<&'static [u8]>, /// The handle the client builds its session on, when not its own. client_auth: Option, + /// The server's session limits, when not the defaults. + server_limits: Option, } struct Pair { @@ -501,6 +519,9 @@ async fn connect(opts: Options) -> Pair { if let Some(subscribe) = opts.server_subscribe { server = server.with_subscriber(subscribe); } + if let Some(limits) = opts.server_limits { + server = server.with_limits(limits); + } let observe = client_transport.clone(); let observe_server = server_transport.clone(); @@ -811,6 +832,128 @@ async fn tokens_union_and_withdrawing_one_shrinks_it(version: &'static str) { .expect("timed out"); } +/// The route events `routes` delivered for `path` so far. +fn kinds(routes: &mut origin::Routes, path: &str) -> Vec { + let mut kinds = Vec::new(); + while let Some(event) = futures::FutureExt::now_or_never(routes.next()).flatten() { + if event.announce.prefix.as_str() == path { + kinds.push(event.kind); + } + } + kinds +} + +/// A relay that limits its publisher to one prefix asks it for that prefix alone, not +/// for everything, so the publisher's floors certify a withdrawal: it ends rather than +/// going offline. +async fn a_scoped_withdrawal_ends(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let mut routes = relay.consume().routes(); + let route = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + route.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); + + route.withdraw(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::End]); + }) + .await + .expect("timed out"); +} + +/// Widening the limit to a second prefix reopens the relay's request above both, and +/// narrowing it reopens the request at the first again, each without letting go of what +/// the limit kept: the old request serves until the new one is synced. +async fn a_scope_change_reopens_without_letting_go(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let mut routes = relay.consume().routes(); + let a = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + a.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); + + // Widened past what the first request asked for: the new one carries `b`. + pair.server.auth().authorize(&grant(&["a", "b"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + let b = publisher.route("b/y", moq_net::origin::RouteOptions::hidden()).unwrap(); + b.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "b/y"), [origin::RouteKind::Start]); + assert_eq!(kinds(&mut routes, "a/x"), [], "the widening let go of a/x"); + + // Narrowed back to `a`, where a withdrawal is certified again. + pair.server.auth().authorize(&grant(&["a"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [], "the narrowing let go of a/x"); + a.withdraw(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::End]); + }) + .await + .expect("timed out"); +} + +/// A session holding all the announce slots it may has no room to hear its peer's claims +/// twice, so the old request gives its slots to the new one during a scope change, +/// rather than the handover running the peer past its cap. +async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let mut limits = moq_net::session::Limits::default(); + // One route and its two claims. + limits.announces = 3; + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + server_limits: Some(limits), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let x = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + x.announce().unwrap(); + let y = publisher.route("a/y", moq_net::origin::RouteOptions::hidden()).unwrap(); + y.announce().unwrap(); + wait_announced(&relay.consume(), "a/y", true).await; + let mut routes = relay.consume().routes(); + kinds(&mut routes, "a/x"); + + pair.server.auth().authorize(&grant(&["a", "b"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [], "the handover let go of a/x"); + assert_eq!(pair.server_transport.close_reason(), None); + assert_eq!(pair.client_transport.close_reason(), None); + }) + .await + .expect("timed out"); +} + /// An update replaces one token's grant and leaves the other alone. async fn an_update_replaces_one_tokens_grant(version: &'static str) { within(async { diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs index 170badef339..b2b94acedd9 100644 --- a/rs/moq-net/tests/integration/claim_continues.rs +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -112,6 +112,10 @@ async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Ins kind: origin::RouteKind::End, announce: a, } => ("end", a), + origin::RouteEvent { + kind: origin::RouteKind::Offline, + announce: a, + } => ("offline", a), }; events.push((kind, announce.prefix.to_string(), announce.route.instance().clone())); } diff --git a/rs/moq-net/tests/integration/dial_split_horizon.rs b/rs/moq-net/tests/integration/dial_split_horizon.rs index 4e5ed664a7a..738520d6aa4 100644 --- a/rs/moq-net/tests/integration/dial_split_horizon.rs +++ b/rs/moq-net/tests/integration/dial_split_horizon.rs @@ -108,7 +108,7 @@ async fn learned_hop(dialer: &origin::Producer, path: &str) -> u64 { announce, } => announce, moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, .. } => continue, }; diff --git a/rs/moq-net/tests/integration/extensions.rs b/rs/moq-net/tests/integration/extensions.rs index 1ff5117218b..b14b1e010da 100644 --- a/rs/moq-net/tests/integration/extensions.rs +++ b/rs/moq-net/tests/integration/extensions.rs @@ -54,7 +54,7 @@ fn announced(origin: &origin::Producer, path: &str) -> bool { live |= update.prefix.as_str() == path; } moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: update, } => { live &= update.prefix.as_str() != path; diff --git a/rs/moq-net/tests/integration/instance.rs b/rs/moq-net/tests/integration/instance.rs index 854b2a25323..f0f98a1d325 100644 --- a/rs/moq-net/tests/integration/instance.rs +++ b/rs/moq-net/tests/integration/instance.rs @@ -45,7 +45,7 @@ async fn ended(announced: &mut origin::Routes) { .expect("announce ended") { origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, .. } => {} other => panic!("expected an end, got {other:?}"), diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index eb3d6105e36..621ecab61d7 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -117,7 +117,7 @@ async fn synced(announced: &mut origin::Routes) -> Vec { announce, } => seen.push(announce.prefix.to_string()), origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce, } => seen.retain(|prefix| *prefix != announce.prefix.as_str()), origin::RouteEvent { @@ -438,7 +438,7 @@ async fn settled(announced: &mut origin::Routes, held: &mut Vec) -> bool announce, } => held.push(announce.prefix.to_string()), origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce, } => { ended = true; diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs index bf40f0c6869..dc3cd74541b 100644 --- a/rs/moq-net/tests/integration/mount.rs +++ b/rs/moq-net/tests/integration/mount.rs @@ -75,6 +75,10 @@ async fn drain(routes: &mut origin::Routes) -> Vec<(bool, String)> { kind: origin::RouteKind::End, announce, } => events.push((false, announce.prefix.to_string())), + origin::RouteEvent { + kind: origin::RouteKind::Offline, + announce, + } => events.push((false, format!("offline {}", announce.prefix))), _ => {} } } @@ -103,6 +107,16 @@ fn ended(paths: &[&str]) -> Vec<(bool, String)> { paths.iter().map(|path| (false, path.to_string())).collect() } +/// What a reader on `version` sees once `paths` leave without a withdrawal it can +/// certify: offline where the route layer carries it, ended where only an ANNOUNCE +/// says it went. +fn lost(version: Version, paths: &[&str]) -> Vec<(bool, String)> { + match version.to_string().as_str() { + "moq-lite-07-wip" => paths.iter().map(|path| (false, format!("offline {path}"))).collect(), + _ => ended(paths), + } +} + async fn mounted(version: &str) { let version: Version = version.parse().unwrap(); let source = produce_origin(1); @@ -115,9 +129,10 @@ async fn mounted(version: &str) { assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); - // What leaves the target leaves the mount. + // What leaves the target leaves the mount. The route layer carries the mount's + // withdrawal without a floor certifying it, so it goes offline there. drop(published.remove(0)); - assert_eq!(drain(&mut routes).await, ended(&["shared/cam"]), "{version}"); + assert_eq!(drain(&mut routes).await, lost(version, &["shared/cam"]), "{version}"); } /// A session rooted inside the mount reads only the target. @@ -152,7 +167,11 @@ async fn relayed(version: &str) { drop(upstream); kio::clock::sleep(Duration::from_millis(500)).await; - assert_eq!(drain(&mut routes).await, ended(&["own", "shared/cam"]), "{version}"); + assert_eq!( + drain(&mut routes).await, + lost(version, &["own", "shared/cam"]), + "{version}" + ); } macro_rules! mounted_tests { diff --git a/rs/moq-net/tests/integration/multi_cdn.rs b/rs/moq-net/tests/integration/multi_cdn.rs index 0dd9d2fc898..9f6bb2c0061 100644 --- a/rs/moq-net/tests/integration/multi_cdn.rs +++ b/rs/moq-net/tests/integration/multi_cdn.rs @@ -138,7 +138,7 @@ fn via_a(announce: &origin::RouteEvent) -> bool { announce: a, } => &a.route, origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, .. } => return false, }; diff --git a/rs/moq-net/tests/integration/session_panic.rs b/rs/moq-net/tests/integration/session_panic.rs index a7817c63dbd..59d11bc9d83 100644 --- a/rs/moq-net/tests/integration/session_panic.rs +++ b/rs/moq-net/tests/integration/session_panic.rs @@ -86,7 +86,7 @@ async fn until(announced: &mut origin::Routes, path: &str, active: bool) { announce, } => (announce, true), origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce, } => (announce, false), }; diff --git a/rs/moq-net/tests/integration/support/mod.rs b/rs/moq-net/tests/integration/support/mod.rs index 6a6d4ba6975..5cbad38e63f 100644 --- a/rs/moq-net/tests/integration/support/mod.rs +++ b/rs/moq-net/tests/integration/support/mod.rs @@ -56,6 +56,7 @@ pub enum Kind { Update, Restart, End, + Offline, } /// Every update per prefix the watcher has seen. Await [`kio::clock::sim::idle`] @@ -94,6 +95,10 @@ pub fn watch(mut announced: origin::Routes) -> mpsc::UnboundedReceiver<(String, kind: origin::RouteKind::End, announce, } => (Kind::End, announce), + origin::RouteEvent { + kind: origin::RouteKind::Offline, + announce, + } => (Kind::Offline, announce), }; if tx.unbounded_send((announce.prefix.to_string(), kind)).is_err() { break; diff --git a/rs/moq-net/tests/integration/synced.rs b/rs/moq-net/tests/integration/synced.rs index 13fa2445dfc..ce0dece509f 100644 --- a/rs/moq-net/tests/integration/synced.rs +++ b/rs/moq-net/tests/integration/synced.rs @@ -37,7 +37,7 @@ async fn read_until_synced(announced: &mut origin::Routes) -> BTreeSet { seen.insert(announce.prefix.to_string()); } origin::RouteEvent { - kind: origin::RouteKind::End, + kind: origin::RouteKind::End | origin::RouteKind::Offline, announce, } => { seen.remove(announce.prefix.as_str()); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index f8758598033..844ee1e1ba6 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2655,7 +2655,7 @@ mod tests { announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index c78f2d3c8b9..af890edbce1 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1338,7 +1338,7 @@ mod tests { announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index cba2e1a92f5..14f53048c8e 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -1410,7 +1410,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 365671688ab..a02653d5758 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1856,7 +1856,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/cluster_idle.rs b/rs/moq-relay/tests/integration/cluster_idle.rs index d04c2e98672..25228bea3b3 100644 --- a/rs/moq-relay/tests/integration/cluster_idle.rs +++ b/rs/moq-relay/tests/integration/cluster_idle.rs @@ -135,8 +135,9 @@ async fn silent_peer_detected() { kio::clock::timeout(DETECT, async { loop { let event = announced.next().await.expect("origin closed"); + // A silent peer is lost, not withdrawn: its route goes offline. if let moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::Offline, announce: update, } = event && update.prefix.as_str() == "peer" { diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 1bd307bd891..1d1c510a640 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -319,7 +319,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/connection.rs b/rs/moq-relay/tests/integration/connection.rs index 816aaeb6af9..a5a00a600f3 100644 --- a/rs/moq-relay/tests/integration/connection.rs +++ b/rs/moq-relay/tests/integration/connection.rs @@ -110,7 +110,7 @@ async fn next_active(announced: &mut moq_net::origin::Routes) -> String { announce: route, } => return route.prefix.to_string(), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, .. } => {} } diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 954dd02cc0d..6b832694d06 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -605,7 +605,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/memory.rs b/rs/moq-relay/tests/integration/memory.rs index a3d9732b1c3..91971cbe19f 100644 --- a/rs/moq-relay/tests/integration/memory.rs +++ b/rs/moq-relay/tests/integration/memory.rs @@ -324,7 +324,7 @@ pub fn record(mut announced: moq_net::origin::Routes) -> std::sync::Arc (route, false), moq_net::origin::RouteEvent { diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index a1204c93f76..d908ed64694 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -930,7 +930,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/runtime_workers.rs b/rs/moq-relay/tests/integration/runtime_workers.rs index 46519ef40da..1f8df92d82c 100644 --- a/rs/moq-relay/tests/integration/runtime_workers.rs +++ b/rs/moq-relay/tests/integration/runtime_workers.rs @@ -172,7 +172,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/smoke.rs b/rs/moq-relay/tests/integration/smoke.rs index a94f8960d4c..d93f9e95fb1 100644 --- a/rs/moq-relay/tests/integration/smoke.rs +++ b/rs/moq-relay/tests/integration/smoke.rs @@ -1004,7 +1004,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index 889bc81da9a..e6a6664a88c 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -356,7 +356,7 @@ async fn synced_listing(url: url::Url) -> BTreeSet { use moq_net::origin::{RouteEvent, RouteKind}; match next { Some(RouteEvent { - kind: RouteKind::End, + kind: RouteKind::End | RouteKind::Offline, announce: route, }) => listed.remove(route.prefix.as_str()), Some(RouteEvent { diff --git a/rs/moq-room/src/room.rs b/rs/moq-room/src/room.rs index c394de44207..541f0c3820a 100644 --- a/rs/moq-room/src/room.rs +++ b/rs/moq-room/src/room.rs @@ -75,7 +75,7 @@ impl Room { loop { match self.announced.poll_next(waiter) { Poll::Ready(Some(announce::Event::Present(announce))) => self.listed(announce.prefix), - Poll::Ready(Some(announce::Event::Absent(announce))) => { + Poll::Ready(Some(announce::Event::Absent(announce) | announce::Event::Offline(announce))) => { let Some(index) = self.participants.iter().position(|p| p.path == announce.prefix) else { continue; }; @@ -87,6 +87,8 @@ impl Room { broadcast: None, })); } + // A participant stays listed while its path is held; its follow resumes it. + Poll::Ready(Some(announce::Event::Available(_) | announce::Event::Unavailable(_))) => {} Poll::Ready(None) => return Poll::Ready(None), Poll::Pending => break, } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 9b0fa66ef94..d4c5319ec1a 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -822,7 +822,7 @@ impl Merged { changed |= self.apply_announce(&update, instance, true) } Poll::Ready(Some(moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: update, })) => { let instance = update.route.instance().clone(); @@ -1164,7 +1164,7 @@ mod tests { announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index 4f942e1bfbb..4a6ea8c3f18 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -183,7 +183,7 @@ mod tests { announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index cc046adb132..10df589f7bd 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1702,7 +1702,7 @@ mod tests { announce: route, } => Some((route, true)), moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce: route, } => Some((route, false)), } diff --git a/rs/moq-tokio/examples/clock.rs b/rs/moq-tokio/examples/clock.rs index d88c9b9c613..9f5dcb7f8b8 100644 --- a/rs/moq-tokio/examples/clock.rs +++ b/rs/moq-tokio/examples/clock.rs @@ -118,8 +118,14 @@ async fn main() -> anyhow::Result<()> { clock = Some(Subscriber::new(track)); } announce::Event::Absent(update) => { + tracing::warn!(broadcast = %update.prefix, "broadcast ended, waiting..."); + } + announce::Event::Offline(update) | announce::Event::Unavailable(update) => { tracing::warn!(broadcast = %update.prefix, "broadcast is offline, waiting..."); } + announce::Event::Available(update) => { + tracing::info!(broadcast = %update.prefix, "broadcast is reachable again"); + } }, res = reconnect.closed() => return Ok(res?), // Drops the previous subscriber on each new announce. diff --git a/rs/moq-tokio/src/origin.rs b/rs/moq-tokio/src/origin.rs index 4a44f316071..3761922dae5 100644 --- a/rs/moq-tokio/src/origin.rs +++ b/rs/moq-tokio/src/origin.rs @@ -41,9 +41,14 @@ mod tests { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 39d9c9c3ec6..0ab848a120f 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1963,9 +1963,14 @@ mod tests { /// The next route and whether it is active. #[cfg(all(feature = "uds", unix))] async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index eba13076027..a18aee9849e 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -1186,8 +1186,11 @@ async fn quic_windows() { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index e71f655a0ab..07eaf63b1ce 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -819,7 +819,7 @@ async fn next_route(routes: &mut moq_net::origin::Routes) -> (moq_net::announce: .expect("origin closed"); match event { moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End, + kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline, announce, } => (announce, false), moq_net::origin::RouteEvent { @@ -3515,8 +3515,11 @@ async fn abort_carries_its_code_to_the_peer() { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-webrtc/src/lib.rs b/rs/moq-webrtc/src/lib.rs index 8bb4df6704d..b460e2a6639 100644 --- a/rs/moq-webrtc/src/lib.rs +++ b/rs/moq-webrtc/src/lib.rs @@ -75,9 +75,14 @@ mod tests { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 2bd4145e184..1c6f74ba093 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -61,9 +61,11 @@ public typealias RouteInfo = MoqFFI.MoqRouteInfo /// wildcard matched (`captures`, a required list, empty without wildcards), and the `route` serving it. /// Resolve a path with `OriginConsumer.request`. public typealias Announce = MoqFFI.MoqAnnounce -/// What an `AnnounceConsumer` yields: `.present` or `.absent`, each carrying an -/// `Announce`. Another publisher instance taking a path over keeps it present: -/// hold a `Following` to see the broadcast change. +/// What an `AnnounceConsumer` yields, each carrying an `Announce`: `.present` once a route +/// covers the path, `.unavailable` and `.available` as a present path loses and recovers a +/// usable path to its origin, `.absent` once the origin withdraws it, and `.offline` once this +/// side lets go of it, which its origin may still serve. Another publisher instance taking a +/// path over keeps it present: hold a `Following` to see the broadcast change. public typealias AnnounceEvent = MoqFFI.MoqAnnounceEvent /// Per-subscription delivery preferences: priority, delay, the ranges of the track to /// deliver, and group ordering. diff --git a/swift/Sources/Moq/Origin.swift b/swift/Sources/Moq/Origin.swift index 57bddc98626..2819be227f1 100644 --- a/swift/Sources/Moq/Origin.swift +++ b/swift/Sources/Moq/Origin.swift @@ -201,7 +201,7 @@ public final class OriginConsumer: Sendable { } /// List the paths under a literal prefix matching an optional pattern filter, as each - /// becomes present or absent. A new publisher instance keeps its path present, so use + /// becomes present, absent, or offline, or changes availability. A new publisher instance keeps its path present, so use /// `follow` to see it. Paths with a segment starting with `.` below the prefix are left /// out unless `hidden`. public func announced(prefix: String = "", filter: String? = nil, hidden: Bool = false) throws -> AnnounceConsumer { @@ -224,7 +224,8 @@ public final class OriginConsumer: Sendable { } } -/// A listing of paths as each becomes present or absent. Iterate directly: +/// A listing of paths as each becomes present, absent, or offline, or changes availability. +/// Iterate directly: /// `for try await event in announced { ... }`. /// The sequence ends when the origin closes; cancelling the consuming task /// cancels the subscription. diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b67b6381510..5482bf08055 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -7,9 +7,10 @@ # name stays announced with nothing behind it. Explicit withdrawals still consume # the handle, so only the implicit retraction on session loss is removed. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs +index 45654a5656..8af799f04c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -8817,7 +8817,7 @@ impl Announced { +@@ -9749,7 +9749,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +19,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -8901,7 +8901,7 @@ fn mint_sources(subscriber: &Subscriber, entry +@@ -9831,7 +9831,7 @@ fn mint_sources(subscriber: &Subscriber, entry let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +28,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -8929,7 +8929,8 @@ struct AnnouncedRoute { +@@ -9859,7 +9859,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +38,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Queues this route on its prefix's ready set when its request queue wakes. -@@ -8949,7 +8950,7 @@ impl AnnouncedRoute { +@@ -9877,7 +9878,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -47,21 +48,13 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. waker: std::task::Waker::from(wake.clone()), wake, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs +index 20fe66dd38..91675a0b9b 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -354,7 +354,7 @@ impl Routed { - self.previous = previous; - self.outputs(subscriber); - for (_, served) in std::mem::take(&mut self.previous) { -- served.route.dynamic.withdrawn(); -+ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); - } - Ok(()) - } -@@ -398,7 +398,7 @@ impl Routed { - } => self.serve(subscriber, id, named, claim, served), - node::Output::Unserve { id, claim } => { - if let Some(served) = self.served.remove(&(id, claim)) { +@@ -402,7 +402,7 @@ impl Routed { + if let Some(served) = self.served.remove(&(id, claim)) + && ended + { - served.route.dynamic.withdrawn(); + std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); }