From 116aa13012ca785bfc80884fccad4e0b24f23f4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 07:18:57 -0800 Subject: [PATCH 01/11] chore(quest): claim quest/m2/cluster-routing/routes/availability.md Quest-Claim: 1dc886e941f195de From 0fc5f79aef9a4db2de260429d2c3daa5ffbd7ce5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 07:35:14 -0800 Subject: [PATCH 02/11] wip(net): announcement availability and offline in Rust Co-Authored-By: Claude Opus 5.5 --- rs/hang/examples/subscribe.rs | 13 + rs/moq-bench/src/connection.rs | 4 +- rs/moq-c/src/api.rs | 23 +- rs/moq-c/src/origin.rs | 6 + rs/moq-c/src/test.rs | 1 + rs/moq-cli/src/announced.rs | 4 +- rs/moq-cli/src/archive.rs | 7 +- rs/moq-cli/src/report.rs | 4 +- rs/moq-cli/tests/integration/instances.rs | 2 +- rs/moq-cli/tests/integration/rtsp.rs | 4 +- rs/moq-ffi/src/origin.rs | 39 ++- rs/moq-net/benches/announce.rs | 2 +- rs/moq-net/src/auth.rs | 2 +- rs/moq-net/src/ietf/publisher.rs | 4 +- rs/moq-net/src/ietf/subscriber.rs | 63 ++-- rs/moq-net/src/lite/publisher.rs | 4 +- rs/moq-net/src/lite/replay.rs | 2 +- rs/moq-net/src/lite/subscriber.rs | 4 +- rs/moq-net/src/lite/subscriber/routed.rs | 19 +- rs/moq-net/src/model/announced.rs | 70 ++++- rs/moq-net/src/model/follow.rs | 26 +- rs/moq-net/src/model/origin.rs | 275 ++++++++++++------ rs/moq-net/src/route/node.rs | 30 +- .../integration/announce_covering_route.rs | 1 + .../tests/integration/announce_to_serve.rs | 1 + rs/moq-net/tests/integration/auth.rs | 2 +- .../tests/integration/claim_continues.rs | 1 + .../tests/integration/dial_split_horizon.rs | 2 +- rs/moq-net/tests/integration/extensions.rs | 2 +- rs/moq-net/tests/integration/instance.rs | 2 +- .../tests/integration/lazy_discovery.rs | 4 +- rs/moq-net/tests/integration/mount.rs | 18 +- rs/moq-net/tests/integration/multi_cdn.rs | 2 +- rs/moq-net/tests/integration/session_panic.rs | 2 +- rs/moq-net/tests/integration/support/mod.rs | 2 + rs/moq-net/tests/integration/synced.rs | 2 +- rs/moq-relay/src/cluster.rs | 2 +- rs/moq-relay/src/internal.rs | 2 +- rs/moq-relay/tests/drills.rs | 2 +- .../tests/integration/auth_lifetime.rs | 2 +- .../tests/integration/cluster_unknown.rs | 2 +- rs/moq-relay/tests/integration/connection.rs | 2 +- rs/moq-relay/tests/integration/embed.rs | 2 +- rs/moq-relay/tests/integration/memory.rs | 2 +- .../tests/integration/runtime_uring.rs | 2 +- .../tests/integration/runtime_workers.rs | 2 +- rs/moq-relay/tests/integration/smoke.rs | 2 +- .../tests/integration/tenant_matrix.rs | 2 +- rs/moq-room/src/room.rs | 4 +- rs/moq-stats/src/aggregate.rs | 4 +- rs/moq-stats/src/consume.rs | 2 +- rs/moq-stats/src/produce.rs | 2 +- rs/moq-tokio/examples/clock.rs | 6 + rs/moq-tokio/src/origin.rs | 9 +- rs/moq-tokio/src/server.rs | 9 +- rs/moq-tokio/tests/integration/backend.rs | 9 +- rs/moq-tokio/tests/integration/broadcast.rs | 11 +- rs/moq-webrtc/src/lib.rs | 9 +- 58 files changed, 518 insertions(+), 220 deletions(-) diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index f11c2f36698..c174a8970a4 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -63,6 +63,19 @@ async fn run_subscribe(consumer: moq_net::origin::Consumer) -> anyhow::Result<() old.abort(); } } + moq_net::announce::Event::Offline(announce) => { + tracing::info!(path = %announce.prefix, "broadcast offline"); + if let Some(old) = playing.take() { + old.abort(); + } + } + // The follow resumes on its own once the path is reachable again. + moq_net::announce::Event::Unavailable(announce) => { + tracing::info!(path = %announce.prefix, "broadcast unreachable"); + } + moq_net::announce::Event::Available(announce) => { + tracing::info!(path = %announce.prefix, "broadcast reachable again"); + } } } anyhow::bail!("origin closed") diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 1d1144c66ab..913eb8e4c5d 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -365,7 +365,7 @@ async fn subscribe( Some( moq_net::announce::Event::Present(update), ) => update, - Some(moq_net::announce::Event::Absent(_)) => continue, + Some(_) => continue, None => break, }; let path = update.prefix.to_string(); @@ -391,7 +391,7 @@ async fn subscribe( while selected < want { let update = match announced.next().await { Some(moq_net::announce::Event::Present(update)) => update, - Some(moq_net::announce::Event::Absent(_)) => continue, + Some(_) => continue, None => break, }; let path = update.prefix.to_string(); diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 458457e1070..f9e6a26cc3e 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -1026,6 +1026,8 @@ pub struct moq_route_info { pub cost: u64, /// The stream the route serves: routes with one instance are one continuous stream. pub instance: moq_string, + /// Whether a usable path reaches the route's origin; requests may fail while it is not. + pub available: bool, } /// Parse [moq_route_options], treating NULL as the default. @@ -1102,11 +1104,20 @@ pub enum moq_announce_kind { /// A route now covers the path, which was absent. Another instance taking it over keeps it /// present: follow it with [moq_origin_follow] to see the broadcast change. MOQ_ANNOUNCE_KIND_PRESENT = 0, - /// No route covers the path any more. + /// The origin withdrew the path: no route covers it any more. MOQ_ANNOUNCE_KIND_ABSENT = 1, -} - -/// An announce event from an origin: a path becoming present or absent. + /// This origin let go of the path, and no route covers it any more: the session it arrived + /// on closed, or the network lost every path to it. Its origin may still serve it. + MOQ_ANNOUNCE_KIND_OFFLINE = 2, + /// The present path lost every usable path to its origin but may recover one: it stays + /// present while requests for it may fail. + MOQ_ANNOUNCE_KIND_UNAVAILABLE = 3, + /// The present path recovered a usable path: request it again. + MOQ_ANNOUNCE_KIND_AVAILABLE = 4, +} + +/// An announce event from an origin: a path becoming present, absent, or offline, or its +/// availability changing. #[repr(C)] pub struct moq_announce_event { /// The covered prefix, relative to the origin, NOT NULL terminated @@ -1123,8 +1134,8 @@ pub struct moq_announce_event { /// Which event this is. pub kind: moq_announce_kind, - /// The route serving the prefix. On `MOQ_ANNOUNCE_KIND_ABSENT` it carries the last - /// advertised route. + /// The route serving the prefix. On `MOQ_ANNOUNCE_KIND_ABSENT` and + /// `MOQ_ANNOUNCE_KIND_OFFLINE` it carries the last advertised route. pub route: moq_route_info, } diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index 2e8db479776..76cc725f4c1 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -85,6 +85,7 @@ struct AnnouncedRecord { capture_views: Vec, kind: moq_announce_kind, cost: u64, + available: bool, /// The route's instance. instance: String, } @@ -99,6 +100,9 @@ impl AnnouncedRecord { let (update, kind) = match event { Event::Present(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_PRESENT), Event::Absent(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_ABSENT), + Event::Offline(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_OFFLINE), + Event::Unavailable(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_UNAVAILABLE), + Event::Available(update) => (update, moq_announce_kind::MOQ_ANNOUNCE_KIND_AVAILABLE), }; let captures: Vec<_> = update .captures @@ -120,6 +124,7 @@ impl AnnouncedRecord { capture_views, kind, cost: update.route.cost().value(), + available: update.route.available(), instance: update.route.instance().to_string(), } } @@ -242,6 +247,7 @@ impl Origin { data: announced.instance.as_ptr().cast::(), len: announced.instance.len(), }, + available: announced.available, }, }; Ok(()) diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index b90e01e91ae..4ad13c1c015 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -207,6 +207,7 @@ fn announce_info(announced: u32) -> moq_announce_event { data: std::ptr::null(), len: 0, }, + available: false, }, }; assert_eq!(unsafe { moq_origin_announced_info(announced, &mut info) }, 0); diff --git a/rs/moq-cli/src/announced.rs b/rs/moq-cli/src/announced.rs index 8ec9b04b934..217d519241b 100644 --- a/rs/moq-cli/src/announced.rs +++ b/rs/moq-cli/src/announced.rs @@ -92,7 +92,9 @@ async fn follow( let (announce, active) = match event { Event::Present(announce) => (announce, true), - Event::Absent(announce) => (announce, false), + Event::Absent(announce) | Event::Offline(announce) => (announce, false), + // The set lists presence; a held path stays listed. + Event::Available(_) | Event::Unavailable(_) => continue, }; let path = announce.prefix.as_str(); diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index bf59acf8043..613f7f805b1 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -91,12 +91,15 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp old.abort(); } } - moq_net::announce::Event::Absent(announce) => { + moq_net::announce::Event::Absent(announce) | moq_net::announce::Event::Offline(announce) => { if let Some(follower) = followers.remove(&announce.prefix) { follower.abort(); } } - moq_net::announce::Event::Present(_) => {} + // A follower rides out its path being held. + moq_net::announce::Event::Present(_) + | moq_net::announce::Event::Available(_) + | moq_net::announce::Event::Unavailable(_) => {} } continue; } diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 959cfe175ed..79d7d066a3f 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -489,7 +489,9 @@ async fn write_echo( let event = event.context("announcements ended")?; let (announce, active) = match event { moq_net::announce::Event::Present(announce) => (announce, true), - moq_net::announce::Event::Absent(announce) => (announce, false), + moq_net::announce::Event::Absent(announce) | moq_net::announce::Event::Offline(announce) => (announce, false), + // A reader rides out its path being held. + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, }; let path = announce.prefix; if !path.has_prefix(&prefix) || !hang::echo::is_echo(&path) { diff --git a/rs/moq-cli/tests/integration/instances.rs b/rs/moq-cli/tests/integration/instances.rs index 8e0ae972275..63d0de3ed6b 100644 --- a/rs/moq-cli/tests/integration/instances.rs +++ b/rs/moq-cli/tests/integration/instances.rs @@ -107,7 +107,7 @@ async fn changed(follow: &mut origin::Routes) -> bool { loop { match follow.next().await.unwrap() { origin::RouteEvent::Restart(_) => return true, - origin::RouteEvent::End(_) => { + origin::RouteEvent::End(_) | origin::RouteEvent::Offline(_) => { assert!(matches!(follow.next().await, Some(origin::RouteEvent::Start(_)))); return false; } diff --git a/rs/moq-cli/tests/integration/rtsp.rs b/rs/moq-cli/tests/integration/rtsp.rs index 71a3e6ad663..1d1f4f6e3d2 100644 --- a/rs/moq-cli/tests/integration/rtsp.rs +++ b/rs/moq-cli/tests/integration/rtsp.rs @@ -65,7 +65,9 @@ async fn next_broadcast( .expect("origin closed") { moq_net::origin::RouteEvent::Start(_) | moq_net::origin::RouteEvent::Restart(_) => break, - moq_net::origin::RouteEvent::End(_) | moq_net::origin::RouteEvent::Update(_) => {} + moq_net::origin::RouteEvent::End(_) + | moq_net::origin::RouteEvent::Offline(_) + | moq_net::origin::RouteEvent::Update(_) => {} } } let broadcast = running(child, "the broadcast", consumer.request("camera", None)) diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 20566ac3ce6..c51e3adf4d4 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -77,7 +77,8 @@ impl TryFrom for moq_net::origin::RouteOptions { } } -/// The read-only facts on a received route: a static cost and the stream's instance. +/// The read-only facts on a received route: a static cost, the stream's instance, and +/// whether a usable path reaches it. /// /// Carried by `MoqAnnounceConsumer` events. A route claims capability, not /// inventory: by convention a publisher creates each broadcast's exact route, @@ -89,6 +90,8 @@ pub struct MoqRouteInfo { pub cost: u64, /// The stream the route serves: routes with one instance are one continuous stream. pub instance: String, + /// Whether a usable path reaches the route's origin; requests may fail while it is not. + pub available: bool, } impl From for MoqRouteInfo { @@ -96,6 +99,7 @@ impl From for MoqRouteInfo { Self { cost: route.cost().value(), instance: route.instance().to_string(), + available: route.available(), } } } @@ -199,7 +203,8 @@ impl From for MoqAnnounce { } } -/// What a [`MoqAnnounceConsumer`] yields: a path becoming present or absent. +/// What a [`MoqAnnounceConsumer`] yields: a path becoming present, absent, or offline, or +/// its availability changing. /// /// Another publisher instance taking a path over keeps it present: follow it with /// `MoqOriginConsumer::follow` to see the broadcast change. @@ -210,11 +215,30 @@ pub enum MoqAnnounceEvent { /// The announcement that made the path present. announce: MoqAnnounce, }, - /// No route covers the path any more. Carries its last advertised route. + /// The present path lost every usable path to its origin but may recover one: it stays + /// present while requests for it may fail. + Unavailable { + /// The announcement, now unavailable. + announce: MoqAnnounce, + }, + /// The present path recovered a usable path: request it again. + Available { + /// The announcement, available again. + announce: MoqAnnounce, + }, + /// The origin withdrew the path: no route covers it any more. Carries its last + /// advertised route. Absent { /// The announcement that went. announce: MoqAnnounce, }, + /// This origin let go of the path, and no route covers it any more: the session it + /// arrived on closed, or the network lost every path to it. Its origin may still serve + /// it. Carries its last advertised route. + Offline { + /// The announcement that went. + announce: MoqAnnounce, + }, } impl From for MoqAnnounceEvent { @@ -224,9 +248,18 @@ impl From for MoqAnnounceEvent { Event::Present(announce) => Self::Present { announce: announce.into(), }, + Event::Unavailable(announce) => Self::Unavailable { + announce: announce.into(), + }, + Event::Available(announce) => Self::Available { + announce: announce.into(), + }, Event::Absent(announce) => Self::Absent { announce: announce.into(), }, + Event::Offline(announce) => Self::Offline { + announce: announce.into(), + }, } } } diff --git a/rs/moq-net/benches/announce.rs b/rs/moq-net/benches/announce.rs index 183fc21f188..14a83e2a78d 100644 --- a/rs/moq-net/benches/announce.rs +++ b/rs/moq-net/benches/announce.rs @@ -37,7 +37,7 @@ fn scanned(events: Vec) -> Vec { origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update) | origin::RouteEvent::Restart(update) => (update, true), - origin::RouteEvent::End(update) => (update, false), + origin::RouteEvent::End(update) | origin::RouteEvent::Offline(update) => (update, false), }; initial.retain(|held| held.prefix != update.prefix); if active { diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index b144175fefe..88b4b6353c1 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -2753,7 +2753,7 @@ impl Enforce { } self.live.insert(gate.path); } - crate::origin::RouteEvent::End(update) => { + crate::origin::RouteEvent::End(update) | crate::origin::RouteEvent::Offline(update) => { self.live.remove(&update.prefix); self.pending.retain(|gate| gate.path != update.prefix); } diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 343b588eeba..5be6e004123 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -4136,7 +4136,7 @@ impl Publisher { | crate::origin::RouteEvent::Restart(update) => { initial.insert(update.prefix.clone(), update); } - crate::origin::RouteEvent::End(update) => { + crate::origin::RouteEvent::End(update) | crate::origin::RouteEvent::Offline(update) => { initial.remove(&update.prefix); } } @@ -5902,7 +5902,7 @@ impl ForwardNamespaces { (update, true, false) } crate::origin::RouteEvent::Restart(update) => (update, true, true), - crate::origin::RouteEvent::End(update) => (update, false, false), + crate::origin::RouteEvent::End(update) | crate::origin::RouteEvent::Offline(update) => (update, false, false), }))); } if self.retry.poll(waiter).is_ready() { diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index d3b76f9e9ca..b4a9bc06ea8 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1694,7 +1694,7 @@ impl Subscriber { // The advertisement itself stays live on the wire. tracing::debug!(%path, "dropping reflected namespace"); if let Some(id) = held.id.take() { - let _ = self.stop_announce(path, id, false); + let _ = self.stop_announce(path, id, false, false); } continue; }; @@ -1725,7 +1725,7 @@ impl Subscriber { let path = prefix.join(&msg.suffix)?; tracing::debug!(%path, "namespace_done"); if let Some(id) = live.remove(&path).and_then(|held| held.id) { - let _ = self.stop_announce(path, id, false); + let _ = self.stop_announce(path, id, false, true); } } _ => { @@ -1833,7 +1833,7 @@ impl NamespaceSubscription { // detaches. for (path, advert) in std::mem::take(&mut request.live) { if let Some(id) = advert.id { - let _ = self.this.stop_announce(path, id, false); + let _ = self.this.stop_announce(path, id, false, false); } } @@ -2072,10 +2072,17 @@ impl Subscriber { } /// Retract an advertisement's route ahead of a withdrawal that waits on the peer, so a peer - /// that stops reading cannot keep it routed. `attached` makes it once only. - fn release_announce(&mut self, path: &Path, attached: &mut Option, token: bool) -> Result<(), Error> { + /// that stops reading cannot keep it routed. `attached` makes it once only. `ended` when + /// the peer withdrew it, rather than the stream failing. + fn release_announce( + &mut self, + path: &Path, + attached: &mut Option, + token: bool, + ended: bool, + ) -> Result<(), Error> { if let Some(id) = attached.take() { - self.stop_announce(path.clone(), id, token)?; + self.stop_announce(path.clone(), id, token, ended)?; } Ok(()) } @@ -2286,7 +2293,9 @@ impl ServePublishNamespace { // cannot keep it routed. A stream that detached mid-flight (a reflected update) has // nothing left to release. let token = live.grant.is_some(); - self.this.release_announce(&live.path, &mut live.attached, token)?; + // Only the peer's own withdrawal ends the route; a failed stream loses it. + let ended = matches!(lived, Ok(Ok(Withdraw::Done))); + self.this.release_announce(&live.path, &mut live.attached, token, ended)?; let withdraw = match lived { Ok(withdraw) => withdraw?, @@ -2769,7 +2778,7 @@ impl PublishNamespaceUpdates { None => { if let Some(id) = attached.take() { tracing::debug!(%path, "publish_namespace now loops back; detaching"); - let _ = this.stop_announce(path.clone(), id, token_grant.is_some()); + let _ = this.stop_announce(path.clone(), id, token_grant.is_some(), false); } Ok(()) } @@ -2989,7 +2998,7 @@ impl Subscriber { /// Release advertisement `id` of `path`, closing its sources when it was the last. /// `token` says its own request token authorized it. - fn stop_announce(&mut self, path: Path, id: u64, token: bool) -> Result<(), Error> { + fn stop_announce(&mut self, path: Path, id: u64, token: bool, ended: bool) -> Result<(), Error> { let mut state = self.state.lock(); match state.broadcasts.entry(path.clone()) { @@ -3003,8 +3012,12 @@ impl Subscriber { if state.adverts.is_empty() { tracing::debug!(route = %crate::path::Rooted(self.origin.root(), &path), "unannounced"); // Dropping the entry retracts the route (its announcement drops) and - // closes its sources (its token closes). - entry.remove(); + // closes its sources (its token closes). Readers see it end only when + // the peer withdrew it; otherwise it goes offline. + let state = entry.remove(); + if ended && let Some(dynamic) = state.dynamic { + dynamic.ended(); + } return Ok(()); } // The newest went while an older one stands: that one is what remains. @@ -10650,8 +10663,8 @@ mod tests { )] ); - subscriber.stop_announce(a.clone(), first, false).unwrap(); - subscriber.stop_announce(a, repeat, false).unwrap(); + subscriber.stop_announce(a.clone(), first, false, true).unwrap(); + subscriber.stop_announce(a, repeat, false, true).unwrap(); subscriber.start_announce(b, last, Admit::Session(None)).unwrap(); } @@ -10836,7 +10849,7 @@ mod tests { fn routed_now(consumer: &crate::origin::Consumer, path: &str) -> Option { use futures::FutureExt; match consumer.path_routes(path).ok()?.next().now_or_never()?? { - crate::origin::RouteEvent::End(_) => None, + crate::origin::RouteEvent::End(_) | crate::origin::RouteEvent::Offline(_) => None, crate::origin::RouteEvent::Start(announce) | crate::origin::RouteEvent::Update(announce) | crate::origin::RouteEvent::Restart(announce) => Some(announce.route), @@ -11097,7 +11110,7 @@ mod tests { .unwrap(); settle().await; - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!( routed_now(&consumer, "room/host").is_none(), "an explicit NAMESPACE_DONE must retract the route", @@ -11246,7 +11259,7 @@ mod tests { settle().await; // One advertisement's stream dies: the other still holds the source. - subscriber.stop_announce(path.clone(), ids[0], false).unwrap(); + subscriber.stop_announce(path.clone(), ids[0], false, true).unwrap(); settle().await; assert!( routed_now(&consumer, "room/host").is_some(), @@ -11254,7 +11267,7 @@ mod tests { ); // The last owner retracts: the broadcast closes with it. - subscriber.stop_announce(path, ids[1], false).unwrap(); + subscriber.stop_announce(path, ids[1], false, true).unwrap(); settle().await; assert!( routed_now(&consumer, "room/host").is_none(), @@ -11357,7 +11370,7 @@ mod tests { // One advertisement, so one unannounce detaches it. If the update had bumped the // refcount, this would leave the route stranded. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11409,7 +11422,7 @@ mod tests { ); // One advertisement, so one unannounce detaches it. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11442,9 +11455,9 @@ mod tests { assert!(routed_now(&consumer, "room/host").is_some()); // Two advertisements, so it takes two unannounces to retract. - subscriber.stop_announce(path.clone(), first, false).unwrap(); + subscriber.stop_announce(path.clone(), first, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_some()); - subscriber.stop_announce(path, second, false).unwrap(); + subscriber.stop_announce(path, second, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11475,9 +11488,9 @@ mod tests { assert!(routed_now(&consumer, "room/host").is_some()); // Two advertisements, so it takes two unannounces to retract. - subscriber.stop_announce(path.clone(), first, false).unwrap(); + subscriber.stop_announce(path.clone(), first, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_some()); - subscriber.stop_announce(path, second, false).unwrap(); + subscriber.stop_announce(path, second, false, true).unwrap(); assert!(routed_now(&consumer, "room/host").is_none()); } @@ -11516,7 +11529,7 @@ mod tests { ); // That supersedes the advertisement it repeats, so the old route is retired. - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); assert!( routed_now(&consumer, "room/host").is_none(), "the superseded route must not stay attached" @@ -12126,7 +12139,7 @@ mod tests { // What the stream's exit path does with whatever it still holds. for (path, id) in live { - subscriber.stop_announce(path, id, false).unwrap(); + subscriber.stop_announce(path, id, false, true).unwrap(); } assert!(routed_now(&consumer, "room/a").is_none(), "room/a leaked a refcount"); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 57e795ef8f2..b8a7f93b378 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1235,7 +1235,7 @@ impl AnnounceRun { | origin::RouteEvent::Restart(update) => { current.insert(update.prefix, update.route); } - origin::RouteEvent::End(update) => { + origin::RouteEvent::End(update) | origin::RouteEvent::Offline(update) => { current.remove(&update.prefix); } } @@ -1599,7 +1599,7 @@ impl AnnounceRun { let (update, active, restart) = match next { Some(origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update)) => (update, true, false), Some(origin::RouteEvent::Restart(update)) => (update, true, true), - Some(origin::RouteEvent::End(update)) => (update, false, false), + Some(origin::RouteEvent::End(update) | origin::RouteEvent::Offline(update)) => (update, false, false), None => { if self.initial.take().is_some() { stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; diff --git a/rs/moq-net/src/lite/replay.rs b/rs/moq-net/src/lite/replay.rs index 29f826037f4..cb474d7fc10 100644 --- a/rs/moq-net/src/lite/replay.rs +++ b/rs/moq-net/src/lite/replay.rs @@ -13,7 +13,7 @@ pub(super) fn initial(events: impl IntoIterator) -> V origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update) | origin::RouteEvent::Restart(update) => (update, true), - origin::RouteEvent::End(update) => (update, false), + origin::RouteEvent::End(update) | origin::RouteEvent::Offline(update) => (update, false), }; if let Some(index) = positions.remove(&update.prefix) { pending[index] = None; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 1dd1be440a9..45654a5656a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -8719,7 +8719,7 @@ mod tests { ) .unwrap(); assert!(!attached, "a restart outside the limit was attached"); - cursor.expect_next_ended("room/host").await; + cursor.expect_next_offline("room/host").await; } /// An announce stream that dies without an explicit `ended` retracts the route @@ -8766,7 +8766,7 @@ mod tests { // The stream ends without retracting anything: the map dies with it and the // route retracts. drop(announced); - cursor.expect_next_ended("room/host").await; + cursor.expect_next_offline("room/host").await; // An explicit retraction retracts it the same way. let hops = crate::Hops::try_from(vec![crate::Hop::new(7).unwrap()]).unwrap(); diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 305432e158b..20fe66dd385 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -356,9 +356,8 @@ impl Routed { drop(std::mem::replace(&mut self.stream, stream)); self.previous = previous; self.outputs(subscriber); - for (_, served) in std::mem::take(&mut self.previous) { - served.route.dynamic.withdrawn(); - } + // What the new stream no longer serves is let go here, not ended by its origin. + self.previous.clear(); Ok(()) } @@ -398,8 +397,11 @@ impl Routed { claim, served, } => self.serve(subscriber, id, named, claim, served), - node::Output::Unserve { id, claim } => { - if let Some(served) = self.served.remove(&(id, claim)) { + node::Output::Unserve { id, claim, ended } => { + // Dropped, a claim the node let go of goes offline rather than ending. + if let Some(served) = self.served.remove(&(id, claim)) + && ended + { served.route.dynamic.withdrawn(); } } @@ -422,15 +424,16 @@ impl Routed { if !self.permit.within_limit(path.as_str()) { return; } - // A route held without a path to it serves only as a last resort, like a draining - // peer's, at its real cost. + // A route held without a path to it is unavailable: it serves only as a last + // resort, like a draining peer's, at its real cost. let route = crate::origin::RouteInfo::new() .with_instance(served.claim.instance.clone(), served.claim.epoch) .with_hops(hops(id)) .with_cost(served.claim.cost.saturating_add(served.metric)) .with_via(subscriber.via(None)) .with_node(Some(id)) - .with_draining(self.drained || !served.available); + .with_draining(self.drained) + .with_available(served.available); let key = (id, claim); if let Some(entry) = self.previous.remove(&key) { self.served.insert(key.clone(), entry); diff --git a/rs/moq-net/src/model/announced.rs b/rs/moq-net/src/model/announced.rs index 4b2d6b8877e..e401b11d793 100644 --- a/rs/moq-net/src/model/announced.rs +++ b/rs/moq-net/src/model/announced.rs @@ -1,5 +1,6 @@ //! The paths announced under a scope, each present or absent. +use std::collections::HashSet; use std::task::Poll; use super::origin_impl::{Announce, RouteEvent, Routes}; @@ -8,33 +9,74 @@ use crate::{Error, Path}; /// What an announcement listing reports for a path. #[derive(Clone, Debug)] pub enum AnnounceEvent { - /// A route now covers the path, which was absent. + /// A route now covers the path, which was absent. It may arrive unavailable; see + /// [`RouteInfo::available`](crate::origin::RouteInfo::available). Present(Announce), - /// No route covers the path any more. Carries its last advertised route. + /// The present path's route lost every usable path to its origin but may recover + /// one, so the path stays present while requests for it may fail. + Unavailable(Announce), + /// The present path's route recovered a usable path: request it again. + Available(Announce), + /// The origin withdrew the path: no route covers it any more. Carries its last + /// advertised route. Absent(Announce), + /// This origin let go of the path, and no route covers it any more: the session it + /// arrived on closed, or the network lost every path to it. Its origin may still + /// serve it. Carries its last advertised route. + Offline(Announce), } /// The paths announced in a scope, each reported as it becomes present or absent. /// /// A path that another instance takes over stays present, and a route's metadata -/// changing is not reported: hold a [`follow`](crate::origin::Consumer::follow) to -/// see the broadcast change. Created by +/// changing is not reported, except its availability: hold a +/// [`follow`](crate::origin::Consumer::follow) to see the broadcast change. Created by /// [`Consumer::announced`](crate::origin::Consumer::announced). Drop to unregister. pub struct AnnounceConsumer { routes: Routes, + /// The present paths last reported unavailable, bounded by the present set. + unavailable: HashSet, } impl AnnounceConsumer { pub(super) fn new(routes: Routes) -> Self { - Self { routes } + Self { + routes, + unavailable: HashSet::new(), + } } /// What a route change means for the listing, if anything. - fn list(event: RouteEvent) -> Option { + fn list(&mut self, event: RouteEvent) -> Option { match event { - RouteEvent::Start(announce) => Some(AnnounceEvent::Present(announce)), - RouteEvent::End(announce) => Some(AnnounceEvent::Absent(announce)), - RouteEvent::Update(_) | RouteEvent::Restart(_) => None, + RouteEvent::Start(announce) => { + if !announce.route.available() { + self.unavailable.insert(announce.prefix.clone()); + } + Some(AnnounceEvent::Present(announce)) + } + RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { + let was = !self.unavailable.contains(&announce.prefix); + match (was, announce.route.available()) { + (true, false) => { + self.unavailable.insert(announce.prefix.clone()); + Some(AnnounceEvent::Unavailable(announce)) + } + (false, true) => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Available(announce)) + } + _ => None, + } + } + RouteEvent::End(announce) => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Absent(announce)) + } + RouteEvent::Offline(announce) => { + self.unavailable.remove(&announce.prefix); + Some(AnnounceEvent::Offline(announce)) + } } } @@ -55,7 +97,7 @@ impl AnnounceConsumer { let Some(event) = std::task::ready!(self.routes.poll_next(waiter)) else { return Poll::Ready(None); }; - if let Some(event) = Self::list(event) { + if let Some(event) = self.list(event) { return Poll::Ready(Some(event)); } } @@ -67,7 +109,8 @@ impl AnnounceConsumer { /// Use [`Self::is_closed`] to check if the listing is closed. pub fn try_next(&mut self) -> Option { loop { - if let Some(event) = Self::list(self.routes.try_next()?) { + let event = self.routes.try_next()?; + if let Some(event) = self.list(event) { return Some(event); } } @@ -113,7 +156,7 @@ impl futures::Stream for AnnounceConsumer { else { return Poll::Ready(None); }; - if let Some(event) = Self::list(event) { + if let Some(event) = this.list(event) { return Poll::Ready(Some(event)); } } @@ -140,6 +183,9 @@ mod tests { match event { AnnounceEvent::Present(announce) => format!("present {}", announce.prefix), AnnounceEvent::Absent(announce) => format!("absent {}", announce.prefix), + AnnounceEvent::Offline(announce) => format!("offline {}", announce.prefix), + AnnounceEvent::Available(announce) => format!("available {}", announce.prefix), + AnnounceEvent::Unavailable(announce) => format!("unavailable {}", announce.prefix), } } diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index b061da491e1..6f4bfd0ac38 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -93,7 +93,7 @@ impl Follow { return Poll::Ready(None); }; if let Some(event) = self.fold(event) { - self.idle = matches!(event, RouteEvent::End(_)); + self.idle = matches!(event, RouteEvent::End(_) | RouteEvent::Offline(_)); return Poll::Ready(Some(event)); } } @@ -126,14 +126,15 @@ impl Follow { /// Apply one route's event, returning what it means for the path, if anything. fn fold(&mut self, event: RouteEvent) -> Option { - let (RouteEvent::Start(announce) - | RouteEvent::Update(announce) - | RouteEvent::Restart(announce) - | RouteEvent::End(announce)) = &event; - if !self.path.has_prefix(&announce.prefix) { + let prefix = event.announce().prefix.clone(); + if !self.path.has_prefix(&prefix) { return None; } - let prefix = announce.prefix.clone(); + // A route that went reports its own way of going: ended by its origin, or let go. + let gone = match event { + RouteEvent::Offline(_) => RouteEvent::Offline, + _ => RouteEvent::End, + }; let before = self .serving() .map(|standing| (standing.announce.clone(), standing.seen)); @@ -144,7 +145,7 @@ impl Follow { .position(|standing| standing.announce.prefix == prefix) .map(|index| self.covering.swap_remove(index).since); let (announce, since, restart) = match event { - RouteEvent::End(_) => (None, None, false), + RouteEvent::End(_) | RouteEvent::Offline(_) => (None, None, false), RouteEvent::Start(announce) => (Some(announce), None, false), RouteEvent::Update(announce) => (Some(announce), standing, false), RouteEvent::Restart(announce) => (Some(announce), standing, true), @@ -163,12 +164,12 @@ impl Follow { match (before, after) { (None, None) => None, (None, Some((after, _))) => Some(RouteEvent::Start(after)), - (Some((before, _)), None) => Some(RouteEvent::End(before)), + (Some((before, _)), None) => Some(gone(before)), // The serving route ended, and the one left may have started after it did. (Some((before, seen)), Some((after, since))) if before.prefix == prefix && before.prefix != after.prefix && since > self.dry.max(seen) => { - Some(RouteEvent::End(before)) + Some(gone(before)) } (Some((before, _)), Some((after, _))) if before.prefix != after.prefix => { let same = before.route.instance == after.route.instance; @@ -377,7 +378,7 @@ impl Following { RouteEvent::Start(_) | RouteEvent::Restart(_) => self.unseen = true, // An end is a gap that ended every request on the old route, so whatever serves the // path next is handed out, even the same instance coming back. - RouteEvent::End(_) => { + RouteEvent::End(_) | RouteEvent::Offline(_) => { self.unseen = false; self.end(); } @@ -412,6 +413,7 @@ mod tests { RouteEvent::Update(announce) => ("update", announce.prefix.to_string()), RouteEvent::Restart(announce) => ("restart", announce.prefix.to_string()), RouteEvent::End(announce) => ("end", announce.prefix.to_string()), + RouteEvent::Offline(announce) => ("offline", announce.prefix.to_string()), } } @@ -464,7 +466,7 @@ mod tests { assert_eq!(followed(&mut follow).await, ("update", "pool".into())); drop(pool); - assert_eq!(followed(&mut follow).await, ("end", "pool".into())); + assert_eq!(followed(&mut follow).await, ("offline", "pool".into())); } /// A follower that joins late starts on the route serving the path, not on whichever diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9d6f064c724..ebfe989415a 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -501,14 +501,17 @@ pub struct RouteInfo { /// peer that names no node. pub(crate) node: Option, - /// The session it arrived on is going away (the peer sent a GOAWAY), or the route - /// layer holds the route without a path to it: it ranks below every other eligible - /// route, whatever its preference or cost, yet still serves when nothing else does. + /// The session it arrived on is going away (the peer sent a GOAWAY): it ranks below + /// every other eligible route, whatever its preference or cost, yet still serves when + /// nothing else does. /// /// A preference state, never a saturated metric. Local, like [`Self::via`]: never /// forwarded, so a relay advertises a draining route at its real cost and its /// readers never see the flag. pub(crate) draining: bool, + + /// Whether a usable path reaches the route's origin; see [`Self::available`]. + pub(crate) available: bool, } impl RouteInfo { @@ -525,6 +528,7 @@ impl RouteInfo { source: Source::Local, node: None, draining: false, + available: true, } } } @@ -562,6 +566,14 @@ impl RouteInfo { self.cost } + /// Whether a usable path reaches the route's origin. An unavailable route is held: + /// the network lost every path to it but may recover one, so the announcement stays. + /// It ranks with a draining route, below every available one, and requests through + /// it may fail until it recovers: request again once it turns available. + pub fn available(&self) -> bool { + self.available + } + /// Serve a stable `instance` at `epoch`; see [`RouteOptions::with_instance`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn with_instance(mut self, instance: Instance, epoch: u64) -> Self { @@ -616,6 +628,13 @@ impl RouteInfo { self } + /// Mark whether a usable path reaches the route's origin; see [`Self::available`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_available(mut self, available: bool) -> Self { + self.available = available; + self + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -773,9 +792,9 @@ impl<'a> Epochs<'a> { } /// Ordering key for an eligible route entry resolving `path` (see [`Epochs`]). -/// Lower wins: first a route that is not draining (see [`RouteInfo::draining`]), so a -/// peer that sent a GOAWAY ranks below every other eligible route and serves only as a -/// last resort. Then the higher link [`Preference`], so metrics from different operators +/// Lower wins: first a route that is neither draining (see [`RouteInfo::draining`]) nor +/// unavailable (see [`RouteInfo::available`]), so a peer that sent a GOAWAY or a held +/// route ranks below every other eligible route and serves only as a last resort. Then the higher link [`Preference`], so metrics from different operators /// are never compared. Then an identified chain (no 0) outranks an anonymous one regardless /// of cost, then the cheapest cost, then a broadcast published on this origin (it serves /// what is here, not a claim that has to ask), then the shortest hop chain, then a hash @@ -795,7 +814,7 @@ impl<'a> Epochs<'a> { /// until the transport times out. fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { ( - entry.draining, + entry.draining || !entry.available, Reverse(entry.preference), entry.is_anonymous(), entry.cost, @@ -826,8 +845,8 @@ type RouteOrder = ( Reverse, ); -/// The `(hops, cost, source, instance, epoch, auth, node)` metadata an announce cursor delivers -/// alongside a prefix. +/// The `(hops, cost, source, instance, epoch, auth, node, available)` metadata an announce +/// cursor delivers alongside a prefix. type RouteMeta = ( Hops, Cost, @@ -836,6 +855,7 @@ type RouteMeta = ( u64, Option, Option, + bool, ); /// The earlier of two optional deadlines. @@ -893,8 +913,22 @@ type AnnounceMeta = (RouteMeta, Option>); enum PendingUpdate { Announce(AnnounceMeta), Restart(AnnounceMeta), - Unannounce(AnnounceMeta), - UnannounceAnnounce { old: AnnounceMeta, new: AnnounceMeta }, + Unannounce(AnnounceMeta, Gone), + UnannounceAnnounce { + old: AnnounceMeta, + gone: Gone, + new: AnnounceMeta, + }, +} + +/// How a route left a cursor: the origin ended it, or this node lost it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Gone { + /// Its origin withdrew it, as far as this node knows: [`RouteEvent::End`]. + End, + /// This node let go of it, such as the session it arrived on closing: + /// [`RouteEvent::Offline`]. + Offline, } // An update is taken only while collapsing it under the cursor state lock. @@ -964,7 +998,7 @@ impl OriginConsumerState { pending.owed = !self.delivered.contains(prefix) || matches!( pending.update, - Some(PendingUpdate::Unannounce(_) | PendingUpdate::UnannounceAnnounce { .. }) + Some(PendingUpdate::Unannounce(..) | PendingUpdate::UnannounceAnnounce { .. }) ); self.owed += usize::from(pending.owed); } @@ -1028,8 +1062,8 @@ impl OriginConsumerState { // The consumer still owes the restart; this only moves its metadata. Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), // Consumer needs to observe the retraction before this announce. - Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { - PendingUpdate::UnannounceAnnounce { old, new: meta } + Some(PendingUpdate::Unannounce(old, gone) | PendingUpdate::UnannounceAnnounce { old, gone, .. }) => { + PendingUpdate::UnannounceAnnounce { old, gone, new: meta } } }; self.put(prefix, new); @@ -1040,8 +1074,8 @@ impl OriginConsumerState { fn apply_restart(&mut self, prefix: Path, meta: RouteMeta, captures: Option>) { let meta = (meta, captures); let new = match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { - Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { - PendingUpdate::UnannounceAnnounce { old, new: meta } + Some(PendingUpdate::Unannounce(old, gone) | PendingUpdate::UnannounceAnnounce { old, gone, .. }) => { + PendingUpdate::UnannounceAnnounce { old, gone, new: meta } } Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), _ if self.delivered.contains(&prefix) => PendingUpdate::Restart(meta), @@ -1051,7 +1085,7 @@ impl OriginConsumerState { self.put(prefix, new); } - fn apply_unannounce(&mut self, prefix: Path, last: RouteMeta, captures: Option>) { + fn apply_unannounce(&mut self, prefix: Path, last: RouteMeta, captures: Option>, gone: Gone) { let last = (last, captures); match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { // The pending announce was never delivered and neither was any earlier @@ -1062,13 +1096,13 @@ impl OriginConsumerState { } // Either nothing is pending or the pending announce was a metadata // update on a delivered route; the consumer still owes a retraction. - None | Some(PendingUpdate::Announce(_) | PendingUpdate::Restart(..) | PendingUpdate::Unannounce(_)) => { - self.put(prefix, PendingUpdate::Unannounce(last)); + None | Some(PendingUpdate::Announce(_) | PendingUpdate::Restart(..) | PendingUpdate::Unannounce(..)) => { + self.put(prefix, PendingUpdate::Unannounce(last, gone)); } // The embedded announce cancels with this retraction; the consumer still - // needs the leading one. + // needs the leading one, reported as the later one went. Some(PendingUpdate::UnannounceAnnounce { old, .. }) => { - self.put(prefix, PendingUpdate::Unannounce(old)); + self.put(prefix, PendingUpdate::Unannounce(old, gone)); } } } @@ -1160,10 +1194,7 @@ impl OriginConsumerState { /// Take the pending update at `prefix` for delivery. fn take_prefix(&mut self, prefix: Path) -> RouteEvent { let (event, owed) = self.take_update(prefix); - let (RouteEvent::Start(announce) - | RouteEvent::Update(announce) - | RouteEvent::Restart(announce) - | RouteEvent::End(announce)) = &event; + let announce = event.announce(); // A trailing announce left pending is still owed. if owed && !self.pending.contains_key(&announce.prefix) { self.owed -= 1; @@ -1177,7 +1208,7 @@ impl OriginConsumerState { Start, Update, Restart, - End, + Gone(Gone), } let pending = self.remove(&prefix).unwrap(); @@ -1195,16 +1226,16 @@ impl OriginConsumerState { self.delivered.insert(prefix.clone()); (meta, Kind::Restart) } - PendingUpdate::Unannounce(meta) => { + PendingUpdate::Unannounce(meta, gone) => { self.delivered.remove(&prefix); - (meta, Kind::End) + (meta, Kind::Gone(gone)) } - PendingUpdate::UnannounceAnnounce { old, new } => { + PendingUpdate::UnannounceAnnounce { old, gone, new } => { // Deliver the retraction now; leave the trailing announce pending so // the next take returns it for the same prefix. self.delivered.remove(&prefix); self.insert(prefix.clone(), PendingUpdate::Announce(new), owed); - (old, Kind::End) + (old, Kind::Gone(gone)) } }; let announce = Announce { @@ -1220,13 +1251,15 @@ impl OriginConsumerState { auth: meta.5, node: meta.6, draining: false, + available: meta.7, }, }; let event = match kind { Kind::Start => RouteEvent::Start(announce), Kind::Update => RouteEvent::Update(announce), Kind::Restart => RouteEvent::Restart(announce), - Kind::End => RouteEvent::End(announce), + Kind::Gone(Gone::End) => RouteEvent::End(announce), + Kind::Gone(Gone::Offline) => RouteEvent::Offline(announce), }; (event, owed) } @@ -1266,6 +1299,8 @@ struct RouteEntry { /// Whether the route ranks last; see [`RouteInfo::draining`]. Local selection state, /// so never part of [`Self::meta`]. draining: bool, + /// Whether a usable path reaches the route's origin; see [`RouteInfo::available`]. + available: bool, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a [`Claim`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -1333,12 +1368,13 @@ impl RouteEntry { self.epoch, self.auth.clone(), self.node, + self.available, ) } /// Whether [`Self::meta`] would equal `meta`, without building it. fn delivers(&self, meta: &RouteMeta) -> bool { - let (hops, cost, source, instance, epoch, auth, node) = meta; + let (hops, cost, source, instance, epoch, auth, node, available) = meta; *hops == self.hops && *cost == self.cost && *source == self.entered() @@ -1346,6 +1382,7 @@ impl RouteEntry { && *epoch == self.epoch && *auth == self.auth && *node == self.node + && *available == self.available } /// Whether a request for `path` can be served through this entry. A served @@ -1675,13 +1712,14 @@ impl TableCursor { && self.named.as_ref().is_none_or(|(mount, _)| mount.names(&entry.prefix)) } - /// Deliver a changed winner at this presented prefix. - fn update(&mut self, presented: &Path, best: Option<&RouteEntry>) { + /// Deliver a changed winner at this presented prefix, reporting a route left with + /// none as `gone`. + fn update(&mut self, presented: &Path, best: Option<&RouteEntry>, gone: Gone) { let Some(entry) = best else { if let Some(last) = self.current.remove(presented) && let Ok(mut state) = self.state.write() { - state.apply_unannounce(self.under.rejoin(presented), last.meta, last.captures); + state.apply_unannounce(self.under.rejoin(presented), last.meta, last.captures, gone); } return; }; @@ -1715,7 +1753,12 @@ impl TableCursor { // identity explicitly so consumers keyed on it remove it. Some(previous) if previous.captures != captures => { if let Ok(mut state) = self.state.write() { - state.apply_unannounce(self.under.rejoin(presented), previous.meta, previous.captures); + state.apply_unannounce( + self.under.rejoin(presented), + previous.meta, + previous.captures, + Gone::End, + ); state.apply_announce(self.under.rejoin(presented), meta, captures); } } @@ -2049,11 +2092,18 @@ impl Drop for HiddenWithholding { pub enum RouteEvent { /// A route now covers the prefix; the cursor had none there. Start(Announce), - /// The route covering the prefix changed hops, cost or epoch, with the same instance; - /// it is delivered in place. + /// The route covering the prefix changed hops, cost, epoch or + /// [availability](RouteInfo::available), with the same instance; it is delivered in + /// place. Update(Announce), - /// No route covers the prefix any more. Carries its last advertised route. + /// The origin withdrew the route, as far as this origin knows, and no other covers + /// the prefix. Carries its last advertised route. End(Announce), + /// This origin let go of the route, and no other covers the prefix: the session it + /// arrived on closed, or the route layer lost every path to it. Its origin may still + /// serve it; only an [`End`](Self::End) says it stopped. Carries its last advertised + /// route. + Offline(Announce), /// Another [`Instance`] now serves the prefix: another stream, such as a publisher /// that restarted, a redundant ingest switching, or a pool failing over. /// @@ -2063,6 +2113,19 @@ pub enum RouteEvent { Restart(Announce), } +impl RouteEvent { + /// The announcement the event carries. + pub(crate) fn announce(&self) -> &Announce { + match self { + Self::Start(announce) + | Self::Update(announce) + | Self::End(announce) + | Self::Offline(announce) + | Self::Restart(announce) => announce, + } + } +} + /// A route over a prefix, as an announcement listing or a [`Routes`] stream reports it. /// /// An announcement is always a prefix, never a broadcast: it advertises that @@ -2450,6 +2513,7 @@ impl Producer { state: Arc::new(RouteState { announcement: entry, control: kio::Lock::new(control), + forwarded: false, _keepalive: self.tasks.keepalive(), }), broadcast: source.clone(), @@ -2668,10 +2732,13 @@ impl Producer { None, )?; let control = RouteControl::new(options, &announcement, None); + // A session's route outlives nothing it was given: dropping it means the session lost it. + let forwarded = publisher.is_none(); Ok(Claim { state: Arc::new(RouteState { announcement, control: kio::Lock::new(control), + forwarded, _keepalive: self.tasks.keepalive(), }), server: serve, @@ -2907,6 +2974,7 @@ impl Announcing { link: self.link, node: route.node, draining: route.draining, + available: route.available, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -2916,7 +2984,7 @@ impl Announcing { lends: serving.lends, lent: None, }); - shared.sync_route(prefix, claim); + shared.sync_route(prefix, claim, Gone::End); entries.push((prefix.clone(), id)); } drop(shared); @@ -3003,6 +3071,9 @@ struct RouteState { /// The advertisement, retracted on drop. announcement: AnnounceProducer, control: kio::Lock, + /// Whether a session landed it for its peer, so dropping it means the session lost + /// the route ([`RouteEvent::Offline`]), not that its origin ended it. + forwarded: bool, /// A route is lifecycle work: the origin's driver keeps running for as long as one /// lives, even once every producer handle is gone. _keepalive: Keepalive, @@ -3060,7 +3131,7 @@ impl RouteState { // first or finds the entries gone, never advertises after it. let mut control = self.control.lock(); let request = control.request.take(); - self.announcement.retract(false); + self.announcement.retract(Retract::Ended); drop(control); // Dropped outside every lock: refusing it wakes the requesters. drop(request); @@ -3182,7 +3253,7 @@ impl Route { // A failed answer can never announce. Retracted under the control lock, as // `ended` does, so a clone's racing announce finds the entries gone rather than // advertising through the ordinary branch now that the request is taken. - self.state.announcement.retract(false); + self.state.announcement.retract(Retract::Ended); drop(control); // The claim moved past the instance the job read: its requesters wait on for // the handler to prepare the job again under the new one. @@ -3384,10 +3455,18 @@ impl Claim { } /// Retire an explicit peer withdrawal, hiding derived paths only once its - /// last live advertisement at the prefix is gone. A lost session just drops. + /// last live advertisement at the prefix is gone. Readers see it end. A lost + /// session just drops, and readers see it go offline. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn withdrawn(self) { - self.state.announcement.retract(true); + self.state.announcement.retract(Retract::Withdrawn); + } + + /// Retire a route its announcer ended without a hop chain to stale, such as a + /// moq-transport PUBLISH_NAMESPACE_DONE: readers see it end, not go offline. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn ended(self) { + self.state.announcement.retract(Retract::Ended); } } @@ -3395,10 +3474,24 @@ impl Drop for RouteState { fn drop(&mut self) { // Retract before a job's unanswered request drops, so its requesters are refused // rather than asked again through this route. - self.announcement.retract(false); + self.announcement.retract(match self.forwarded { + true => Retract::Lost, + false => Retract::Ended, + }); } } +/// Why an advertisement leaves the table. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Retract { + /// The session it arrived on lost it: readers see it go offline. + Lost, + /// Its announcer ended it: readers see it end. + Ended, + /// A peer withdrew it, which also stales the routes derived through that peer. + Withdrawn, +} + /// The write half of an advertisement: a live claim that paths under a /// [`Pattern`] can be served. /// @@ -3497,9 +3590,10 @@ impl AnnounceProducer { entry.via = route.via; entry.node = route.node; entry.draining = route.draining; + entry.available = route.available; entry.advertised = true; let claim = entry.claim.clone(); - shared.sync_route(prefix, &claim); + shared.sync_route(prefix, &claim, Gone::End); if let Some(((lender, lender_id), instance)) = spent { shared.advance(&lender, lender_id, &instance); } @@ -3557,6 +3651,7 @@ impl AnnounceProducer { source: entry.entered(), node: entry.node, draining: entry.draining, + available: entry.available, }) } @@ -3584,7 +3679,7 @@ impl AnnounceProducer { } let claim = entry.claim.clone(); let spent = entry.lent.take().map(|lender| (lender, entry.instance.clone())); - shared.sync_route(prefix, &claim); + shared.sync_route(prefix, &claim, Gone::End); if let Some(((lender, lender_id), instance)) = spent { shared.advance(&lender, lender_id, &instance); } @@ -3602,7 +3697,7 @@ impl AnnounceProducer { /// Retract the route now: remove its table entries and reject anything still /// waiting on its queue. Idempotent, and what dropping the advertisement does. - fn retract(&self, withdrawn: bool) { + fn retract(&self, how: Retract) { let mut shared = self.shared.lock(); let guard = self.guard.lock().take(); let mut servers = Vec::new(); @@ -3615,7 +3710,7 @@ impl AnnounceProducer { // prefix. Remove this session's route before testing the remaining // claims, under the same lock, so overlapping withdrawals cannot // invalidate a newer advertisement or miss the last withdrawal. - if withdrawn + if how == Retract::Withdrawn && let Some(&peer) = entry.hops.iter().last() && peer != Hop::UNKNOWN && !shared @@ -3626,7 +3721,11 @@ impl AnnounceProducer { shared.withdrawn.entry(prefix.clone()).or_default().insert(peer); shared.restale(prefix); } - shared.sync_route(&entry.prefix, &entry.claim); + let gone = match how { + Retract::Lost => Gone::Offline, + Retract::Ended | Retract::Withdrawn => Gone::End, + }; + shared.sync_route(&entry.prefix, &entry.claim, gone); if entry.advertised && let Some((lender, lender_id)) = &entry.lent { @@ -3658,7 +3757,7 @@ impl AnnounceProducer { impl Drop for AnnounceProducer { fn drop(&mut self) { - self.retract(false); + self.retract(Retract::Ended); } } @@ -3728,7 +3827,7 @@ impl OwnClaims { fn apply(&mut self, event: RouteEvent) { let announce = match event { RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => announce, - RouteEvent::End(announce) => { + RouteEvent::End(announce) | RouteEvent::Offline(announce) => { self.best.remove(&announce.prefix); return self.node.own(announce.prefix, None); } @@ -3753,7 +3852,7 @@ impl OwnClaims { /// go to an upstream peer follows it. fn local(&mut self, event: RouteEvent) { match event { - RouteEvent::End(announce) => { + RouteEvent::End(announce) | RouteEvent::Offline(announce) => { self.local.remove(&announce.prefix); self.feed(announce.prefix); } @@ -6044,7 +6143,7 @@ impl OriginState { } } if let Some(claim) = changed { - self.sync_route(prefix, &claim); + self.sync_route(prefix, &claim, Gone::End); } } @@ -6092,7 +6191,7 @@ impl OriginState { /// every cursor it can present on. Called after an entry covering `prefix` /// was added, updated, or removed. `claim` is `prefix`'s [`prefix_claim`], /// held by the entry that changed. - fn sync_route(&mut self, prefix: &Path, claim: &Pattern) { + fn sync_route(&mut self, prefix: &Path, claim: &Pattern, gone: Gone) { // Split borrows: the recompute reads `routes` while mutating a cursor. let routes = &self.routes; let mut candidates = None; @@ -6103,7 +6202,7 @@ impl OriginState { if let Some(presented) = cursor.presented(prefix, claim) { if presented.is_empty() { // A cursor root can collapse several covering prefixes into one. - Self::sync_cursor(routes, cursor, &presented); + Self::sync_cursor(routes, cursor, &presented, gone); } else { // Every non-root presentation refers to this exact prefix. Rank // its routes once, then take each cursor's first eligible visible entry. @@ -6121,7 +6220,7 @@ impl OriginState { .filter(|entry| cursor.visible(entry)); let epochs = Epochs::of(visible.clone()); let best = visible.clone().find(|entry| !epochs.superseded(entry)); - cursor.update(&presented, best); + cursor.update(&presented, best, gone); } } } @@ -6239,7 +6338,7 @@ impl OriginState { /// Recompute the best visible route presenting at `presented` (relative) for /// one cursor and deliver the change, if any. - fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path) { + fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path, gone: Gone) { // The entries presenting here are the ones announced at the absolute // prefix, or, for the cursor's own root, at the root and every prefix // above it (all of which present as the empty path). Among them, the @@ -6253,7 +6352,7 @@ impl OriginState { } }; - cursor.update(presented, best); + cursor.update(presented, best, gone); } /// Register a cursor and replay the current best route per presented prefix. @@ -6287,7 +6386,7 @@ impl OriginState { } } for p in &presented { - Self::sync_cursor(&self.routes, &mut cursor, p); + Self::sync_cursor(&self.routes, &mut cursor, p, Gone::End); } for head in &cursor.heads { self.routes.add_cursor(head, id); @@ -6427,7 +6526,7 @@ impl OriginState { entry.instance = Instance::random(); entry.epoch = 0; let claim = entry.claim.clone(); - self.sync_route(prefix, &claim); + self.sync_route(prefix, &claim, Gone::End); } /// Whether `instance` still serves `path` as a request would resolve it now: the winning @@ -6855,7 +6954,7 @@ impl Requesting { if let Some(discovery) = &mut self.discovery { loop { match discovery.poll_next(waiter) { - Poll::Ready(Some(RouteEvent::End(_))) => self.routed = false, + Poll::Ready(Some(RouteEvent::End(_) | RouteEvent::Offline(_))) => self.routed = false, Poll::Ready(Some( RouteEvent::Start(announce) | RouteEvent::Restart(announce) | RouteEvent::Update(announce), )) => { @@ -8000,6 +8099,18 @@ impl Routes { } } + #[cfg(test)] + pub(crate) async fn expect_next_offline(&mut self, expected: impl TryInto) { + match self.expect_next().await { + RouteEvent::Offline(announce) => assert_eq!( + announce.prefix, + expected.try_into().expect("expected path"), + "wrong prefix" + ), + other => panic!("should go offline: got {other:?}"), + } + } + fn new( root: Path, cursors: Vec, @@ -8054,7 +8165,7 @@ impl Routes { /// Start a retraction's quarantine. A delivered announcement leaves it to /// expire: later ones for the prefix are updates, which it never holds. fn quarantine(&mut self, event: &RouteEvent, now: Instant) { - let RouteEvent::End(update) = event else { + let (RouteEvent::End(update) | RouteEvent::Offline(update)) = event else { return; }; if self.update_hold.is_zero() { @@ -8133,7 +8244,7 @@ impl Routes { .entry(announce.prefix.clone()) .or_insert_with(|| scope.announce()); } - RouteEvent::End(announce) => { + RouteEvent::End(announce) | RouteEvent::Offline(announce) => { self.guards.remove(&announce.prefix); } } @@ -8279,14 +8390,14 @@ impl Routes { for (presented, CursorRoute { meta, captures, .. }) in &cursor.current { let prefix = cursor.under.rejoin(presented); if revoked.contains(&prefix) { - state.apply_unannounce(prefix, meta.clone(), captures.clone()); + state.apply_unannounce(prefix, meta.clone(), captures.clone(), Gone::End); continue; } if advertised.contains(&prefix) { if state.pending.get(&prefix).is_some_and(|pending| { matches!( pending.update, - Some(PendingUpdate::Unannounce(_) | PendingUpdate::UnannounceAnnounce { .. }) + Some(PendingUpdate::Unannounce(..) | PendingUpdate::UnannounceAnnounce { .. }) ) }) { state.apply_announce(prefix, meta.clone(), captures.clone()); @@ -11059,7 +11170,7 @@ mod tests { let replacement = broadcast::Info::new().produce(); queued(&echo).await.serve(&replacement); resolved.assert_not_closed(); - view.expect_next_ended("room").await; + view.expect_next_offline("room").await; } /// A route that ends abruptly releases its open groups, and the next route may @@ -11469,7 +11580,7 @@ mod tests { assert_eq!(announced.expect_next_restarted("live").await.cost, Cost::new(3)); drop(first); - announced.expect_next_ended("live").await; + announced.expect_next_offline("live").await; announced.assert_next_wait(); } @@ -11594,8 +11705,8 @@ mod tests { drop(video_route); assert_eq!(all.expect_next_active("room/live").await.cost, Cost::new(9)); assert_eq!(rooted.expect_next_active("").await.cost, Cost::new(9)); - excluded.expect_next_ended("room/live").await; - video.expect_next_ended("room/live").await; + excluded.expect_next_offline("room/live").await; + video.expect_next_offline("room/live").await; } #[kio::test] @@ -11667,7 +11778,7 @@ mod tests { assert_eq!(update.prefix.as_str(), "live"); assert!(next().is_none()); drop(server); - assert!(matches!(next(), Some(Some(RouteEvent::End(_))))); + assert!(matches!(next(), Some(Some(RouteEvent::Offline(_))))); } #[kio::test] @@ -11678,7 +11789,7 @@ mod tests { announced.expect_next_active("live").await; drop(server); - announced.expect_next_ended("live").await; + announced.expect_next_offline("live").await; } #[kio::test] @@ -11960,8 +12071,8 @@ mod tests { for (i, route) in far.into_iter().enumerate().rev() { drop(route); match announced.try_next() { - Some(RouteEvent::End(end)) => assert_eq!(end.prefix, Path::new(&format!("room/{i}")).unwrap()), - other => panic!("should be a retraction: got {other:?}"), + Some(RouteEvent::Offline(end)) => assert_eq!(end.prefix, Path::new(&format!("room/{i}")).unwrap()), + other => panic!("should go offline: got {other:?}"), } } @@ -12008,7 +12119,7 @@ mod tests { // The last route goes: retracted at once. drop(farther); - announced.expect_next_ended("room").await; + announced.expect_next_offline("room").await; } /// After a retraction, a worse path to the same publisher waits out the @@ -12032,7 +12143,7 @@ mod tests { let near = peer.forward("named", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("named").await; drop(near); - announced.expect_next_ended("named").await; + announced.expect_next_offline("named").await; let start = kio::clock::now(); let _stale = peer.forward("named", route(&[9, 3, 4], 4)).unwrap(); match announced.next().await.unwrap() { @@ -12044,7 +12155,7 @@ mod tests { let near = peer.forward("anonymous", route(&[0, 2], 2)).unwrap(); announced.expect_next_active("anonymous").await; drop(near); - announced.expect_next_ended("anonymous").await; + announced.expect_next_offline("anonymous").await; let _other = peer.forward("anonymous", route(&[0, 3, 4], 4)).unwrap(); announced.expect_next_active("anonymous").await; @@ -12055,7 +12166,7 @@ mod tests { let churn = peer.forward("churn", route(&chain, 2)).unwrap(); announced.expect_next_active("churn").await; drop(churn); - announced.expect_next_ended("churn").await; + announced.expect_next_offline("churn").await; } assert_eq!(announced.anchoring.len(), queued + 1); @@ -12064,7 +12175,7 @@ mod tests { let gone = peer.forward("gone", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("gone").await; drop(gone); - announced.expect_next_ended("gone").await; + announced.expect_next_offline("gone").await; assert!(!announced.retracted.is_empty()); for _ in 0..2 { announced.assert_next_wait(); @@ -12212,7 +12323,7 @@ mod tests { .unwrap(); assert_eq!(local.expect_next_active("both").await.source(), Source::Local); drop(ingest); - local.expect_next_ended("both").await; + local.expect_next_offline("both").await; // Resolution agrees with the cursor: a peer-only path is unroutable here, // while the full view queues the request on the peer's route. @@ -12269,7 +12380,7 @@ mod tests { .unwrap(); assert_eq!(toward_core.expect_next_active("both").await.source(), Source::Peer(8)); drop(mesh); - toward_core.expect_next_ended("both").await; + toward_core.expect_next_offline("both").await; // Resolution agrees with the cursor: an upstream-only path is unroutable // toward an upstream, while the full view queues it on the core's route. @@ -12997,7 +13108,7 @@ mod tests { // Nothing serves an uncovered path once the handler is gone. drop(dynamic); - announced.expect_next_ended("").await; + announced.expect_next_offline("").await; let err = consumer .request("something/else", None) .now_or_never() diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 46a0ac02475..b5d1be8cd1a 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -80,8 +80,9 @@ pub(crate) enum Output { claim: Path, served: Served, }, - /// Stop serving `claim` of `id` through this stream. - Unserve { id: Id, claim: Path }, + /// Stop serving `claim` of `id` through this stream. `ended` when the claim's origin + /// withdrew it, as far as this node knows; otherwise the node let go of it here. + Unserve { id: Id, claim: Path, ended: bool }, } /// The node every lite-07 session of one origin shares. @@ -227,7 +228,7 @@ impl Outbox { } // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. - Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim, .. } => { let key = (id, claim.clone()); if let Some(&seq) = self.serving.get(&key) && let Output::Serve { served, .. } = &output @@ -262,7 +263,7 @@ impl Outbox { } None => { let (_, output) = self.serves.pop_first()?; - if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim } = &output { + if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim, .. } = &output { self.serving.remove(&(*id, claim.clone())); } return Some(output); @@ -376,6 +377,7 @@ impl Home { Output::Unserve { id, claim: claim.clone(), + ended: false, }, ); } @@ -646,13 +648,17 @@ impl State { self.router.echo(key, id); } } - // The shown claims below are read whole; the events only say which changed. - self.router.events(); + // The shown claims below are read whole; the events only say which changed, and + // whether a claim that went was ended by its origin or let go here. + let mut gone = BTreeMap::new(); + for event in self.router.events() { + gone.insert((event.id, event.claim), event.kind == super::Kind::End); + } let changed = self.router.changed(); let declared = self.router.declared(); for id in changed { if Some(id) != declared { - self.home(id); + self.home(id, &gone); } } } @@ -660,7 +666,7 @@ impl State { /// Serve each shown claim of `id` through every stream with a usable feasible /// route to it that can carry the claim, keeping it where it is, unavailable, while /// there is none. - fn home(&mut self, id: Id) { + fn home(&mut self, id: Id, ended: &BTreeMap<(Id, Path), bool>) { let claims: Vec<(Path, Claim, bool)> = (self.router.shown().range((id, Path::empty())..)) .take_while(|((shown, _), _)| *shown == id) .map(|((_, claim), shown)| (claim.clone(), shown.item.clone(), shown.available)) @@ -728,6 +734,7 @@ impl State { let Some(home) = self.homes.remove(&(id, claim.clone())) else { continue; }; + let ended = ended.get(&(id, claim.clone())).copied().unwrap_or(false); for key in home.keys.into_keys() { push( &self.queues, @@ -735,6 +742,7 @@ impl State { Output::Unserve { id, claim: claim.clone(), + ended, }, ); } @@ -838,7 +846,8 @@ mod tests { served(outputs(&receive)), [Output::Unserve { id: 1, - claim: p("live/x") + claim: p("live/x"), + ended: true }] ); } @@ -878,7 +887,8 @@ mod tests { served(outputs(&far)), [Output::Unserve { id: 1, - claim: p("live/x") + claim: p("live/x"), + ended: false }] ); assert!(served(outputs(&near)).is_empty(), "the remaining route serves on"); diff --git a/rs/moq-net/tests/integration/announce_covering_route.rs b/rs/moq-net/tests/integration/announce_covering_route.rs index 9c56dee001f..642a0aeb7c1 100644 --- a/rs/moq-net/tests/integration/announce_covering_route.rs +++ b/rs/moq-net/tests/integration/announce_covering_route.rs @@ -36,6 +36,7 @@ fn drain(announced: &mut moq_net::origin::Routes) -> Vec { Event::Update(announce) => ("Update", announce), Event::Restart(announce) => ("Restart", announce), Event::End(announce) => ("End", announce), + Event::Offline(announce) => ("Offline", announce), }; seen.push(format!("{kind} {}", announce.prefix)); } diff --git a/rs/moq-net/tests/integration/announce_to_serve.rs b/rs/moq-net/tests/integration/announce_to_serve.rs index bfe2a367c1c..58687888909 100644 --- a/rs/moq-net/tests/integration/announce_to_serve.rs +++ b/rs/moq-net/tests/integration/announce_to_serve.rs @@ -53,6 +53,7 @@ fn drain(announced: &mut moq_net::origin::Routes) -> Vec { Event::Update(announce) => ("Update", announce), Event::Restart(announce) => ("Restart", announce), Event::End(announce) => ("End", announce), + Event::Offline(announce) => ("Offline", announce), }; seen.push(format!("{kind} {}", announce.prefix)); } diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index f2a95eb22a1..b75a412ecd1 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -413,7 +413,7 @@ async fn wait_announced(origin: &origin::Consumer, path: &str, active: bool) { | moq_net::origin::RouteEvent::Restart(update) => { live.insert(update.prefix.to_string()); } - moq_net::origin::RouteEvent::End(update) => { + moq_net::origin::RouteEvent::End(update) | moq_net::origin::RouteEvent::Offline(update) => { live.remove(update.prefix.as_str()); } }; diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs index fb2cb7fe1c1..8e1bd07dc7d 100644 --- a/rs/moq-net/tests/integration/claim_continues.rs +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -100,6 +100,7 @@ async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Ins origin::RouteEvent::Update(a) => ("update", a), origin::RouteEvent::Restart(a) => ("restart", a), origin::RouteEvent::End(a) => ("end", a), + origin::RouteEvent::Offline(a) => ("offline", a), }; events.push((kind, announce.prefix.to_string(), announce.route.instance().clone())); } diff --git a/rs/moq-net/tests/integration/dial_split_horizon.rs b/rs/moq-net/tests/integration/dial_split_horizon.rs index f7f4daa86fb..bf9c675578b 100644 --- a/rs/moq-net/tests/integration/dial_split_horizon.rs +++ b/rs/moq-net/tests/integration/dial_split_horizon.rs @@ -103,7 +103,7 @@ async fn learned_hop(dialer: &origin::Producer, path: &str) -> u64 { moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Update(announce) | moq_net::origin::RouteEvent::Restart(announce) => announce, - moq_net::origin::RouteEvent::End(_) => continue, + moq_net::origin::RouteEvent::End(_) | moq_net::origin::RouteEvent::Offline(_) => continue, }; if announce.prefix.as_str() == path { return announce; diff --git a/rs/moq-net/tests/integration/extensions.rs b/rs/moq-net/tests/integration/extensions.rs index c86ac8d64c1..2111503b400 100644 --- a/rs/moq-net/tests/integration/extensions.rs +++ b/rs/moq-net/tests/integration/extensions.rs @@ -49,7 +49,7 @@ fn announced(origin: &origin::Producer, path: &str) -> bool { | moq_net::origin::RouteEvent::Restart(update) => { live |= update.prefix.as_str() == path; } - moq_net::origin::RouteEvent::End(update) => { + moq_net::origin::RouteEvent::End(update) | moq_net::origin::RouteEvent::Offline(update) => { live &= update.prefix.as_str() != path; } } diff --git a/rs/moq-net/tests/integration/instance.rs b/rs/moq-net/tests/integration/instance.rs index ee64af26bd8..9dab184587d 100644 --- a/rs/moq-net/tests/integration/instance.rs +++ b/rs/moq-net/tests/integration/instance.rs @@ -41,7 +41,7 @@ async fn ended(announced: &mut origin::Routes) { .expect("announce hung") .expect("announce ended") { - origin::RouteEvent::End(_) => {} + origin::RouteEvent::End(_) | origin::RouteEvent::Offline(_) => {} other => panic!("expected an end, got {other:?}"), } } diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index 3e7a72d2a94..252393cdcbe 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -112,7 +112,7 @@ async fn synced(announced: &mut origin::Routes) -> Vec { origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { seen.push(announce.prefix.to_string()) } - origin::RouteEvent::End(announce) => seen.retain(|prefix| *prefix != announce.prefix.as_str()), + origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => seen.retain(|prefix| *prefix != announce.prefix.as_str()), origin::RouteEvent::Update(_) => {} } } @@ -426,7 +426,7 @@ async fn settled(announced: &mut origin::Routes, held: &mut Vec) -> bool origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { held.push(announce.prefix.to_string()) } - origin::RouteEvent::End(announce) => { + origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => { ended = true; held.retain(|prefix| *prefix != announce.prefix.as_str()) } diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs index 62f3d94e8dc..50e19dd6894 100644 --- a/rs/moq-net/tests/integration/mount.rs +++ b/rs/moq-net/tests/integration/mount.rs @@ -69,6 +69,7 @@ async fn drain(routes: &mut origin::Routes) -> Vec<(bool, String)> { match event.expect("announce ended") { origin::RouteEvent::Start(announce) => events.push((true, announce.prefix.to_string())), origin::RouteEvent::End(announce) => events.push((false, announce.prefix.to_string())), + origin::RouteEvent::Offline(announce) => events.push((false, format!("offline {}", announce.prefix))), _ => {} } } @@ -97,6 +98,16 @@ fn ended(paths: &[&str]) -> Vec<(bool, String)> { paths.iter().map(|path| (false, path.to_string())).collect() } +/// What a reader on `version` sees once `paths` leave without a withdrawal it can +/// certify: offline where the route layer carries it, ended where only an ANNOUNCE +/// says it went. +fn lost(version: Version, paths: &[&str]) -> Vec<(bool, String)> { + match version.to_string().as_str() { + "moq-lite-07-wip" => paths.iter().map(|path| (false, format!("offline {path}"))).collect(), + _ => ended(paths), + } +} + async fn mounted(version: &str) { let version: Version = version.parse().unwrap(); let source = produce_origin(1); @@ -109,9 +120,10 @@ async fn mounted(version: &str) { assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); - // What leaves the target leaves the mount. + // What leaves the target leaves the mount. The route layer carries the mount's + // withdrawal without a floor certifying it, so it goes offline there. drop(published.remove(0)); - assert_eq!(drain(&mut routes).await, ended(&["shared/cam"]), "{version}"); + assert_eq!(drain(&mut routes).await, lost(version, &["shared/cam"]), "{version}"); } /// A session rooted inside the mount reads only the target. @@ -146,7 +158,7 @@ async fn relayed(version: &str) { drop(upstream); kio::clock::sleep(Duration::from_millis(500)).await; - assert_eq!(drain(&mut routes).await, ended(&["own", "shared/cam"]), "{version}"); + assert_eq!(drain(&mut routes).await, lost(version, &["own", "shared/cam"]), "{version}"); } macro_rules! mounted_tests { diff --git a/rs/moq-net/tests/integration/multi_cdn.rs b/rs/moq-net/tests/integration/multi_cdn.rs index bd335b12d2e..b5849ba5da8 100644 --- a/rs/moq-net/tests/integration/multi_cdn.rs +++ b/rs/moq-net/tests/integration/multi_cdn.rs @@ -134,7 +134,7 @@ async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Option bool { let route = match announce { origin::RouteEvent::Start(a) | origin::RouteEvent::Update(a) | origin::RouteEvent::Restart(a) => &a.route, - origin::RouteEvent::End(_) => return false, + origin::RouteEvent::End(_) | origin::RouteEvent::Offline(_) => return false, }; route.cost().value() >= A_COST } diff --git a/rs/moq-net/tests/integration/session_panic.rs b/rs/moq-net/tests/integration/session_panic.rs index 8ffe4bdf489..6d74c3eb164 100644 --- a/rs/moq-net/tests/integration/session_panic.rs +++ b/rs/moq-net/tests/integration/session_panic.rs @@ -84,7 +84,7 @@ async fn until(announced: &mut origin::Routes, path: &str, active: bool) { origin::RouteEvent::Start(announce) | origin::RouteEvent::Update(announce) | origin::RouteEvent::Restart(announce) => (announce, true), - origin::RouteEvent::End(announce) => (announce, false), + origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => (announce, false), }; if announce.prefix.as_str() == path && now_active == active { return; diff --git a/rs/moq-net/tests/integration/support/mod.rs b/rs/moq-net/tests/integration/support/mod.rs index f6b86d93749..825f5275a0a 100644 --- a/rs/moq-net/tests/integration/support/mod.rs +++ b/rs/moq-net/tests/integration/support/mod.rs @@ -57,6 +57,7 @@ pub enum Kind { Update, Restart, End, + Offline, } /// Every update per prefix the watcher has seen. Await [`kio::clock::sim::idle`] @@ -83,6 +84,7 @@ pub fn watch(mut announced: origin::Routes) -> mpsc::UnboundedReceiver<(String, origin::RouteEvent::Update(announce) => (Kind::Update, announce), origin::RouteEvent::Restart(announce) => (Kind::Restart, announce), origin::RouteEvent::End(announce) => (Kind::End, announce), + origin::RouteEvent::Offline(announce) => (Kind::Offline, announce), }; if tx.unbounded_send((announce.prefix.to_string(), kind)).is_err() { break; diff --git a/rs/moq-net/tests/integration/synced.rs b/rs/moq-net/tests/integration/synced.rs index b4c9d96f199..a780064697f 100644 --- a/rs/moq-net/tests/integration/synced.rs +++ b/rs/moq-net/tests/integration/synced.rs @@ -33,7 +33,7 @@ async fn read_until_synced(announced: &mut origin::Routes) -> BTreeSet { origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { seen.insert(announce.prefix.to_string()); } - origin::RouteEvent::End(announce) => { + origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => { seen.remove(announce.prefix.as_str()); } origin::RouteEvent::Update(_) => {} diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 26ad25d1d1e..573208b060a 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2650,7 +2650,7 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 67191e53a22..8ec4195b570 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1333,7 +1333,7 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index 53536d57dc7..6d59f9d5029 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -1403,7 +1403,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index b4b4e480811..39af68f1144 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1853,7 +1853,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 268fc0c0713..e1f6a444b50 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -316,6 +316,6 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/connection.rs b/rs/moq-relay/tests/integration/connection.rs index 6b7ba1ee241..b38286fa4f7 100644 --- a/rs/moq-relay/tests/integration/connection.rs +++ b/rs/moq-relay/tests/integration/connection.rs @@ -105,7 +105,7 @@ async fn next_active(announced: &mut moq_net::origin::Routes) -> String { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => return route.prefix.to_string(), - moq_net::origin::RouteEvent::End(_) => {} + moq_net::origin::RouteEvent::End(_) | moq_net::origin::RouteEvent::Offline(_) => {} } } } diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 625c101f4d6..e9ccc547a90 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -602,6 +602,6 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/memory.rs b/rs/moq-relay/tests/integration/memory.rs index 46c94b3f773..aa0b410cdc7 100644 --- a/rs/moq-relay/tests/integration/memory.rs +++ b/rs/moq-relay/tests/integration/memory.rs @@ -323,7 +323,7 @@ pub fn record(mut announced: moq_net::origin::Routes) -> std::sync::Arc (route, false), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => (route, false), moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => (route, true), diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index badc918d50f..8f3be156f3c 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -927,7 +927,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/runtime_workers.rs b/rs/moq-relay/tests/integration/runtime_workers.rs index 00bdf2fce12..8277a3971e0 100644 --- a/rs/moq-relay/tests/integration/runtime_workers.rs +++ b/rs/moq-relay/tests/integration/runtime_workers.rs @@ -169,6 +169,6 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/smoke.rs b/rs/moq-relay/tests/integration/smoke.rs index 1f32059e683..b0cc1971b0b 100644 --- a/rs/moq-relay/tests/integration/smoke.rs +++ b/rs/moq-relay/tests/integration/smoke.rs @@ -997,6 +997,6 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index 8520f54e094..ca72eb85be9 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -351,7 +351,7 @@ async fn synced_listing(url: url::Url) -> BTreeSet { .await; use moq_net::origin::RouteEvent; match next { - Some(RouteEvent::End(route)) => listed.remove(route.prefix.as_str()), + Some(RouteEvent::End(route) | RouteEvent::Offline(route)) => listed.remove(route.prefix.as_str()), Some(RouteEvent::Start(route) | RouteEvent::Update(route) | RouteEvent::Restart(route)) => { listed.insert(route.prefix.to_string()) } diff --git a/rs/moq-room/src/room.rs b/rs/moq-room/src/room.rs index c394de44207..541f0c3820a 100644 --- a/rs/moq-room/src/room.rs +++ b/rs/moq-room/src/room.rs @@ -75,7 +75,7 @@ impl Room { loop { match self.announced.poll_next(waiter) { Poll::Ready(Some(announce::Event::Present(announce))) => self.listed(announce.prefix), - Poll::Ready(Some(announce::Event::Absent(announce))) => { + Poll::Ready(Some(announce::Event::Absent(announce) | announce::Event::Offline(announce))) => { let Some(index) = self.participants.iter().position(|p| p.path == announce.prefix) else { continue; }; @@ -87,6 +87,8 @@ impl Room { broadcast: None, })); } + // A participant stays listed while its path is held; its follow resumes it. + Poll::Ready(Some(announce::Event::Available(_) | announce::Event::Unavailable(_))) => {} Poll::Ready(None) => return Poll::Ready(None), Poll::Pending => break, } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 42f6157c80f..3296d779fab 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -816,7 +816,7 @@ impl Merged { let instance = update.route.instance().clone(); changed |= self.apply_announce(&update, instance, true) } - Poll::Ready(Some(moq_net::origin::RouteEvent::End(update))) => { + Poll::Ready(Some(moq_net::origin::RouteEvent::End(update) | moq_net::origin::RouteEvent::Offline(update))) => { let instance = update.route.instance().clone(); changed |= self.apply_announce(&update, instance, false) } @@ -1145,7 +1145,7 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index 83e2fce4597..b15f1c6b155 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -178,7 +178,7 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index d9b3b8da094..4eb62f742b2 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1675,7 +1675,7 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), } } diff --git a/rs/moq-tokio/examples/clock.rs b/rs/moq-tokio/examples/clock.rs index d88c9b9c613..9f5dcb7f8b8 100644 --- a/rs/moq-tokio/examples/clock.rs +++ b/rs/moq-tokio/examples/clock.rs @@ -118,8 +118,14 @@ async fn main() -> anyhow::Result<()> { clock = Some(Subscriber::new(track)); } announce::Event::Absent(update) => { + tracing::warn!(broadcast = %update.prefix, "broadcast ended, waiting..."); + } + announce::Event::Offline(update) | announce::Event::Unavailable(update) => { tracing::warn!(broadcast = %update.prefix, "broadcast is offline, waiting..."); } + announce::Event::Available(update) => { + tracing::info!(broadcast = %update.prefix, "broadcast is reachable again"); + } }, res = reconnect.closed() => return Ok(res?), // Drops the previous subscriber on each new announce. diff --git a/rs/moq-tokio/src/origin.rs b/rs/moq-tokio/src/origin.rs index 4a44f316071..f2f9c6aaa20 100644 --- a/rs/moq-tokio/src/origin.rs +++ b/rs/moq-tokio/src/origin.rs @@ -41,9 +41,12 @@ mod tests { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 1834a1b800d..7b7addbe1ef 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1962,9 +1962,12 @@ mod tests { /// The next route and whether it is active. #[cfg(all(feature = "uds", unix))] async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index 29a00dc44ba..6777c17651b 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -1186,8 +1186,11 @@ async fn quic_windows() { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 4e5cca42e6a..cbcd83563ad 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -818,7 +818,7 @@ async fn next_route(routes: &mut moq_net::origin::Routes) -> (moq_net::announce: .expect("announce timeout") .expect("origin closed"); match event { - moq_net::origin::RouteEvent::End(announce) => (announce, false), + moq_net::origin::RouteEvent::End(announce) | moq_net::origin::RouteEvent::Offline(announce) => (announce, false), moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Update(announce) | moq_net::origin::RouteEvent::Restart(announce) => (announce, true), @@ -3507,8 +3507,11 @@ async fn abort_carries_its_code_to_the_peer() { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } diff --git a/rs/moq-webrtc/src/lib.rs b/rs/moq-webrtc/src/lib.rs index 8bb4df6704d..fa47bf71eb3 100644 --- a/rs/moq-webrtc/src/lib.rs +++ b/rs/moq-webrtc/src/lib.rs @@ -75,9 +75,12 @@ mod tests { /// The next route and whether it is active. async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { - match announced.next().await? { - moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) => Some((route, false)), + loop { + return match announced.next().await? { + moq_net::announce::Event::Present(route) => Some((route, true)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, + }; } } From 43d4871c1895111ed991ef289e15bab1875f35f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 07:42:45 -0800 Subject: [PATCH 03/11] wip(js): announcement availability and offline Co-Authored-By: Claude Opus 5.5 --- js/moq-boy/src/element.tsx | 5 +- js/net/examples/discovery.ts | 2 +- js/net/src/announced.test-util.ts | 2 +- js/net/src/announced.test.ts | 34 ++++++++++- js/net/src/announced.ts | 59 ++++++++++++++----- js/net/src/connection/forward.test.ts | 2 +- js/net/src/connection/forward.ts | 10 ++-- js/net/src/connection/reload.test.ts | 6 +- js/net/src/connection/reload.ts | 7 ++- js/net/src/hop.ts | 15 ++++- js/net/src/internal.ts | 7 ++- js/net/src/origin.test.ts | 22 ++++++- js/net/src/origin.ts | 26 +++++++- js/net/src/wire.ts | 6 ++ js/room/src/room.ts | 5 +- js/watch/src/broadcast.ts | 4 +- js/watch/src/echo.test.ts | 4 +- rs/moq-gst/src/source/imp.rs | 2 +- .../tests/integration/cluster_idle.rs | 3 +- 19 files changed, 173 insertions(+), 48 deletions(-) diff --git a/js/moq-boy/src/element.tsx b/js/moq-boy/src/element.tsx index 6220c3f9530..698b063dd01 100644 --- a/js/moq-boy/src/element.tsx +++ b/js/moq-boy/src/element.tsx @@ -224,7 +224,8 @@ export default class MoqBoy extends HTMLElement { if (!capture?.isLiteral) continue; const id = capture.text; - if (entry.kind !== "absent" && !this.#sessions.has(id)) { + const gone = entry.kind === "absent" || entry.kind === "offline"; + if (entry.kind === "present" && !this.#sessions.has(id)) { const config: GameConfig = { sessionId: id, connection: this.connection, @@ -236,7 +237,7 @@ export default class MoqBoy extends HTMLElement { const game = new Game(config); this.#sessions.set(id, game); this.games.set(new Map(this.#sessions)); - } else if (entry.kind === "absent") { + } else if (gone) { const game = this.#sessions.get(id); if (game) { game.close(); diff --git a/js/net/examples/discovery.ts b/js/net/examples/discovery.ts index eb070025ff1..5b239a6a544 100644 --- a/js/net/examples/discovery.ts +++ b/js/net/examples/discovery.ts @@ -11,7 +11,7 @@ async function main() { // Discover broadcasts announced by the server for await (const announcement of announced) { - if (announcement.kind === "absent" || requests.has(announcement.prefix)) continue; + if (announcement.kind !== "present" || requests.has(announcement.prefix)) continue; console.log("New stream available:", announcement.prefix); // One request per path follows its publisher restarts until closed. diff --git a/js/net/src/announced.test-util.ts b/js/net/src/announced.test-util.ts index 36056ca9d5f..32de92725fe 100644 --- a/js/net/src/announced.test-util.ts +++ b/js/net/src/announced.test-util.ts @@ -15,7 +15,7 @@ export async function announcedView( for await (const event of stream) value.update((previous) => { const next = new Map(previous); - if (event.kind === "end") next.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") next.delete(event.prefix); else next.set(event.prefix, event.route); return next; }); diff --git a/js/net/src/announced.test.ts b/js/net/src/announced.test.ts index 31f0e24fc56..6cdeb8aaca3 100644 --- a/js/net/src/announced.test.ts +++ b/js/net/src/announced.test.ts @@ -63,9 +63,39 @@ test("a listing shows a route's facts, never its epoch, hops, credential, or dra kind: "present", prefix: p("a"), captures: undefined, - route: { instance, cost: 5n }, + route: { instance, cost: 5n, available: true }, }); - expect(Object.keys(event?.route ?? {}).sort()).toEqual(["cost", "instance"]); + expect(Object.keys(event?.route ?? {}).sort()).toEqual(["available", "cost", "instance"]); +}); + +test("a listing reports availability flips and tells an offline path from an absent one", async () => { + const producer = new Announce.Producer(); + const listing = Announce.list(producer.consume()); + const route = WireRoute.normalize({ instance: Instance.random() }); + const held = { ...route, available: false }; + + producer.append({ prefix: p("a"), captures: undefined, kind: "start", route: held }); + // A re-price that leaves availability as it was is not reported. + producer.append({ prefix: p("a"), captures: undefined, kind: "update", route: { ...held, cost: 3n } }); + producer.append({ prefix: p("a"), captures: undefined, kind: "update", route }); + producer.append({ prefix: p("a"), captures: undefined, kind: "restart", route: held }); + producer.append({ prefix: p("a"), captures: undefined, kind: "offline", route: held }); + producer.append({ prefix: p("b"), captures: undefined, kind: "start", route }); + producer.append({ prefix: p("b"), captures: undefined, kind: "end", route }); + + const seen = []; + for (let i = 0; i < 6; i++) { + const event = await listing.next(); + seen.push(`${event?.kind} ${event?.prefix} ${event?.route.available}`); + } + expect(seen).toEqual([ + "present a false", + "available a true", + "unavailable a false", + "offline a false", + "present b true", + "absent b true", + ]); }); test("the consumer is an async iterable of the same events", async () => { diff --git a/js/net/src/announced.ts b/js/net/src/announced.ts index f188c6ab9e6..1189aadca64 100644 --- a/js/net/src/announced.ts +++ b/js/net/src/announced.ts @@ -37,26 +37,35 @@ export interface Announce { export type Routed = Omit & { route: WireRoute }; /** - * What an announcement listing yields: a path becoming present or absent. + * What an announcement listing yields: a path becoming present, absent, or offline, or its + * availability changing. * - * `present`: a route now covers the path, which was absent. `absent`: no route covers it any - * more, carrying its last advertised route. Another instance taking a path over keeps it - * present: follow it with the origin's `follow(path)` to see the broadcast change. + * `present`: a route now covers the path, which was absent; it may arrive unavailable (see + * {@link RouteInfo.available}). `unavailable`: the present path lost every usable path to its + * origin but may recover one, so it stays present while requests for it may fail. + * `available`: it recovered one: request it again. `absent`: the origin withdrew the path, and + * no route covers it any more. `offline`: this side let go of the path, and no route covers it + * any more: the session it arrived on closed, or the network lost every path to it, so its + * origin may still serve it. Both carry its last advertised route. + * + * Another instance taking a path over keeps it present: follow it with the origin's + * `follow(path)` to see the broadcast change. * * @public */ -export type Event = { kind: "present" | "absent" } & Announce; +export type Event = { kind: "present" | "unavailable" | "available" | "absent" | "offline" } & Announce; /** * Every change to a route, for code that forwards or routes them. * - * `start`: a route now covers the prefix. `update`: the route covering it changed hops or cost, - * in place. `restart`: another instance now serves the prefix. `end`: no route covers it any - * more. + * `start`: a route now covers the prefix. `update`: the route covering it changed hops, cost, + * or availability, in place. `restart`: another instance now serves the prefix. `end`: the + * origin withdrew it, and no route covers it any more. `offline`: this side let go of it, + * and no route covers it any more. * * @internal */ -export type RouteEvent = { kind: "start" | "update" | "restart" | "end" } & Routed; +export type RouteEvent = { kind: "start" | "update" | "restart" | "end" | "offline" } & Routed; /** * Options for an announcement stream. @@ -274,7 +283,8 @@ export class Routes { let makeConsumer: (routes: Routes) => Consumer; /** - * The paths announced under a scope, each reported as it becomes present or absent. + * The paths announced under a scope, each reported as it becomes present, absent, or offline, + * and as its availability changes. * * Created internally: obtain one from an origin or the connection's `announced(scope)`. * @@ -282,6 +292,8 @@ let makeConsumer: (routes: Routes) => Consumer; */ export class Consumer { #routes: Routes; + // The present paths last reported unavailable, bounded by the present set. + #unavailable = new Set(); private constructor(routes: Routes) { this.#routes = routes; @@ -318,14 +330,30 @@ export class Consumer { } } - /** Returns the next path that became present or absent, or undefined once the listing or this handle closes. */ + /** Returns the next change to a path, or undefined once the listing or this handle closes. */ async next(): Promise { for (;;) { const event = await this.#routes.next(); if (!event) return undefined; - const { kind, prefix, captures, route } = event; - if (kind === "start") return { kind: "present", prefix, captures, route: routeInfo(route) }; - if (kind === "end") return { kind: "absent", prefix, captures, route: routeInfo(route) }; + const { kind, prefix, captures } = event; + const route = routeInfo(event.route); + switch (kind) { + case "start": + if (!route.available) this.#unavailable.add(prefix); + return { kind: "present", prefix, captures, route }; + case "update": + case "restart": { + const was = !this.#unavailable.has(prefix); + if (was === route.available) continue; + if (route.available) this.#unavailable.delete(prefix); + else this.#unavailable.add(prefix); + return { kind: route.available ? "available" : "unavailable", prefix, captures, route }; + } + case "end": + case "offline": + this.#unavailable.delete(prefix); + return { kind: kind === "end" ? "absent" : "offline", prefix, captures, route }; + } } } @@ -339,7 +367,8 @@ export class Consumer { } /** - * The listing over a route stream: only paths becoming present or absent. + * The listing over a route stream: only paths becoming present, absent, or offline, and their + * availability changing. * * @internal */ diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index 0ba6f9a6a41..f3d8cfd8470 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -383,7 +383,7 @@ test("an exclusive session closes every request beneath a broader one before it expect(wireOf(origin).routes(Path.from("a/b/x"))).toBe(false); start(session.stream("a/**"), "a/b/x"); await settle(); - expect(events).toEqual(["start a/b/x", "end a/b/x", "start a/b/x"]); + expect(events).toEqual(["start a/b/x", "offline a/b/x", "start a/b/x"]); expect(origin.discovery.peek()).toBe(true); // The narrower watch rides the broader request from here on. diff --git a/js/net/src/connection/forward.ts b/js/net/src/connection/forward.ts index 3b69ab8d4dc..fc73b00e290 100644 --- a/js/net/src/connection/forward.ts +++ b/js/net/src/connection/forward.ts @@ -91,8 +91,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { const report = (interest: Interest, event: Announce.RouteEvent) => { const { prefix, route } = event; - if (event.kind === "end") { - unreport(interest, prefix); + if (event.kind === "end" || event.kind === "offline") { + unreport(interest, prefix, event.kind === "end"); return; } interest.reported.add(prefix); @@ -132,7 +132,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { } }; - const unreport = (interest: Interest, prefix: Path.Valid) => { + /** Stop `interest` reporting `prefix`, which its peer `ended` rather than lost. */ + const unreport = (interest: Interest, prefix: Path.Valid, ended = false) => { interest.reported.delete(prefix); const existing = routes.get(prefix); if (!existing) return; @@ -142,7 +143,8 @@ export function forwardAnnounced(conn: Session, origin: OriginProducer): void { const next = existing.reporters.values().next(); if (next.done) { routes.delete(prefix); - existing.handle.close(); + if (ended) wireOf(existing.handle).end(); + else existing.handle.close(); } else if (driving) { // The next request takes over driving. Without an instance nothing proves it reports // the stream the handle serves, so the route restarts rather than stitch two together. diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 28345bbea48..7a1848585fe 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -215,7 +215,7 @@ function present(routes: Announce.Routes): Getter> { for await (const event of routes) { paths.update((previous) => { const next = new Set(previous); - if (event.kind === "end") next.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") next.delete(event.prefix); else next.add(event.prefix); return next; }); @@ -1128,7 +1128,7 @@ test("a watcher syncs with the server's initial set, and again after a reconnect const seen = new Set(); void (async () => { for await (const event of watching) { - if (event.kind === "end") seen.delete(event.prefix); + if (event.kind === "end" || event.kind === "offline") seen.delete(event.prefix); else seen.add(event.prefix); } })(); @@ -1178,7 +1178,7 @@ test("a pump owes the retractions of the upstream it replaces", async () => { const second = new Announce.Producer(); second.synced(); upstream.set(second); - expect(await parked).toMatchObject({ prefix: Path.from("a"), kind: "end" }); + expect(await parked).toMatchObject({ prefix: Path.from("a"), kind: "offline" }); await tick(); expect(consumer.synced.peek()).toBe(false); void consumer.next(); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 30175fed0c8..731a2cdaa2d 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -975,7 +975,7 @@ export class Reload { } /** - * Pump `upstream` into `producer` for the rest of `effect`'s run: its events, then an `end` + * Pump `upstream` into `producer` for the rest of `effect`'s run: its events, then an `offline` * for each route still active once it closes or the run ends, so nothing clings to a source * that went away. Synced follows the upstream's, and owes those retractions too. * @@ -990,7 +990,8 @@ export function pumpAnnounced(effect: Effect, upstream: Announce.Routes, produce if (producer.closed.peek() !== undefined || active.size === 0) return; // Behind first, so a reader parked on the stream takes these as owed, not as news. producer.behind(); - for (const entry of active.values()) producer.append({ ...entry, kind: "end" }); + // The source went away, which says nothing of whether the origin ended them. + for (const entry of active.values()) producer.append({ ...entry, kind: "offline" }); active.clear(); }; @@ -1010,7 +1011,7 @@ export function pumpAnnounced(effect: Effect, upstream: Announce.Routes, produce for (;;) { const entry = await effect.race(upstream.next()); if (!entry || !live || producer.closed.peek() !== undefined) break; - if (entry.kind === "end") active.delete(entry.prefix); + if (entry.kind === "end" || entry.kind === "offline") active.delete(entry.prefix); else active.set(entry.prefix, entry); producer.append(entry); } diff --git a/js/net/src/hop.ts b/js/net/src/hop.ts index aab560e6b4e..940e7579c31 100644 --- a/js/net/src/hop.ts +++ b/js/net/src/hop.ts @@ -104,6 +104,13 @@ export interface RouteInfo { readonly instance?: Instance.Valid; /** What pulling content via this route costs; lower wins. */ readonly cost: Cost; + /** + * Whether a usable path reaches the route's origin. An unavailable route is held: the + * network lost every path to it but may recover one, so the announcement stays. It ranks + * with a draining route, below every available one, and requests through it may fail until + * it recovers: request again once it turns available. + */ + readonly available: boolean; } /** @@ -116,6 +123,7 @@ export function routeInfo(route: WireRoute): RouteInfo { return { instance: route.instance, cost: route.cost, + available: route.available ?? true, }; } @@ -155,6 +163,8 @@ export interface WireRoute { * preference state, never a saturated cost. Local: never forwarded, and readers never see it. */ readonly draining?: boolean; + /** See {@link RouteInfo.available}; unset means available. */ + readonly available?: boolean; } /** Constructors and helpers for {@link WireRoute}. @internal */ @@ -175,6 +185,7 @@ export const WireRoute = { cost: cost > max ? max : cost, auth: route.auth, draining: route.draining, + available: route.available, }; }, }; @@ -192,7 +203,8 @@ export function isAnonymous(route: WireRoute): boolean { } /** - * Whether two routes name the same instance, epoch, hop chain, cost, auth, and drain state. + * Whether two routes name the same instance, epoch, hop chain, cost, auth, drain state, and + * availability. * * @internal */ @@ -205,6 +217,7 @@ export function routesEqual(a: WireRoute | undefined, b: WireRoute | undefined): a.cost === b.cost && a.auth === b.auth && (a.draining ?? false) === (b.draining ?? false) && + (a.available ?? true) === (b.available ?? true) && a.hops.length === b.hops.length && a.hops.every((hop, i) => hop === b.hops[i]) ); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index bc014747b3c..d68bb06816b 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -1309,12 +1309,13 @@ export class Epochs { } /** - * Prefer a route that is not draining (a peer that sent GOAWAY ranks below every other), then - * identified publishers, then lower static cost. + * Prefer a route that is neither draining (a peer that sent GOAWAY) nor unavailable (held), + * which rank below every other, then identified publishers, then lower static cost. */ export function compareRoutes(a: WireRoute, b: WireRoute): number { + const last = (route: WireRoute) => Number((route.draining ?? false) || route.available === false); return ( - Number(a.draining ?? false) - Number(b.draining ?? false) || + last(a) - last(b) || Number(isAnonymous(a)) - Number(isAnonymous(b)) || (a.cost < b.cost ? -1 : a.cost > b.cost ? 1 : 0) ); diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 0940c9c6ade..6a8535450ce 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -444,7 +444,7 @@ test("a remote entry resolves by path and retracts on dispose", async () => { expect(await announced.next()).toMatchObject({ prefix: path, kind: "start", route: WireRoute.default }); dispose(); - expect(await announced.next()).toMatchObject({ prefix: path, kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: path, kind: "offline" }); expect(wireOf(consumer).routes(path)).toBe(false); announced.close(); @@ -452,6 +452,24 @@ test("a remote entry resolves by path and retracts on dispose", async () => { origin.close(); }); +test("a received route its announcer ends ends, and one its session loses goes offline", async () => { + const origin = new Producer(); + const announced = origin.consume().routes(); + + const ended = wireOf(origin).receive(Path.from("ended"), WireRoute.default); + const lost = wireOf(origin).receive(Path.from("lost"), WireRoute.default); + expect(await announced.next()).toMatchObject({ prefix: Path.from("ended"), kind: "start" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("lost"), kind: "start" }); + + wireOf(ended).end(); + expect(await announced.next()).toMatchObject({ prefix: Path.from("ended"), kind: "end" }); + lost.close(); + expect(await announced.next()).toMatchObject({ prefix: Path.from("lost"), kind: "offline" }); + + announced.close(); + origin.close(); +}); + test("announced keeps a broader covering route at its prefix", async () => { const origin = new Producer(); const consumer = origin.consume(); @@ -465,7 +483,7 @@ test("announced keeps a broader covering route at its prefix", async () => { expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "start" }); dispose(); - expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("room"), kind: "offline" }); announced.close(); upstream.close(); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index cec0e001414..76705917448 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -288,11 +288,18 @@ interface RouteEntry { readonly route: Signal; readonly originated: boolean; readonly server?: ServeState; + /** + * Set as a received route leaves because its session lost it, rather than its announcer + * ending it: readers then see it go offline, not end. + */ + lost: boolean; } /** One advertisement at a prefix. `exact` marks an announced local broadcast, which is only its own path. */ interface Candidate extends Advertised { readonly exact: boolean; + /** Whether the route left because its session lost it; see {@link RouteEntry.lost}. */ + readonly lost?: () => boolean; } /** @@ -495,6 +502,8 @@ interface Handoff { interface Presented extends Advertised { readonly captures: Path.Pattern[] | undefined; + /** See {@link Candidate.lost}. */ + readonly lost?: () => boolean; } /** A table mutation invalidates the shared route snapshot before its async notification. */ @@ -577,6 +586,7 @@ class OriginState { route: entry.route.peek(), claim: entry.claim, exact: false, + lost: () => entry.lost, }); } if (list.length > 0) out.set(path, list); @@ -1549,6 +1559,7 @@ export class Producer implements Table { announced, change: (update) => change(WireRoute.normalize({ ...info, ...update })), close: release, + end: release, info: () => (announced() ? this.#state.advertisedLocal.peek()?.get(path) : undefined), update: (next) => { const named = WireRoute.normalize(next); @@ -1635,6 +1646,7 @@ export class Producer implements Table { route: new Signal(route), originated, server, + lost: false, }; server.onReject = (path, err) => this.#state.refuse(path, entry, err); @@ -1683,6 +1695,11 @@ export class Producer implements Table { this.#state.servers.delete(server); remove(() => server.close()); }); + // A received route its handle closes was lost with its session, unless its announcer ended it. + const close = (lost: boolean) => { + if (!ended) entry.lost = lost; + release(); + }; // A route made after its origin (or this handle) closed is born retracted. if (this.#borrowed?.closed || !this.#state.routes.peek()) { ended = true; @@ -1736,7 +1753,8 @@ export class Producer implements Table { }, announced: () => inserted && this.#state.routes.peek() !== undefined, change: (changed) => update(WireRoute.normalize({ ...entry.route.peek(), ...changed })), - close: release, + close: () => close(!originated), + end: () => close(false), info: () => (inserted && this.#state.routes.peek() ? entry.route.peek() : undefined), update: (next) => { // A route naming no instance keeps the one it serves. @@ -2526,6 +2544,7 @@ export class Consumer implements Reader { // forwards to, see the route as it is. route: entry.route.draining === undefined ? entry.route : { ...entry.route, draining: undefined }, captures: scopeCaptures(scope.pattern, path), + lost: entry.lost, }); } return next; @@ -2556,7 +2575,7 @@ export class Consumer implements Reader { producer.append({ prefix: path, captures: snap.captures, - kind: "end", + kind: snap.lost?.() ? "offline" : "end", route: snap.route, }); } @@ -2724,6 +2743,8 @@ interface RouteControl { announced(): boolean; change(update: Partial>): void; close(): void; + /** Close as its announcer ending it; {@link close} on a received route means its session lost it. */ + end(): void; info(): WireRoute | undefined; update(info: Partial): void; } @@ -2735,6 +2756,7 @@ function registerControl(handle: Route | Claim, control: RouteControl): void { return control.info(); }, update: (info) => control.update(info), + end: () => control.end(), }); } diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index ff81466452c..c9213747bb9 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -87,6 +87,12 @@ export interface Route { * Announces a hidden route. */ update(info: Partial): void; + /** + * Close the route as its announcer withdrawing it, so readers see it end. Closing the handle + * of a route a session received instead means the session lost it, and readers see it go + * offline. + */ + end(): void; } /** The protocol-facing operations behind an origin consumer. */ diff --git a/js/room/src/room.ts b/js/room/src/room.ts index 700d4df4cd3..8176805a5c3 100644 --- a/js/room/src/room.ts +++ b/js/room/src/room.ts @@ -95,9 +95,10 @@ export class Room { const local = this.identity.peek(); if (local && parsed.identity === local) continue; - if (update.kind !== "absent") { + // A held path keeps its participant; only presence changes membership. + if (update.kind === "present") { this.#add(parsed.identity, parsed.kind, Moq.Path.from(covered)); - } else { + } else if (update.kind === "absent" || update.kind === "offline") { this.#remove(parsed.identity, parsed.kind); } } diff --git a/js/watch/src/broadcast.ts b/js/watch/src/broadcast.ts index a71ab564d54..a237ab2a67c 100644 --- a/js/watch/src/broadcast.ts +++ b/js/watch/src/broadcast.ts @@ -60,8 +60,8 @@ function watchCovered( for (;;) { const entry = await announced.next(); if (!entry) break; - if (entry.kind === "absent") active.delete(entry.prefix); - else active.add(entry.prefix); + if (entry.kind === "absent" || entry.kind === "offline") active.delete(entry.prefix); + else if (entry.kind === "present") active.add(entry.prefix); covered.set([...active].some((prefix) => Path.hasPrefix(prefix, path))); } } catch { diff --git a/js/watch/src/echo.test.ts b/js/watch/src/echo.test.ts index e80b3509954..bebba72e0de 100644 --- a/js/watch/src/echo.test.ts +++ b/js/watch/src/echo.test.ts @@ -31,8 +31,8 @@ function announced(origin: Origin.Producer): Set { for (;;) { const entry = await listing.next(); if (!entry) break; - if (entry.kind === "absent") paths.delete(entry.prefix); - else paths.add(entry.prefix); + if (entry.kind === "absent" || entry.kind === "offline") paths.delete(entry.prefix); + else if (entry.kind === "present") paths.add(entry.prefix); } })(); return paths; diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 6f780bd3f15..d0ff88e9032 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -6110,7 +6110,7 @@ mod session_tests { let mut announced = announced(&relay.origin); session.abort(moq_net::Error::App(1)); await_event(&mut announced, |event| { - matches!(event, moq_net::origin::RouteEvent::End(_)) + matches!(event, moq_net::origin::RouteEvent::Offline(_)) }); let mut new = Publisher::new(&relay.origin, Default::default(), video_rendition()); diff --git a/rs/moq-relay/tests/integration/cluster_idle.rs b/rs/moq-relay/tests/integration/cluster_idle.rs index b1f4822a5e0..42bde05cf31 100644 --- a/rs/moq-relay/tests/integration/cluster_idle.rs +++ b/rs/moq-relay/tests/integration/cluster_idle.rs @@ -133,7 +133,8 @@ async fn silent_peer_detected() { kio::clock::timeout(DETECT, async { loop { let event = announced.next().await.expect("origin closed"); - if let moq_net::origin::RouteEvent::End(update) = event + // A silent peer is lost, not withdrawn: its route goes offline. + if let moq_net::origin::RouteEvent::Offline(update) = event && update.prefix.as_str() == "peer" { break; From 8ab1667f57f6edc8848fa3b34eaf14e89cecd8da Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 07:45:19 -0800 Subject: [PATCH 04/11] wip: bindings, docs, quest for announcement availability Co-Authored-By: Claude Opus 5.5 --- dart/moq/lib/src/aliases.dart | 21 ++- dart/moq/lib/src/client.dart | 2 +- dart/moq_ffi/lib/src/moq.dart | 156 +++++++++++++++++- doc/concept/identity.md | 9 +- doc/lib/go/index.md | 2 +- doc/lib/kt/index.md | 3 +- doc/lib/py/index.md | 2 +- doc/lib/rs/moq-net.md | 2 +- go/wrapper/README.md | 2 + go/wrapper/origin.go | 6 +- go/wrapper/types.go | 15 +- kt/README.md | 3 +- .../kotlin/dev/moq/Aliases.kt | 22 ++- .../jvmAndAndroidMain/kotlin/dev/moq/Moq.kt | 2 +- py/moq-rs/README.md | 4 +- py/moq-rs/examples/announced.py | 2 +- py/moq-rs/moq/__init__.py | 6 + py/moq-rs/moq/origin.py | 21 ++- quest/m2/cluster-routing/routes/README.md | 1 - .../m2/cluster-routing/routes/availability.md | 26 --- quest/m2/cluster-routing/routes/js.md | 3 +- rs/moq-net/src/model/announced.rs | 32 ++++ swift/Sources/Moq/Aliases.swift | 8 +- swift/Sources/Moq/Origin.swift | 5 +- 24 files changed, 288 insertions(+), 67 deletions(-) delete mode 100644 quest/m2/cluster-routing/routes/availability.md diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 6c4e3aacafa..77bb94beced 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -73,19 +73,32 @@ typedef RouteInfo = MoqRouteInfo; /// A route over a prefix: its origin-relative path, required wildcard captures (empty without wildcards), and route metadata. typedef Announce = MoqAnnounce; -/// What an [AnnounceConsumer] yields: [AnnounceEventPresent] or -/// [AnnounceEventAbsent]. +/// What an [AnnounceConsumer] yields: [AnnounceEventPresent], +/// [AnnounceEventUnavailable], [AnnounceEventAvailable], [AnnounceEventAbsent], +/// or [AnnounceEventOffline]. /// /// Another publisher instance taking a path over keeps it present: hold a /// [Following] to see the broadcast change. typedef AnnounceEvent = MoqAnnounceEvent; -/// A route now covers the path, which was absent. +/// A route now covers the path, which was absent; it may arrive unavailable. typedef AnnounceEventPresent = PresentMoqAnnounceEvent; -/// No route covers the path any more; carries its last route. +/// The present path lost every usable path to its origin but may recover one: +/// it stays present while requests for it may fail. +typedef AnnounceEventUnavailable = UnavailableMoqAnnounceEvent; + +/// The present path recovered a usable path: request it again. +typedef AnnounceEventAvailable = AvailableMoqAnnounceEvent; + +/// The origin withdrew the path; carries its last route. typedef AnnounceEventAbsent = AbsentMoqAnnounceEvent; +/// This side let go of the path, which its origin may still serve: the session +/// it arrived on closed, or the network lost every path to it. Carries its last +/// route. +typedef AnnounceEventOffline = OfflineMoqAnnounceEvent; + /// The write side of a broadcast: publish tracks into it. typedef BroadcastProducer = MoqBroadcastProducer; diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 77533a88a00..eeb4271a5a2 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -222,7 +222,7 @@ final class Moq { MoqRoute route(String path, {RouteOptions? options}) => session.publish().route(path: path, options: options ?? RouteOptions()); - /// List the paths matching [options] as each becomes present or absent; + /// List the paths matching [options] as each becomes present, absent, or offline, or changes availability; /// prefixes stay relative to the origin. /// /// Listen to `announced(...).updates()` for a [Stream] of [AnnounceEvent] diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index a8dba5dcbb4..c011034ad1e 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -8,7 +8,9 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; + import "package:ffi/ffi.dart"; + import "uniffi_runtime.dart"; export "uniffi_runtime.dart"; @@ -3009,7 +3011,12 @@ class FfiConverterMoqOriginConfig { class MoqRouteInfo { final int cost; final String instance; - MoqRouteInfo({required this.cost, required this.instance}); + final bool available; + MoqRouteInfo({ + required this.cost, + required this.instance, + required this.available, + }); } class FfiConverterMoqRouteInfo { @@ -3029,8 +3036,13 @@ class FfiConverterMoqRouteInfo { ); final instance = instance_lifted.value; new_offset += instance_lifted.bytesRead; + final available_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final available = available_lifted.value; + new_offset += available_lifted.bytesRead; return LiftRetVal( - MoqRouteInfo(cost: cost, instance: instance), + MoqRouteInfo(cost: cost, instance: instance, available: available), new_offset - buf.offsetInBytes, ); } @@ -3039,6 +3051,7 @@ class FfiConverterMoqRouteInfo { final total_length = FfiConverterUInt64.allocationSize(value.cost) + FfiConverterString.allocationSize(value.instance) + + FfiConverterBool.allocationSize(value.available) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -3055,12 +3068,17 @@ class FfiConverterMoqRouteInfo { value.instance, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterBool.write( + value.available, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } static int allocationSize(MoqRouteInfo value) { return FfiConverterUInt64.allocationSize(value.cost) + FfiConverterString.allocationSize(value.instance) + + FfiConverterBool.allocationSize(value.available) + 0; } } @@ -8440,11 +8458,29 @@ class FfiConverterMoqAnnounceEvent { lifted.bytesRead - subview.offsetInBytes + 4, ); case 2: + final lifted = UnavailableMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + case 3: + final lifted = AvailableMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); + case 4: final lifted = AbsentMoqAnnounceEvent.read(subview); return LiftRetVal( lifted.value, lifted.bytesRead - subview.offsetInBytes + 4, ); + case 5: + final lifted = OfflineMoqAnnounceEvent.read(subview); + return LiftRetVal( + lifted.value, + lifted.bytesRead - subview.offsetInBytes + 4, + ); default: throw UniffiInternalError( UniffiInternalError.unexpectedEnumCase, @@ -8504,6 +8540,82 @@ class PresentMoqAnnounceEvent extends MoqAnnounceEvent { } } +class UnavailableMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + UnavailableMoqAnnounceEvent(MoqAnnounce this.announce); + UnavailableMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(UnavailableMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 2); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +class AvailableMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + AvailableMoqAnnounceEvent(MoqAnnounce this.announce); + AvailableMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(AvailableMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 3); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + class AbsentMoqAnnounceEvent extends MoqAnnounceEvent { final MoqAnnounce announce; AbsentMoqAnnounceEvent(MoqAnnounce this.announce); @@ -8532,7 +8644,45 @@ class AbsentMoqAnnounceEvent extends MoqAnnounceEvent { @override int write(Uint8List buf) { - buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 2); + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 4); + int new_offset = buf.offsetInBytes + 4; + new_offset += FfiConverterMoqAnnounce.write( + announce, + Uint8List.view(buf.buffer, new_offset), + ); + return new_offset; + } +} + +class OfflineMoqAnnounceEvent extends MoqAnnounceEvent { + final MoqAnnounce announce; + OfflineMoqAnnounceEvent(MoqAnnounce this.announce); + OfflineMoqAnnounceEvent._(MoqAnnounce this.announce); + static LiftRetVal read(Uint8List buf) { + int new_offset = buf.offsetInBytes; + final announce_lifted = FfiConverterMoqAnnounce.read( + Uint8List.view(buf.buffer, new_offset), + ); + final announce = announce_lifted.value; + new_offset += announce_lifted.bytesRead; + return LiftRetVal(OfflineMoqAnnounceEvent._(announce), new_offset); + } + + @override + RustBuffer lower() { + final buf = Uint8List(allocationSize()); + write(buf); + return toRustBuffer(buf); + } + + @override + int allocationSize() { + return FfiConverterMoqAnnounce.allocationSize(announce) + 4; + } + + @override + int write(Uint8List buf) { + buf.buffer.asByteData(buf.offsetInBytes).setInt32(0, 5); int new_offset = buf.offsetInBytes + 4; new_offset += FfiConverterMoqAnnounce.write( announce, diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 9dac288466a..205e3ccbd39 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -134,9 +134,12 @@ goes offline; JavaScript `origin.follow(path)`, whose `active` is undefined between instances; the moq-ffi bindings `follow(path)`, whose `available()` returns null once the path goes offline; and C `moq_origin_follow`, whose callback reports `MOQ_FOLLOW_EVENT_OFFLINE`. -An announcement listing (`announced()`) only reports a path becoming present or -absent: an instance change keeps it present, so a reader that needs one holds a -follower. `@moq/watch`, `moq play`, and `moqsrc` follow this way, and +An announcement listing (`announced()`) reports a path becoming present, +unavailable while the network holds its route without a path to it, available +again once it recovers one, and finally absent once the origin withdraws it or +offline once this side lets go of it, which says nothing of whether the origin +still serves it. An instance change keeps it present, so a reader that needs +one holds a follower. `@moq/watch`, `moq play`, and `moqsrc` follow this way, and `moq export ts --stitch` can splice a replacement as a marked program switch. A request failing is never the signal to restart; the follower handing out a new broadcast is. Nor is a track's error kind: a track withdrawn diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 52ad1805195..8f3a4e95c1c 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -70,7 +70,7 @@ for event, err := range announced.All(ctx) { // handed each new run, with Available returning nil while the path is offline. present, ok := event.(moq.AnnounceEventPresent) if !ok { - continue // AnnounceEventAbsent + continue // absent, offline, or a change in availability } ann := present.Announce // Prefix stays origin-relative; Captures is a list, empty without wildcards. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index c509aa5d7e2..369105923ba 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -58,7 +58,8 @@ Moq.connect("https://relay.example.com", ClientConfig(tls = ClientTls(roots = li // each new run, or null once it goes offline. val announce = when (event) { is AnnounceEventPresent -> event.announce - is AnnounceEventAbsent -> return@collect + // Absent, offline, or a change in availability. + else -> return@collect } // Prefixes stay origin-relative; captures is a list of wildcard matches, empty without wildcards. println(announce.captures) diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 50a3b436545..51ca45f9b3c 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -50,7 +50,7 @@ async def main(): # A path stays present across publisher restarts: hold client.follow(path) to # be handed each new run, or None once it goes offline. if not isinstance(event, moq.AnnounceEventPresent): - continue # AnnounceEventAbsent + continue # absent, offline, or a change in availability announcement = event.announce print(announcement.captures) # list of wildcard matches, empty without wildcards broadcast = await client.request(announcement.prefix) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index c01fa09c7fa..5a4f8b13b42 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -19,7 +19,7 @@ What you use it for, beyond what the concept page already describes: - **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()` and `cost()`; an application sets only `RouteOptions`. The hop chain a relay uses for loop detection stays internal. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. - **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). -- **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. +- **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths as present, then unavailable or available as a held route loses and recovers its path, and finally absent once the origin withdraws one or offline once this side lets go of it, which says nothing of whether the origin still serves it. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-webrtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. - **Relay by opting into proxying.** On moq-lite-07 (`moq-lite-07-wip`, opt-in) a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)` to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` is a server, a client, or both: its first dial announces a hidden client endpoint at `/.client/`, and its first accept a hidden server endpoint at `/.server/`, both used only to route sessions, so grants decide which roles it may take. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(server)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(server, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 50ef6ae8710..173e0bb1c48 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -55,6 +55,8 @@ for event, err := range announced.All(ctx) { // Prefix stays origin-relative; Captures reports wildcard matches. fmt.Println("broadcast online", event.Announce.Prefix) case moq.AnnounceEventAbsent: + fmt.Println("broadcast ended", event.Announce.Prefix) + case moq.AnnounceEventOffline: fmt.Println("broadcast offline", event.Announce.Prefix) } } diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index 44de3326386..7a62a0a9b0b 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -236,7 +236,7 @@ type AnnounceOptions struct { } // Announced lists the paths under a literal prefix matching an optional pattern filter, as each -// becomes present or absent. To see another publisher instance take a path over, use Follow. +// becomes present, absent, or offline, or changes availability. To see another publisher instance take a path over, use Follow. func (o *OriginConsumer) Announced(options AnnounceOptions) (*AnnounceConsumer, error) { inner, err := o.inner.Announced(ffi.MoqAnnounceConfig{ Prefix: options.Prefix, @@ -275,12 +275,12 @@ func (o *OriginConsumer) Request(ctx context.Context, path string, instance *str return &BroadcastConsumer{inner: inner}, nil } -// AnnounceConsumer is a listing of paths, each present or absent. +// AnnounceConsumer is a listing of paths, each present, absent, or offline. type AnnounceConsumer struct { inner *ffi.MoqAnnounceConsumer } -// Next returns the next path becoming present or absent, or (nil, nil) when the stream ends. +// Next returns the next change to a path, or (nil, nil) when the stream ends. func (a *AnnounceConsumer) Next(ctx context.Context) (AnnounceEvent, error) { res, err := bridge.Call(ctx, nil, a.inner.Next) if err != nil || res == nil { diff --git a/go/wrapper/types.go b/go/wrapper/types.go index e11162a5b81..d85696dde3f 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -46,13 +46,22 @@ type ( // resolve a path with [OriginConsumer.Request]. Captures is a required slice, // empty when the filter has no wildcards. Announce = ffi.MoqAnnounce - // AnnounceEvent is what an AnnounceConsumer yields: AnnounceEventPresent or AnnounceEventAbsent. + // AnnounceEvent is what an AnnounceConsumer yields: AnnounceEventPresent, AnnounceEventUnavailable, + // AnnounceEventAvailable, AnnounceEventAbsent, or AnnounceEventOffline. // Another publisher instance taking a path over keeps it present; see it with [OriginConsumer.Follow]. AnnounceEvent = ffi.MoqAnnounceEvent - // AnnounceEventPresent reports a route now covering a path that was absent. + // AnnounceEventPresent reports a route now covering a path that was absent; it may arrive unavailable. AnnounceEventPresent = ffi.MoqAnnounceEventPresent - // AnnounceEventAbsent reports that no route covers a path any more, carrying its last route. + // AnnounceEventUnavailable reports that a present path lost every usable path to its origin but may + // recover one: it stays present while requests for it may fail. + AnnounceEventUnavailable = ffi.MoqAnnounceEventUnavailable + // AnnounceEventAvailable reports that a present path recovered a usable path: request it again. + AnnounceEventAvailable = ffi.MoqAnnounceEventAvailable + // AnnounceEventAbsent reports that the origin withdrew a path, carrying its last route. AnnounceEventAbsent = ffi.MoqAnnounceEventAbsent + // AnnounceEventOffline reports that this side let go of a path, carrying its last route: the session + // it arrived on closed, or the network lost every path to it, so its origin may still serve it. + AnnounceEventOffline = ffi.MoqAnnounceEventOffline // Range is a span of a track to deliver: from Start up to End, or onward when End is nil. // A relative start is the first frame of its group, a relative end includes its whole // group, and an absolute end is exclusive. diff --git a/kt/README.md b/kt/README.md index 10556290dc8..e04f91e1e78 100644 --- a/kt/README.md +++ b/kt/README.md @@ -35,7 +35,8 @@ Moq.connect("https://relay.example.com").use { moq -> // restarts; `moq.follow(path)` hands out each new run, or null once it goes offline. val announce = when (event) { is AnnounceEventPresent -> event.announce - is AnnounceEventAbsent -> return@collect + // Absent, offline, or a change in availability. + else -> return@collect } // Prefix stays origin-relative; captures reports what * matched. println("got broadcast ${announce.prefix}") diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index c9a302908e2..58ae441ac48 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -95,7 +95,8 @@ typealias Following = uniffi.moq.MoqFollowing typealias Announce = uniffi.moq.MoqAnnounce /** - * What an [AnnounceConsumer] yields: [AnnounceEventPresent] or [AnnounceEventAbsent]. + * What an [AnnounceConsumer] yields: [AnnounceEventPresent], [AnnounceEventUnavailable], + * [AnnounceEventAvailable], [AnnounceEventAbsent], or [AnnounceEventOffline]. * * Another publisher instance taking a path over keeps it present: hold a [Following] to see the * broadcast change. @@ -104,12 +105,27 @@ typealias AnnounceEvent = uniffi.moq.MoqAnnounceEvent // Kotlin cannot reach a sealed class's subtypes through its typealias, so each // variant gets its own. -/** A route now covers the path, which was absent. */ +/** A route now covers the path, which was absent; it may arrive unavailable. */ typealias AnnounceEventPresent = uniffi.moq.MoqAnnounceEvent.Present -/** No route covers the path any more; carries its last route. */ +/** + * The present path lost every usable path to its origin but may recover one: it stays present + * while requests for it may fail. + */ +typealias AnnounceEventUnavailable = uniffi.moq.MoqAnnounceEvent.Unavailable + +/** The present path recovered a usable path: request it again. */ +typealias AnnounceEventAvailable = uniffi.moq.MoqAnnounceEvent.Available + +/** The origin withdrew the path; carries its last route. */ typealias AnnounceEventAbsent = uniffi.moq.MoqAnnounceEvent.Absent +/** + * This side let go of the path, which its origin may still serve: the session it arrived on + * closed, or the network lost every path to it. Carries its last route. + */ +typealias AnnounceEventOffline = uniffi.moq.MoqAnnounceEvent.Offline + // Broadcast / track / group producers and consumers. /** The write side of a broadcast: publish tracks into it. */ typealias BroadcastProducer = uniffi.moq.MoqBroadcastProducer diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt index c7d738c2eb4..e561c421210 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt @@ -27,7 +27,7 @@ internal constructor( session.publish().route(path, options) /** - * List the paths matching [config] as each becomes present or absent; prefixes stay relative to + * List the paths matching [config] as each becomes present, absent, or offline, or changes availability; prefixes stay relative to * the origin. Collect `announced(config).updates()` for a [Flow] of [AnnounceEvent] that * cancels the handle when collection ends. A new publisher instance keeps its path present, so * use [follow] to see it. diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 8fe08fbc52b..5a7202a11b5 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -30,7 +30,7 @@ async def main(): async with moq.connect("https://cdn.moq.dev/anon") as client: async for event in client.announced(): if not isinstance(event, moq.AnnounceEventPresent): - continue # AnnounceEventAbsent + continue # absent, offline, or a change in availability # A route covers a prefix and carries no broadcast, so resolve the path. broadcast = await client.request(event.announce.prefix) catalog = await moq.media.catalog(broadcast) @@ -295,7 +295,7 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect - `.announce()`, `.withdraw()`, `.set_cost(cost)`. Show, hide, or re-price it in place. - `await .requested() → BroadcastRequest | None`, or async iteration. Call `.serve(consumer)` to answer with held content under the claim's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the claim's instance until the job ends and then moves the claim to a fresh one (unless either names an instance: a named claim lends none, and a named job keeps its own), or `.reject(code)` to fail the requester. - **`OriginConsumer`**. Discover broadcasts. - - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent` or `AnnounceEventAbsent`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards + - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent`, `AnnounceEventUnavailable`, `AnnounceEventAvailable`, `AnnounceEventAbsent`, or `AnnounceEventOffline`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards - `.follow(path, instance=None) → Following` (awaitable, waits until something serves the path, then returns each stream that takes over, or `None` once the path goes offline; an `instance` follows only that stream) - `.request(path, instance=None) → BroadcastConsumer` (awaitable; owns discovery and resolves one stream, or raises once synced without a route; an `instance` refuses any other stream) diff --git a/py/moq-rs/examples/announced.py b/py/moq-rs/examples/announced.py index 77dc6617c28..451cfef48af 100644 --- a/py/moq-rs/examples/announced.py +++ b/py/moq-rs/examples/announced.py @@ -16,7 +16,7 @@ async def run(url: str, prefix: str, insecure: bool) -> None: async for event in client.announced(prefix): if isinstance(event, moq.AnnounceEventPresent): print(f" + {event.announce.prefix}") - elif isinstance(event, moq.AnnounceEventAbsent): + elif isinstance(event, (moq.AnnounceEventAbsent, moq.AnnounceEventOffline)): print(f" - {event.announce.prefix}") diff --git a/py/moq-rs/moq/__init__.py b/py/moq-rs/moq/__init__.py index f467273ee76..de0c789ed03 100644 --- a/py/moq-rs/moq/__init__.py +++ b/py/moq-rs/moq/__init__.py @@ -20,7 +20,10 @@ AnnounceConsumer, AnnounceEvent, AnnounceEventAbsent, + AnnounceEventAvailable, + AnnounceEventOffline, AnnounceEventPresent, + AnnounceEventUnavailable, BroadcastRequest, Claim, Following, @@ -103,7 +106,10 @@ "AnnounceConsumer", "AnnounceEvent", "AnnounceEventAbsent", + "AnnounceEventAvailable", + "AnnounceEventOffline", "AnnounceEventPresent", + "AnnounceEventUnavailable", "AudioCodec", "AudioEncoderInput", "AudioSampleFormat", diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index 5cdb90182ff..a4b8c19ea3f 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -32,18 +32,31 @@ """ AnnounceEvent = MoqAnnounceEvent -"""What :class:`AnnounceConsumer` yields: :data:`AnnounceEventPresent` or -:data:`AnnounceEventAbsent`, each carrying an :data:`Announce` as ``announce``. +"""What :class:`AnnounceConsumer` yields: :data:`AnnounceEventPresent`, +:data:`AnnounceEventUnavailable`, :data:`AnnounceEventAvailable`, +:data:`AnnounceEventAbsent`, or :data:`AnnounceEventOffline`, each carrying an +:data:`Announce` as ``announce``. Another publisher instance taking a path over keeps it present; hold :meth:`OriginConsumer.follow` to see the broadcast change. """ AnnounceEventPresent = MoqAnnounceEvent.PRESENT -"""A route now covers the path, which was absent.""" +"""A route now covers the path, which was absent; it may arrive unavailable.""" + +AnnounceEventUnavailable = MoqAnnounceEvent.UNAVAILABLE +"""The present path lost every usable path to its origin but may recover one: +it stays present while requests for it may fail.""" + +AnnounceEventAvailable = MoqAnnounceEvent.AVAILABLE +"""The present path recovered a usable path: request it again.""" AnnounceEventAbsent = MoqAnnounceEvent.ABSENT -"""No route covers the path any more; carries its last route.""" +"""The origin withdrew the path; carries its last route.""" + +AnnounceEventOffline = MoqAnnounceEvent.OFFLINE +"""This side let go of the path, which its origin may still serve: the session it +arrived on closed, or the network lost every path to it. Carries its last route.""" class AnnounceConsumer(_Stream[MoqAnnounceEvent]): diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index cac32fc49f2..19e6aa22d5b 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -538,7 +538,6 @@ benchmarks. ## Required - [A relay folds received routes incrementally](/quest/m2/cluster-routing/routes/relay-receive.md) - a received route batch costs the claims it changes, not the origin's size -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks - [Firefox tabs prove their route key](/quest/m2/cluster-routing/routes/browser-proof.md) - a current Firefox tab opens a lite-07 session with a route-key proof the relay verifies, once neqo's exporter follows the draft diff --git a/quest/m2/cluster-routing/routes/availability.md b/quest/m2/cluster-routing/routes/availability.md deleted file mode 100644 index 746565f2b58..00000000000 --- a/quest/m2/cluster-routing/routes/availability.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] Announcement availability - -## Goal - -An announcement carries availability, and letting go is a distinct offline -event, separate from the origin's END, in Rust and in `js/net`'s -announcement API (decision 3 of the epic's handoff proof). A held route -shows its claims unavailable; a let-go shows them offline; only the origin -ends them. Bindings mirror Rust. - -## Plan - -- Decided 2026-10-16: an announcement carries availability, and letting go - is a distinct offline event, separate from the origin's END. Rejected: - presence alone, which gives a reader nothing to re-request on when a held - route recovers; let-go reported as END, as today; and availability with - let-go kept as END, which hides whether the origin ended or the network - lost it. -- Build on [Listing and follow](https://github.com/moq-dev/moq/pull/7678)'s - listing and its offline report rather than adding event kinds beside it. -- JS gets the API shape here, fed by what `js/net` knows today; the - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) feeds it from the wire. - -## Related - -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index cd01f10c22e..088e2bc4072 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -16,7 +16,7 @@ Decided 2026-10-27 (maintainer), from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s `origin.ts` a detached request stays on its own route instead of following the cheapest one; fix it with the route-selection rework here. -- Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). +- Feed the availability API from the wire: set `WireRoute.available` from ROUTE_HOLD, and close a route the route layer lets go of with `close()` so readers see it offline, keeping `end()` for a certified withdrawal. - Rust/JS interop covers the route messages, the floor marker, and the echo. - Decode the temporary lite-06 compatibility chain that lite-07 announcements carry ([mixed-version ring](/quest/m1/mixed-version-ring.md)). JS translates nothing, so it keeps no chain. - Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, @@ -42,7 +42,6 @@ the cheapest one; fix it with the route-selection rework here. ## Required -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - the API this feeds - [Mixed-version ring](/quest/m1/mixed-version-ring.md) - defines the compatibility chain field this decodes ## Related diff --git a/rs/moq-net/src/model/announced.rs b/rs/moq-net/src/model/announced.rs index e401b11d793..6f6ee5275b7 100644 --- a/rs/moq-net/src/model/announced.rs +++ b/rs/moq-net/src/model/announced.rs @@ -230,4 +230,36 @@ mod tests { drop(second); assert_eq!(listed(&mut announced).await, "absent live"); } + + /// A held route keeps its path present, reported unavailable and available as it flips; a + /// session losing a route takes its path offline, while a peer's withdrawal ends it. + #[kio::test] + async fn availability_flips_and_offline_is_not_absent() { + let origin = Hop::new(1).unwrap().produce(); + let mut announced = origin.consume().announced(); + let instance = Instance::random(); + let held = RouteInfo::new().with_instance(instance.clone(), 0).with_available(false); + + let lost = origin.forward("lost", held.clone()).unwrap(); + assert_eq!(listed(&mut announced).await, "present lost"); + lost.update(held.clone().with_cost(Cost::new(3))).unwrap(); + quiet(&mut announced).await; + lost.update(held.clone().with_available(true)).unwrap(); + assert_eq!(listed(&mut announced).await, "available lost"); + lost.update(held.clone()).unwrap(); + assert_eq!(listed(&mut announced).await, "unavailable lost"); + drop(lost); + assert_eq!(listed(&mut announced).await, "offline lost"); + + let ended = origin.forward("ended", held.with_available(true)).unwrap(); + assert_eq!(listed(&mut announced).await, "present ended"); + ended.withdrawn(); + assert_eq!(listed(&mut announced).await, "absent ended"); + + // A route published here ends with its handle. + let local = origin.route("local", route(&instance)).unwrap(); + assert_eq!(listed(&mut announced).await, "present local"); + drop(local); + assert_eq!(listed(&mut announced).await, "absent local"); + } } diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index dac3ec4d3ca..5485cf8ee73 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -58,9 +58,11 @@ public typealias RouteInfo = MoqFFI.MoqRouteInfo /// wildcard matched (`captures`, a required list, empty without wildcards), and the `route` serving it. /// Resolve a path with `OriginConsumer.request`. public typealias Announce = MoqFFI.MoqAnnounce -/// What an `AnnounceConsumer` yields: `.present` or `.absent`, each carrying an -/// `Announce`. Another publisher instance taking a path over keeps it present: -/// hold a `Following` to see the broadcast change. +/// What an `AnnounceConsumer` yields, each carrying an `Announce`: `.present` once a route +/// covers the path, `.unavailable` and `.available` as a present path loses and recovers a +/// usable path to its origin, `.absent` once the origin withdraws it, and `.offline` once this +/// side lets go of it, which its origin may still serve. Another publisher instance taking a +/// path over keeps it present: hold a `Following` to see the broadcast change. public typealias AnnounceEvent = MoqFFI.MoqAnnounceEvent /// Per-subscription delivery preferences: priority, delay, the ranges of the track to /// deliver, and group ordering. diff --git a/swift/Sources/Moq/Origin.swift b/swift/Sources/Moq/Origin.swift index 091804c8885..bbcffcbc5af 100644 --- a/swift/Sources/Moq/Origin.swift +++ b/swift/Sources/Moq/Origin.swift @@ -200,7 +200,7 @@ public final class OriginConsumer: Sendable { } /// List the paths under a literal prefix matching an optional pattern filter, as each - /// becomes present or absent. A new publisher instance keeps its path present, so use + /// becomes present, absent, or offline, or changes availability. A new publisher instance keeps its path present, so use /// `follow` to see it. Paths with a segment starting with `.` below the prefix are left /// out unless `hidden`. public func announced(prefix: String = "", filter: String? = nil, hidden: Bool = false) throws -> AnnounceConsumer { @@ -223,7 +223,8 @@ public final class OriginConsumer: Sendable { } } -/// A listing of paths as each becomes present or absent. Iterate directly: +/// A listing of paths as each becomes present, absent, or offline, or changes availability. +/// Iterate directly: /// `for try await event in announced { ... }`. /// The sequence ends when the origin closes; cancelling the consuming task /// cancels the subscription. From 4fc189e01f55d02fc382191e1536ca9abe6bff0b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 07:52:22 -0800 Subject: [PATCH 05/11] wip: format, drill retarget, room fixture Co-Authored-By: Claude Opus 5.5 --- js/net/api/index.api.md | 3 ++- js/room/src/room.test.ts | 2 +- .../kotlin/dev/moq/Aliases.kt | 8 +++--- .../jvmAndAndroidMain/kotlin/dev/moq/Moq.kt | 8 +++--- rs/moq-net/src/ietf/publisher.rs | 4 ++- rs/moq-net/src/ietf/subscriber.rs | 3 ++- rs/moq-net/src/model/announced.rs | 4 ++- .../tests/integration/lazy_discovery.rs | 4 ++- rs/moq-net/tests/integration/mount.rs | 6 ++++- rs/moq-relay/src/cluster.rs | 4 ++- rs/moq-relay/src/internal.rs | 4 ++- rs/moq-stats/src/aggregate.rs | 8 ++++-- rs/moq-stats/src/consume.rs | 4 ++- rs/moq-stats/src/produce.rs | 4 ++- rs/moq-tokio/src/origin.rs | 4 ++- rs/moq-tokio/src/server.rs | 4 ++- rs/moq-tokio/tests/integration/broadcast.rs | 4 ++- rs/moq-webrtc/src/lib.rs | 4 ++- .../relay-withdraws-lost-publisher.patch | 27 +++++++------------ 19 files changed, 67 insertions(+), 42 deletions(-) diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 89a479f6323..3580bb69741 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -18,7 +18,7 @@ export namespace Announce { get synced(): Signals.Getter; } export type Event = { - kind: "present" | "absent"; + kind: "present" | "unavailable" | "available" | "absent" | "offline"; } & Announce.Announce; export interface Options { covering?: boolean; @@ -513,6 +513,7 @@ export namespace Origin { withdraw(): void; } export interface RouteInfo { + readonly available: boolean; readonly cost: Origin.Cost; readonly instance?: Instance.Valid; } diff --git a/js/room/src/room.test.ts b/js/room/src/room.test.ts index 5f1938b1201..f0db998052c 100644 --- a/js/room/src/room.test.ts +++ b/js/room/src/room.test.ts @@ -21,7 +21,7 @@ test("room restores the announce prefix and reconciles local identity changes", prefix: Net.Path.from("room-a/bob/camera.hang"), captures: [Net.Path.Pattern.literal(Net.Path.from("bob/camera.hang"))], kind: "present", - route: { cost: 0n }, + route: { cost: 0n, available: true }, }; const stream = { next: async () => { diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 58ae441ac48..a0572e4e81d 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -109,8 +109,8 @@ typealias AnnounceEvent = uniffi.moq.MoqAnnounceEvent typealias AnnounceEventPresent = uniffi.moq.MoqAnnounceEvent.Present /** - * The present path lost every usable path to its origin but may recover one: it stays present - * while requests for it may fail. + * The present path lost every usable path to its origin but may recover one: it stays present while + * requests for it may fail. */ typealias AnnounceEventUnavailable = uniffi.moq.MoqAnnounceEvent.Unavailable @@ -121,8 +121,8 @@ typealias AnnounceEventAvailable = uniffi.moq.MoqAnnounceEvent.Available typealias AnnounceEventAbsent = uniffi.moq.MoqAnnounceEvent.Absent /** - * This side let go of the path, which its origin may still serve: the session it arrived on - * closed, or the network lost every path to it. Carries its last route. + * This side let go of the path, which its origin may still serve: the session it arrived on closed, + * or the network lost every path to it. Carries its last route. */ typealias AnnounceEventOffline = uniffi.moq.MoqAnnounceEvent.Offline diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt index e561c421210..69e5679f354 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Moq.kt @@ -27,10 +27,10 @@ internal constructor( session.publish().route(path, options) /** - * List the paths matching [config] as each becomes present, absent, or offline, or changes availability; prefixes stay relative to - * the origin. Collect `announced(config).updates()` for a [Flow] of [AnnounceEvent] that - * cancels the handle when collection ends. A new publisher instance keeps its path present, so - * use [follow] to see it. + * List the paths matching [config] as each becomes present, absent, or offline, or changes + * availability; prefixes stay relative to the origin. Collect `announced(config).updates()` for + * a [Flow] of [AnnounceEvent] that cancels the handle when collection ends. A new publisher + * instance keeps its path present, so use [follow] to see it. */ fun announced(config: AnnounceConfig = AnnounceConfig()): AnnounceConsumer = session.consume().announced(config) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 5be6e004123..0dfbf6c1ddd 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -5902,7 +5902,9 @@ impl ForwardNamespaces { (update, true, false) } crate::origin::RouteEvent::Restart(update) => (update, true, true), - crate::origin::RouteEvent::End(update) | crate::origin::RouteEvent::Offline(update) => (update, false, false), + crate::origin::RouteEvent::End(update) | crate::origin::RouteEvent::Offline(update) => { + (update, false, false) + } }))); } if self.retry.poll(waiter).is_ready() { diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index b4a9bc06ea8..05ab7d78a94 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2295,7 +2295,8 @@ impl ServePublishNamespace { let token = live.grant.is_some(); // Only the peer's own withdrawal ends the route; a failed stream loses it. let ended = matches!(lived, Ok(Ok(Withdraw::Done))); - self.this.release_announce(&live.path, &mut live.attached, token, ended)?; + self.this + .release_announce(&live.path, &mut live.attached, token, ended)?; let withdraw = match lived { Ok(withdraw) => withdraw?, diff --git a/rs/moq-net/src/model/announced.rs b/rs/moq-net/src/model/announced.rs index 6f6ee5275b7..44cec01223f 100644 --- a/rs/moq-net/src/model/announced.rs +++ b/rs/moq-net/src/model/announced.rs @@ -238,7 +238,9 @@ mod tests { let origin = Hop::new(1).unwrap().produce(); let mut announced = origin.consume().announced(); let instance = Instance::random(); - let held = RouteInfo::new().with_instance(instance.clone(), 0).with_available(false); + let held = RouteInfo::new() + .with_instance(instance.clone(), 0) + .with_available(false); let lost = origin.forward("lost", held.clone()).unwrap(); assert_eq!(listed(&mut announced).await, "present lost"); diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index 252393cdcbe..07d38a34c7e 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -112,7 +112,9 @@ async fn synced(announced: &mut origin::Routes) -> Vec { origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { seen.push(announce.prefix.to_string()) } - origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => seen.retain(|prefix| *prefix != announce.prefix.as_str()), + origin::RouteEvent::End(announce) | origin::RouteEvent::Offline(announce) => { + seen.retain(|prefix| *prefix != announce.prefix.as_str()) + } origin::RouteEvent::Update(_) => {} } } diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs index 50e19dd6894..0b9da6ed082 100644 --- a/rs/moq-net/tests/integration/mount.rs +++ b/rs/moq-net/tests/integration/mount.rs @@ -158,7 +158,11 @@ async fn relayed(version: &str) { drop(upstream); kio::clock::sleep(Duration::from_millis(500)).await; - assert_eq!(drain(&mut routes).await, lost(version, &["own", "shared/cam"]), "{version}"); + assert_eq!( + drain(&mut routes).await, + lost(version, &["own", "shared/cam"]), + "{version}" + ); } macro_rules! mounted_tests { diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 573208b060a..1b7a533ce4a 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2650,7 +2650,9 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => { + Some((route, false)) + } } } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 8ec4195b570..5f722e68148 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1333,7 +1333,9 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => { + Some((route, false)) + } } } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 3296d779fab..8f576cb5028 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -816,7 +816,9 @@ impl Merged { let instance = update.route.instance().clone(); changed |= self.apply_announce(&update, instance, true) } - Poll::Ready(Some(moq_net::origin::RouteEvent::End(update) | moq_net::origin::RouteEvent::Offline(update))) => { + Poll::Ready(Some( + moq_net::origin::RouteEvent::End(update) | moq_net::origin::RouteEvent::Offline(update), + )) => { let instance = update.route.instance().clone(); changed |= self.apply_announce(&update, instance, false) } @@ -1145,7 +1147,9 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => { + Some((route, false)) + } } } diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index b15f1c6b155..32f72633d01 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -178,7 +178,9 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => { + Some((route, false)) + } } } diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 4eb62f742b2..df0e54946c6 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1675,7 +1675,9 @@ mod tests { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), - moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => Some((route, false)), + moq_net::origin::RouteEvent::End(route) | moq_net::origin::RouteEvent::Offline(route) => { + Some((route, false)) + } } } diff --git a/rs/moq-tokio/src/origin.rs b/rs/moq-tokio/src/origin.rs index f2f9c6aaa20..3761922dae5 100644 --- a/rs/moq-tokio/src/origin.rs +++ b/rs/moq-tokio/src/origin.rs @@ -44,7 +44,9 @@ mod tests { loop { return match announced.next().await? { moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, }; } diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 7b7addbe1ef..4b6f55fe3c1 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -1965,7 +1965,9 @@ mod tests { loop { return match announced.next().await? { moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, }; } diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index cbcd83563ad..de4e9f46217 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -818,7 +818,9 @@ async fn next_route(routes: &mut moq_net::origin::Routes) -> (moq_net::announce: .expect("announce timeout") .expect("origin closed"); match event { - moq_net::origin::RouteEvent::End(announce) | moq_net::origin::RouteEvent::Offline(announce) => (announce, false), + moq_net::origin::RouteEvent::End(announce) | moq_net::origin::RouteEvent::Offline(announce) => { + (announce, false) + } moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Update(announce) | moq_net::origin::RouteEvent::Restart(announce) => (announce, true), diff --git a/rs/moq-webrtc/src/lib.rs b/rs/moq-webrtc/src/lib.rs index fa47bf71eb3..b460e2a6639 100644 --- a/rs/moq-webrtc/src/lib.rs +++ b/rs/moq-webrtc/src/lib.rs @@ -78,7 +78,9 @@ mod tests { loop { return match announced.next().await? { moq_net::announce::Event::Present(route) => Some((route, true)), - moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => Some((route, false)), + moq_net::announce::Event::Absent(route) | moq_net::announce::Event::Offline(route) => { + Some((route, false)) + } moq_net::announce::Event::Available(_) | moq_net::announce::Event::Unavailable(_) => continue, }; } diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b67b6381510..5482bf08055 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -7,9 +7,10 @@ # name stays announced with nothing behind it. Explicit withdrawals still consume # the handle, so only the implicit retraction on session loss is removed. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs +index 45654a5656..8af799f04c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -8817,7 +8817,7 @@ impl Announced { +@@ -9749,7 +9749,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +19,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -8901,7 +8901,7 @@ fn mint_sources(subscriber: &Subscriber, entry +@@ -9831,7 +9831,7 @@ fn mint_sources(subscriber: &Subscriber, entry let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +28,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -8929,7 +8929,8 @@ struct AnnouncedRoute { +@@ -9859,7 +9859,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +38,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Queues this route on its prefix's ready set when its request queue wakes. -@@ -8949,7 +8950,7 @@ impl AnnouncedRoute { +@@ -9877,7 +9878,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -47,21 +48,13 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. waker: std::task::Waker::from(wake.clone()), wake, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs +index 20fe66dd38..91675a0b9b 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -354,7 +354,7 @@ impl Routed { - self.previous = previous; - self.outputs(subscriber); - for (_, served) in std::mem::take(&mut self.previous) { -- served.route.dynamic.withdrawn(); -+ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); - } - Ok(()) - } -@@ -398,7 +398,7 @@ impl Routed { - } => self.serve(subscriber, id, named, claim, served), - node::Output::Unserve { id, claim } => { - if let Some(served) = self.served.remove(&(id, claim)) { +@@ -402,7 +402,7 @@ impl Routed { + if let Some(served) = self.served.remove(&(id, claim)) + && ended + { - served.route.dynamic.withdrawn(); + std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); } From b4b158dddeefcf8aa7fc6cb7d9ba375f2c82dd73 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 08:36:05 -0800 Subject: [PATCH 06/11] chore(quest): keep announcement availability open, waiting on scoped certification Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/README.md | 1 + .../m2/cluster-routing/routes/availability.md | 54 +++++++++++++++++++ quest/m2/cluster-routing/routes/js.md | 3 +- 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 quest/m2/cluster-routing/routes/availability.md diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 19e6aa22d5b..cac32fc49f2 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -538,6 +538,7 @@ benchmarks. ## Required - [A relay folds received routes incrementally](/quest/m2/cluster-routing/routes/relay-receive.md) - a received route batch costs the claims it changes, not the origin's size +- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks - [Firefox tabs prove their route key](/quest/m2/cluster-routing/routes/browser-proof.md) - a current Firefox tab opens a lite-07 session with a route-key proof the relay verifies, once neqo's exporter follows the draft diff --git a/quest/m2/cluster-routing/routes/availability.md b/quest/m2/cluster-routing/routes/availability.md new file mode 100644 index 00000000000..7a935b2431b --- /dev/null +++ b/quest/m2/cluster-routing/routes/availability.md @@ -0,0 +1,54 @@ +# [M] Announcement availability + +## Goal + +An announcement carries availability, and letting go is a distinct offline +event, separate from the origin's END, in Rust and in `js/net`'s +announcement API (decision 3 of the epic's handoff proof). A held route +shows its claims unavailable; a let-go shows them offline; only the origin +ends them. Bindings mirror Rust. + +## Plan + +- Decided 2026-10-16: an announcement carries availability, and letting go + is a distinct offline event, separate from the origin's END. Rejected: + presence alone, which gives a reader nothing to re-request on when a held + route recovers; let-go reported as END, as today; and availability with + let-go kept as END, which hides whether the origin ended or the network + lost it. +- Build on [Listing and follow](https://github.com/moq-dev/moq/pull/7678)'s + listing and its offline report rather than adding event kinds beside it. +- JS gets the API shape here, fed by what `js/net` knows today; the + [js/net route layer](/quest/m2/cluster-routing/routes/js.md) feeds it from the wire. +- Implemented in draft [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375): + `RouteEvent::Offline`, listing `Unavailable`/`Available`/`Offline` beside + `Present`/`Absent`, and `RouteInfo::available()`, in Rust, `js/net`, FFI, the + C ABI, and every binding. A session drop, an auth limit loss, or a narrower + interest goes offline; a certified withdrawal ends. +- Remaining: a cleanly finished broadcast still reaches a reader through a relay + as offline over token-scoped lite-07 sessions, so every `finish X -> Y` lane + of `just test interop --all` fails. The router ends only a claim a floor + certifies, and a scoped session's floor certifies nothing: its grant or limit + (such as `finish-…/**`) does not cover the requested `""`. Certifying "the + prefix as the grant shows it" was tried and reverted as unsound: a receiver + need not mirror the grant, and a grant shrink drops a claim at the same + revision (`auth::lite_07::tokens_union_and_withdrawing_one_shrinks_it`). + +## Waiting + +- Maintainer: choose how a token-scoped lite-07 session certifies withdrawals + so a clean finish ENDs across a relay. It changes the draft's floor rules + (epic decisions 10 and 12), so an agent should not pick it; done when the + choice is recorded in this Plan. Options: + 1. (Recommended) The subscriber narrows its ANNOUNCE_REQUEST to its scope's + single region (its limit and subscribe grant meet there) and reopens it + when that region changes. Floors keep their exact whole-prefix meaning; no + wire change, one SHOULD in the draft. A multi-region scope still certifies + nothing. Best as its own prerequisite PR. + 2. Floors carry regions on the wire (lite-07-wip), reversing decision 10. + 3. Land with the finish lanes accepting offline and do 1 later; readers then + cannot tell a finished broadcast from a lost one until it lands. + +## Related + +- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index 088e2bc4072..cd01f10c22e 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -16,7 +16,7 @@ Decided 2026-10-27 (maintainer), from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s `origin.ts` a detached request stays on its own route instead of following the cheapest one; fix it with the route-selection rework here. -- Feed the availability API from the wire: set `WireRoute.available` from ROUTE_HOLD, and close a route the route layer lets go of with `close()` so readers see it offline, keeping `end()` for a certified withdrawal. +- Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). - Rust/JS interop covers the route messages, the floor marker, and the echo. - Decode the temporary lite-06 compatibility chain that lite-07 announcements carry ([mixed-version ring](/quest/m1/mixed-version-ring.md)). JS translates nothing, so it keeps no chain. - Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, @@ -42,6 +42,7 @@ the cheapest one; fix it with the route-selection rework here. ## Required +- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - the API this feeds - [Mixed-version ring](/quest/m1/mixed-version-ring.md) - defines the compatibility chain field this decodes ## Related From 5e0c25fbe427f128608f759bd65552c2f05f7e83 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:36:22 -0800 Subject: [PATCH 07/11] feat(net): a scoped lite-07 subscriber requests its one region A token-scoped session asked its peer for everything, which its scope did not cover, so the peer's floors certified nothing and a clean finish reached readers through a relay as offline. The subscriber now narrows each ANNOUNCE_REQUEST to the one region where its limit and subscribe grant meet, and reopens it when that region changes, keeping the old request until the new one is synced. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 2 + rs/moq-cli/tests/integration/rtsp.rs | 5 +- rs/moq-net/src/auth.rs | 47 ++++++++++++ rs/moq-net/src/lite/subscriber.rs | 107 +++++++++++++++++++++++++-- rs/moq-net/src/model/origin.rs | 3 +- rs/moq-net/tests/integration/auth.rs | 95 ++++++++++++++++++++++++ 6 files changed, 251 insertions(+), 8 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 7234ec57f4f..864a07cce79 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -450,6 +450,7 @@ The publisher sends a ROUTE_START before any claim under it, and SYNCED only onc A floor names a Revision and certifies that the stream's claims for the route, across the whole requested prefix, are the origin's claims at that Revision as the request presents them (see [Announce](#announce)). It certifies only what its own request shows: a stream whose ANNOUNCE_REQUEST did not set `Hidden` certifies nothing about hidden claims. A publisher that does not know the route's claims across the whole requested prefix at one Revision, such as a relay that knows them only beneath part of it, still closes each batch, with a floor that certifies nothing. +A subscriber that may receive claims only beneath one prefix, because its [scope](#auth-stream) or the publisher's confines the stream there, SHOULD request that prefix rather than one above it, so the publisher can certify the request. When that prefix changes, it SHOULD open a request for the new one and end the old one once the new one is SYNCED. The subscriber applies a batch whole, once its floor arrives. A receiver that hears one origin's claims on several streams merges them claim by claim, each at the prefix its stream presents it: of the streams whose floor certifies the claim, the one with the highest Revision decides whether it exists, and a claim no floor certifies exists while some stream carries it. @@ -2194,6 +2195,7 @@ The `Message Length` describes the payload size on the wire. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. - A route presents only its most specific claim covering the requested prefix, moved to the next covering claim when it is withdrawn rather than ended. +- A subscriber whose scope confines a stream to one prefix SHOULD request that prefix, and replace the request when the prefix changes. - A route goes on a stream only while it has a claim the request presents, and ends after the batch withdrawing its last claim. It starts on that stream again only once the subscriber echoed the ROUTE_END. - Added `Node ID` to TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS: the origin node the first hop selected, followed by every later hop. - Replaced Warm and Cold Route Cost with the origin's Announce Cost, unchanged in transit. ANNOUNCE_UPDATE carries only the Announce Cost. diff --git a/rs/moq-cli/tests/integration/rtsp.rs b/rs/moq-cli/tests/integration/rtsp.rs index 776656e73da..49b21f5c656 100644 --- a/rs/moq-cli/tests/integration/rtsp.rs +++ b/rs/moq-cli/tests/integration/rtsp.rs @@ -69,7 +69,10 @@ async fn next_broadcast( .. } => break, moq_net::origin::RouteEvent { - kind: moq_net::origin::RouteKind::End | moq_net::origin::RouteKind::Offline | moq_net::origin::RouteKind::Update, + kind: + moq_net::origin::RouteKind::End + | moq_net::origin::RouteKind::Offline + | moq_net::origin::RouteKind::Update, .. } => {} } diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index e32d7537168..09128722f8f 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -591,6 +591,35 @@ impl Permit { Some(crate::model::interest_prefixes(limit)) } + /// The one prefix the grant and the limit together confine this direction to, or + /// `None` when they confine it to several or to none. A request for it, rather than + /// one above it, is one the peer's floors can certify. + #[cfg(feature = "async")] + pub(crate) fn region(&self) -> Option { + let heads = |patterns: &Option| patterns.as_ref().map(crate::model::interest_prefixes); + let regions = match (heads(&self.granted), heads(&self.limit)) { + (None, None) => return None, + (Some(regions), None) | (None, Some(regions)) => regions, + (Some(granted), Some(limit)) => { + let mut meet: Vec = (granted.iter()) + .flat_map(|a| limit.iter().map(move |b| (a, b))) + .filter_map(|(a, b)| match () { + _ if a.has_prefix(b) => Some(a.clone()), + _ if b.has_prefix(a) => Some(b.clone()), + _ => None, + }) + .collect(); + meet.sort(); + meet.dedup(); + meet + } + }; + match regions.as_slice() { + [region] => Some(region.clone()), + _ => None, + } + } + /// Whether both set the same limit on the peer, whatever either was granted. #[cfg(feature = "async")] pub(crate) fn same_limit(&self, other: &Self) -> bool { @@ -3029,6 +3058,24 @@ mod tests { assert_eq!(everything.regions(), None, "a root grant confines nothing"); } + /// The region is where the grant and the limit meet, and only when that is one prefix. + #[cfg(feature = "async")] + #[test] + fn the_region_is_where_grant_and_limit_meet() { + let permit = |granted: Option<&[&str]>, limit: Option<&[&str]>| Permit { + granted: granted.map(patterns), + limit: limit.map(patterns), + }; + let path = crate::Path::from_static; + assert_eq!(permit(None, None).region(), None); + assert_eq!(permit(Some(&["a"]), None).region(), Some(path("a"))); + assert_eq!(permit(None, Some(&["a/b"])).region(), Some(path("a/b"))); + assert_eq!(permit(Some(&["a"]), Some(&["a/b", "c"])).region(), Some(path("a/b"))); + assert_eq!(permit(Some(&[""]), Some(&["a"])).region(), Some(path("a"))); + assert_eq!(permit(Some(&["a", "b"]), None).region(), None, "two regions"); + assert_eq!(permit(Some(&["a"]), Some(&["b"])).region(), None, "they never meet"); + } + /// A limit replaces the last one, narrower or wider, and applies to a version /// without AUTH too. #[test] diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index de7a298feb4..0e5f73dedc2 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1007,6 +1007,8 @@ pub(super) struct SubscriberDriver { announces: kio::Tasks, /// Each open prefix's machine in `announces`. prefixes: HashMap, + /// Lite-07: the subscribe permit's epoch the requests were last narrowed at. + auth_epoch: u64, uni: UniAccept, /// PROBE feedback; finishes quietly when unsupported or given up on. bandwidth: Option, @@ -1029,6 +1031,7 @@ impl SubscriberDriver { discovery: crate::discovery::Discovery::new(&subscriber.origin, lazy, subscriber.runtime.clone()), announces: kio::Tasks::new(), prefixes: HashMap::new(), + auth_epoch: 0, cleanup: SubscriptionCleanup(subscriber.subscribes.clone(), subscriber.datagrams.clone()), uni: UniAccept::new(subscriber.clone(), early), bandwidth: Some(RecvBandwidth::new(subscriber.clone())), @@ -1076,6 +1079,25 @@ impl SubscriberDriver { } pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + // A new permit may narrow a request to another region: it reopens there, so a + // renewal that widens the scope is not lost. + if self.subscriber.version.has_routes() { + let auth = &self.subscriber.auth; + let mut changed = false; + while auth + .poll_permit(crate::auth::Direction::Subscribe, &mut self.auth_epoch, waiter) + .is_ready() + { + changed = true; + } + if changed { + for key in self.prefixes.values_mut() { + if let Some(machine) = self.announces.remove(*key) { + *key = self.announces.insert(machine.reopen()); + } + } + } + } loop { while let Poll::Ready(change) = self.discovery.poll(waiter) { match change { @@ -2099,12 +2121,20 @@ struct AnnouncePrefix { /// Its place in line for the peer's stream credit while its stream opens. open: crate::transport::open::Open, subscriber: Subscriber, + /// The prefix discovery wants, which keys the machine. prefix: Path, + /// The prefix on the wire: lite-07 narrows `prefix` to the region the session's + /// subscribe permit confines it to, so the peer's floors can certify it. + request: Path, state: PrefixState, /// Holds the origin's announce consumers under the prefix unsynced /// until the peer's initial set is in. Dropped at the boundary the version /// marks, or with the stream. snapshot: Option, + /// Lite-07: the request this one replaces, after the region changed. It keeps + /// serving until this one is synced, so the node lets go only of what the new + /// request no longer carries. + replacing: Option>, } impl kio::Task for AnnouncePrefix { @@ -2115,6 +2145,15 @@ impl kio::Task for AnnouncePrefix { } } +/// The prefix to request for `prefix`: the region the permit confines it to when +/// that lies beneath it, since a floor certifies only a request its scope covers whole. +fn narrow(prefix: &Path, region: Option) -> Path { + match region { + Some(region) if region.has_prefix(prefix) => region, + _ => prefix.clone(), + } +} + /// The announce-decode loop's state. struct PrefixRun { responder_origin: Option, @@ -2135,6 +2174,8 @@ struct PrefixRun { initial: u64, /// Lite-07: the route layer the stream feeds in place of `announced`. routed: Option, + /// Lite-07: SYNCED arrived. + synced: bool, } #[kio::machine(self.state: PrefixState)] @@ -2142,18 +2183,63 @@ impl AnnouncePrefix { /// `snapshot` was taken when the prefix was first wanted, so a consumer of the /// origin never reads as synced before the request even goes out. fn new(subscriber: Subscriber, prefix: Path, snapshot: crate::model::Snapshot) -> Self { + let request = Self::narrowed(&subscriber, &prefix); Self { open: Default::default(), subscriber, prefix, + request, state: PrefixState::Start, snapshot: Some(snapshot), + replacing: None, + } + } + + /// The prefix to put on the wire for `prefix`, as the session's permit stands now. + fn narrowed(subscriber: &Subscriber, prefix: &Path) -> Path { + match subscriber.version.has_routes() { + true => narrow( + prefix, + subscriber.auth.permit(crate::auth::Direction::Subscribe).region(), + ), + false => prefix.clone(), + } + } + + /// A request for the region the permit narrows the prefix to now, replacing this + /// one, or this one while its request still is that region. The replacement takes + /// over the initial set's snapshot if this one was still waiting on it. + fn reopen(mut self) -> Self { + let request = Self::narrowed(&self.subscriber, &self.prefix); + if request == self.request { + return self; + } + // A replacement never waits on another: the one before is let go at once. + self.replacing = None; + let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); + // One that never reached its loop has fed the node nothing to hand over. + let replacing = (matches!(self.state, PrefixState::Run { .. })).then(|| Box::new(self)); + Self { + open: Default::default(), + subscriber, + prefix, + request, + state: PrefixState::Start, + snapshot, + replacing, } } /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + if let Some(replacing) = &mut self.replacing + && let Poll::Ready(res) = replacing.poll(waiter) + { + // The peer finished the old request: nothing left to hand over. + self.replacing = None; + res?; + } self.poll_run(waiter) .map_err(|err| lite::error::control(&mut self.subscriber.session.clone(), err)) } @@ -2176,7 +2262,7 @@ impl AnnouncePrefix { // the origin, and each local reader opts in on its own // (`origin::Consumer::with_hidden`). stream.writer.buffer(&lite::AnnounceRequest { - prefix: self.prefix.clone(), + prefix: self.request.clone(), exclude_hop: self.subscriber.self_origin, hidden: true, })?; @@ -2229,10 +2315,16 @@ impl AnnouncePrefix { decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, initial, + synced: false, routed: match version.has_routes() { true => { let permit = self.subscriber.auth.permit(crate::auth::Direction::Subscribe); - Some(routed::Routed::new(&self.subscriber, &self.prefix, link_cost, &permit)?) + Some(routed::Routed::new( + &self.subscriber, + &self.request, + link_cost, + &permit, + )?) } false => None, }, @@ -2254,7 +2346,7 @@ impl AnnouncePrefix { })?; if let Some(init) = init { for suffix in init.suffixes { - let path = self.prefix.join(&suffix)?; + let path = self.request.join(&suffix)?; // Lite01/02 don't carry hop information; the broadcast starts with // an empty chain and an unpriced link. Stats are attributed in the // model when this enters the origin via `route`. @@ -2309,9 +2401,12 @@ impl AnnouncePrefix { if let Some(routed) = &mut run.routed { routed.outputs(&self.subscriber); } - if self.snapshot.take().is_none() { + if std::mem::replace(&mut run.synced, true) { return Poll::Ready(Err(Error::ProtocolViolation)); } + self.snapshot = None; + // This request now carries what the one it replaces did. + self.replacing = None; continue; } if run.initial > 0 { @@ -2322,13 +2417,13 @@ impl AnnouncePrefix { _ => return Poll::Ready(Err(Error::ProtocolViolation)), } } - self.subscriber.handle_announce(&self.prefix, announce, run)?; + self.subscriber.handle_announce(&self.request, announce, run)?; // Released only once the last counted route is in the origin. if run.initial == 0 && !self.subscriber.version.has_synced() { self.snapshot = None; } } - Poll::Ready(Ok(None)) if run.initial > 0 || self.snapshot.is_some() => { + Poll::Ready(Ok(None)) if run.initial > 0 || (version.has_synced() && !run.synced) => { return Poll::Ready(Err(Error::ProtocolViolation)); } Poll::Ready(Ok(None)) => { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 527d3022417..74122073cff 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -6956,7 +6956,8 @@ impl Requesting { loop { match discovery.poll_next(waiter) { Poll::Ready(Some(RouteEvent { - kind: RouteKind::End | RouteKind::Offline, .. + kind: RouteKind::End | RouteKind::Offline, + .. })) => self.routed = false, Poll::Ready(Some(RouteEvent { kind: RouteKind::Start | RouteKind::Restart | RouteKind::Update, diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index b7df76e792c..df5066921a3 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -121,6 +121,17 @@ async fn lite_07_opens_an_auth_stream_per_side() { .expect("timed out"); } +/// Floors certify withdrawals only on lite-07. +#[kio::test] +async fn lite_07_a_scoped_withdrawal_ends() { + a_scoped_withdrawal_ends(LITE_07).await +} + +#[kio::test] +async fn lite_07_a_scope_change_reopens_without_letting_go() { + a_scope_change_reopens_without_letting_go(LITE_07).await +} + #[kio::test] async fn lite_05_has_no_grant() { older_versions_have_no_grant("moq-lite-05").await @@ -811,6 +822,90 @@ async fn tokens_union_and_withdrawing_one_shrinks_it(version: &'static str) { .expect("timed out"); } +/// The route events `routes` delivered for `path` so far. +fn kinds(routes: &mut origin::Routes, path: &str) -> Vec { + let mut kinds = Vec::new(); + while let Some(event) = futures::FutureExt::now_or_never(routes.next()).flatten() { + if event.announce.prefix.as_str() == path { + kinds.push(event.kind); + } + } + kinds +} + +/// A relay that limits its publisher to one prefix asks it for that prefix alone, not +/// for everything, so the publisher's floors certify a withdrawal: it ends rather than +/// going offline. +async fn a_scoped_withdrawal_ends(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let mut routes = relay.consume().routes(); + let route = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + route.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); + + route.withdraw(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::End]); + }) + .await + .expect("timed out"); +} + +/// Widening the limit to a second prefix reopens the relay's request above both, and +/// narrowing it reopens the request at the first again, each without letting go of what +/// the limit kept: the old request serves until the new one is synced. +async fn a_scope_change_reopens_without_letting_go(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let mut routes = relay.consume().routes(); + let a = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + a.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::Start]); + + // Widened past what the first request asked for: the new one carries `b`. + pair.server.auth().authorize(&grant(&["a", "b"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + let b = publisher.route("b/y", moq_net::origin::RouteOptions::hidden()).unwrap(); + b.announce().unwrap(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "b/y"), [origin::RouteKind::Start]); + assert_eq!(kinds(&mut routes, "a/x"), [], "the widening let go of a/x"); + + // Narrowed back to `a`, where a withdrawal is certified again. + pair.server.auth().authorize(&grant(&["a"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [], "the narrowing let go of a/x"); + a.withdraw(); + kio::clock::sleep(Duration::from_secs(1)).await; + assert_eq!(kinds(&mut routes, "a/x"), [origin::RouteKind::End]); + }) + .await + .expect("timed out"); +} + /// An update replaces one token's grant and leaves the other alone. async fn an_update_replaces_one_tokens_grant(version: &'static str) { within(async { From c931b5d93e11bb48594a653a6c216eaa72acd9aa Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:18:33 -0800 Subject: [PATCH 08/11] chore(quest): finish announcement availability, record the scoped-request decision Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/README.md | 2 +- .../m2/cluster-routing/routes/availability.md | 54 ------------------- quest/m2/cluster-routing/routes/js.md | 3 +- 3 files changed, 2 insertions(+), 57 deletions(-) delete mode 100644 quest/m2/cluster-routing/routes/availability.md diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 26e66f7f560..93143c1f30c 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -28,6 +28,7 @@ claim whose withdrawal is already known. 13. ✅ Decided 2026-11-05 (maintainer): **ANNOUNCE_FLOOR encodes `Revision + 1`, with 0 meaning the floor certifies nothing,** the same offset as `Max Age`. Rust decodes it to `Option`. It replaces a Revision of 2^64-1 as the marker: a huge sentinel is a bad pattern and costs 9 bytes as a lite-07 varint. Rejected: a separate message; a boolean field. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. 15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293) implements it. +16. ✅ Decided 2026-11-09 (maintainer, /quest-unblock on [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)), asked how a token-scoped lite-07 session certifies withdrawals so a clean finish ENDs across a relay: ✅ **the subscriber narrows its ANNOUNCE_REQUEST to its scope's single region** (where its limit and its subscribe grant meet), and reopens the request when that region changes so a widening renewal isn't lost. Floors keep their exact whole-prefix meaning, with no wire change and one SHOULD in the lite-07 draft. A multi-region scope still certifies nothing. Rejected: floors carrying regions on the wire, which reverses item 10; landing with the finish lanes accepting offline. The route layer's core landed in @@ -537,7 +538,6 @@ benchmarks. ## Required -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - announcements report availability, and a let-go is offline rather than END, in Rust and JS - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) - `js/net` encodes, decodes, and resolves the route layer - [Route layer acceptance](/quest/m2/cluster-routing/routes/acceptance.md) - forwarding fixtures ported and the model deleted, incident fixtures, end-to-end checks, and benchmarks - [Firefox tabs prove their route key](/quest/m2/cluster-routing/routes/browser-proof.md) - a current Firefox tab opens a lite-07 session with a route-key proof the relay verifies, once neqo's exporter follows the draft diff --git a/quest/m2/cluster-routing/routes/availability.md b/quest/m2/cluster-routing/routes/availability.md deleted file mode 100644 index 7a935b2431b..00000000000 --- a/quest/m2/cluster-routing/routes/availability.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] Announcement availability - -## Goal - -An announcement carries availability, and letting go is a distinct offline -event, separate from the origin's END, in Rust and in `js/net`'s -announcement API (decision 3 of the epic's handoff proof). A held route -shows its claims unavailable; a let-go shows them offline; only the origin -ends them. Bindings mirror Rust. - -## Plan - -- Decided 2026-10-16: an announcement carries availability, and letting go - is a distinct offline event, separate from the origin's END. Rejected: - presence alone, which gives a reader nothing to re-request on when a held - route recovers; let-go reported as END, as today; and availability with - let-go kept as END, which hides whether the origin ended or the network - lost it. -- Build on [Listing and follow](https://github.com/moq-dev/moq/pull/7678)'s - listing and its offline report rather than adding event kinds beside it. -- JS gets the API shape here, fed by what `js/net` knows today; the - [js/net route layer](/quest/m2/cluster-routing/routes/js.md) feeds it from the wire. -- Implemented in draft [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375): - `RouteEvent::Offline`, listing `Unavailable`/`Available`/`Offline` beside - `Present`/`Absent`, and `RouteInfo::available()`, in Rust, `js/net`, FFI, the - C ABI, and every binding. A session drop, an auth limit loss, or a narrower - interest goes offline; a certified withdrawal ends. -- Remaining: a cleanly finished broadcast still reaches a reader through a relay - as offline over token-scoped lite-07 sessions, so every `finish X -> Y` lane - of `just test interop --all` fails. The router ends only a claim a floor - certifies, and a scoped session's floor certifies nothing: its grant or limit - (such as `finish-…/**`) does not cover the requested `""`. Certifying "the - prefix as the grant shows it" was tried and reverted as unsound: a receiver - need not mirror the grant, and a grant shrink drops a claim at the same - revision (`auth::lite_07::tokens_union_and_withdrawing_one_shrinks_it`). - -## Waiting - -- Maintainer: choose how a token-scoped lite-07 session certifies withdrawals - so a clean finish ENDs across a relay. It changes the draft's floor rules - (epic decisions 10 and 12), so an agent should not pick it; done when the - choice is recorded in this Plan. Options: - 1. (Recommended) The subscriber narrows its ANNOUNCE_REQUEST to its scope's - single region (its limit and subscribe grant meet there) and reopens it - when that region changes. Floors keep their exact whole-prefix meaning; no - wire change, one SHOULD in the draft. A multi-region scope still certifies - nothing. Best as its own prerequisite PR. - 2. Floors carry regions on the wire (lite-07-wip), reversing decision 10. - 3. Land with the finish lanes accepting offline and do 1 later; readers then - cannot tell a finished broadcast from a lost one until it lands. - -## Related - -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the handoff contract's held and let-go states diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index cd01f10c22e..da8370fc4b4 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -16,7 +16,7 @@ Decided 2026-10-27 (maintainer), from [moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947): in `js/net`'s `origin.ts` a detached request stays on its own route instead of following the cheapest one; fix it with the route-selection rework here. -- Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). +- Feed the availability API (`RouteInfo.available` and the `offline` route kind) from the route layer. - Rust/JS interop covers the route messages, the floor marker, and the echo. - Decode the temporary lite-06 compatibility chain that lite-07 announcements carry ([mixed-version ring](/quest/m1/mixed-version-ring.md)). JS translates nothing, so it keeps no chain. - Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, @@ -42,7 +42,6 @@ the cheapest one; fix it with the route-selection rework here. ## Required -- [Announcement availability](/quest/m2/cluster-routing/routes/availability.md) - the API this feeds - [Mixed-version ring](/quest/m1/mixed-version-ring.md) - defines the compatibility chain field this decodes ## Related From 3e48ea6ea3fceaa8998c313d7be05ed5de93a245 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:04:48 -0800 Subject: [PATCH 09/11] fix(net): a scope handover respects the announce cap; JS moq-transport losses go offline A replacement request takes slots for the claims its predecessor still holds, so it keeps the predecessor only while the session has room for both, and lets it go first otherwise. The JS moq-transport subscriber ends a namespace only on NAMESPACE_DONE or the peer finishing a PUBLISH_NAMESPACE, as Rust does; a reflected namespace or a request that went away goes offline. Co-Authored-By: Claude Opus 5.5 --- js/net/src/connection/close.test.ts | 4 +- js/net/src/ietf/peer-violations.test.ts | 2 +- js/net/src/ietf/subscriber.test.ts | 10 ++--- js/net/src/ietf/subscriber.ts | 27 ++++++++++---- rs/moq-net/src/lite/subscriber.rs | 11 +++++- rs/moq-net/src/lite/subscriber/routed.rs | 5 +++ rs/moq-net/tests/integration/auth.rs | 47 ++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 17 deletions(-) diff --git a/js/net/src/connection/close.test.ts b/js/net/src/connection/close.test.ts index b592496faad..8be3d80e421 100644 --- a/js/net/src/connection/close.test.ts +++ b/js/net/src/connection/close.test.ts @@ -79,7 +79,9 @@ for (const protocol of [Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18]) { const closing = server.close(); await fin.reached; expect(closed).toBe(false); - expect((await announced.next())?.kind).toBe("end"); + // lite-07 withdraws its claims; moq-transport only ends the stream, which + // withdraws nothing without a NAMESPACE_DONE. + expect((await announced.next())?.kind).toBe(protocol === Lite.ALPN_07_WIP ? "end" : "offline"); if (outcome === "timeout") { // bun's `rejects` blocks on a pending promise, so catch first and advance. const timedOut = closing.catch((err: unknown) => err); diff --git a/js/net/src/ietf/peer-violations.test.ts b/js/net/src/ietf/peer-violations.test.ts index 27b4e41a37a..28539a57aab 100644 --- a/js/net/src/ietf/peer-violations.test.ts +++ b/js/net/src/ietf/peer-violations.test.ts @@ -188,7 +188,7 @@ for (const [version, alpn] of versions) { await entry(stream, "a"); expect((await announced.next())?.kind).toBe("start"); await entry(stream, "a", true); - expect((await announced.next())?.kind).toBe("end"); + expect((await announced.next())?.kind).toBe("offline"); await entry(stream, "a"); expect((await announced.next())?.kind).toBe("start"); if (ending === "done") { diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index d3d5ce7f1ae..4c3ac4ffe05 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -281,10 +281,10 @@ test("a subscription that dies releases what it advertised", async () => { expect(await doomed.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "start" }); expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "start" }); - // The stream that advertised it goes away without a NAMESPACE_DONE. + // The stream that advertised it goes away without a NAMESPACE_DONE: lost, not withdrawn. streamA.writer.close(); - expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "end" }); + expect(await survivor.next()).toMatchObject({ prefix: Path.from("orphan"), kind: "offline" }); }); /** @@ -358,7 +358,7 @@ test("an inline NAMESPACE that starts looping back is retracted", async () => { expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "start" }); await inlineNamespace(subscription, Path.from("theirs"), { hops: [SELF, PEER], cost: 0n }); - expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "offline" }); }); /** @@ -513,7 +513,7 @@ test("a PUBLISH_NAMESPACE update that starts looping back is detached", async () VERSION, ); - expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "end" }); + expect(await announced.next()).toMatchObject({ prefix: Path.from("theirs"), kind: "offline" }); expect(await peer.reader.u53()).toBe(RequestOk.id); await RequestOk.decode(peer.reader, VERSION); @@ -1697,7 +1697,7 @@ test("a request overlapping one on its way out waits for the peer to end it", as expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "start" }); narrow.close(); const broad = subscriber.routes(Path.Pattern.subtree(Path.from("a"))); - expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "end" }); + expect(await watcher.next()).toMatchObject({ prefix: Path.from("a/b/x"), kind: "offline" }); // Draft 22 cancels with a reset, and the broader request waits on the peer's half. expect(stream.reader.done()).rejects.toMatchObject({ code: StreamCode.Cancel }); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 517f86a997c..e4cb6ac5d0a 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -446,7 +446,7 @@ export class Subscriber { /** * Drop advertisement `id` for a path, retracting it only once the last one goes. */ - #detachAnnounce(path: Path.Valid, id: number) { + #detachAnnounce(path: Path.Valid, id: number, ended: boolean) { const existing = this.#announced.get(path); const index = existing?.adverts.findIndex((advert) => advert.id === id) ?? -1; if (existing === undefined || index === -1) return; @@ -468,7 +468,13 @@ export class Subscriber { // A consumer that closed stays registered until its request is off the wire. if (!sees(filter, path) || consumer.closed.peek() !== undefined) continue; const scope = filter.scope; - consumer.append({ prefix: path, captures: scopeCaptures(scope, path), kind: "end", route: existing.route }); + // Only the peer's withdrawal ends the route; losing its stream or path lets it go. + consumer.append({ + prefix: path, + captures: scopeCaptures(scope, path), + kind: ended ? "end" : "offline", + route: existing.route, + }); } } @@ -505,13 +511,14 @@ export class Subscriber { let released = false; // The stream owns every advertisement it carried, so give them back however it ends: - // as the request starts leaving, a decode error, or the peer resetting it. Without + // as the request starts leaving, a decode error, or the peer resetting it. Closing the + // stream withdraws nothing, which NAMESPACE_DONE is for, so they go offline. Without // this each namespace keeps its count and the source never detaches, which would pin // the path for the session even after the other source withdrew. const release = () => { released = true; for (const [path, held] of live) { - if (held.id !== undefined) this.#detachAnnounce(path, held.id); + if (held.id !== undefined) this.#detachAnnounce(path, held.id, false); } live.clear(); }; @@ -633,7 +640,7 @@ export class Subscriber { // over: the path is gone even though the message says active. if (this.#reflected(params.cluster)) { console.debug(`dropping reflected namespace: broadcast=${path}`); - if (held?.id !== undefined) this.#detachAnnounce(path, held.id); + if (held?.id !== undefined) this.#detachAnnounce(path, held.id, false); live.set(path, {}); continue; } @@ -652,7 +659,7 @@ export class Subscriber { const held = live.get(path); live.delete(path); - if (held?.id !== undefined) this.#detachAnnounce(path, held.id); + if (held?.id !== undefined) this.#detachAnnounce(path, held.id, true); } else { throw new Error( `unexpected message on subscribe_namespace stream: 0x${msgType.toString(16)}`, @@ -1735,6 +1742,9 @@ export class Subscriber { // stream ends. this.#announceSlots.acquire(); let attached: number | undefined; + // Set once the peer ends the request, by FIN or reset, which withdraws the + // advertisement; a violation or a refused update here loses it instead. + let withdrawn = false; try { // Send OK first. This must complete before notifying consumers, @@ -1755,6 +1765,7 @@ export class Subscriber { if (framed === undefined) { // FIN cancels only on some drafts; on the rest the request lasts until reset. await request.ended; + withdrawn = true; break; } @@ -1802,7 +1813,7 @@ export class Subscriber { if (this.#reflected(held)) { if (attached !== undefined) { console.debug(`publish_namespace now loops back, detaching: broadcast=${path}`); - this.#detachAnnounce(path, attached); + this.#detachAnnounce(path, attached, false); attached = undefined; } } else if (attached === undefined) { @@ -1823,7 +1834,7 @@ export class Subscriber { // Give back exactly what was taken: a request that never got its OK out never // referenced the path, and retracting there would drop someone else's count. if (attached !== undefined) { - this.#detachAnnounce(path, attached); + this.#detachAnnounce(path, attached, withdrawn); } } } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 0e5f73dedc2..f28699d1692 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2217,8 +2217,15 @@ impl AnnouncePrefix { // A replacement never waits on another: the one before is let go at once. self.replacing = None; let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); - // One that never reached its loop has fed the node nothing to hand over. - let replacing = (matches!(self.state, PrefixState::Run { .. })).then(|| Box::new(self)); + // One that never reached its loop has fed the node nothing to hand over. The + // replacement takes slots for the same claims the old one holds, so without room + // for both, the old one is let go first rather than the peer running past its cap. + let held = match &self.state { + PrefixState::Run(step) => step.run.routed.as_ref().map(routed::Routed::held), + _ => None, + }; + let room = held.is_some_and(|held| self.subscriber.announces.available() >= held); + let replacing = room.then(|| Box::new(self)); Self { open: Default::default(), subscriber, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 20fe66dd385..84d12729cf9 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -302,6 +302,11 @@ impl Routed { Ok(()) } + /// How many of the session's announce slots the stream holds. + pub(super) fn held(&self) -> usize { + self.held.len() + self.echoing.len() + } + /// The peer sent a GOAWAY: its routes rank below every other. Idempotent. pub(super) fn goaway(&mut self, subscriber: &Subscriber) { if self.drained { diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index df5066921a3..c1dc455667a 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -132,6 +132,11 @@ async fn lite_07_a_scope_change_reopens_without_letting_go() { a_scope_change_reopens_without_letting_go(LITE_07).await } +#[kio::test] +async fn lite_07_a_scope_change_at_the_announce_cap_keeps_the_session() { + a_scope_change_at_the_announce_cap_keeps_the_session(LITE_07).await +} + #[kio::test] async fn lite_05_has_no_grant() { older_versions_have_no_grant("moq-lite-05").await @@ -462,6 +467,8 @@ struct Options { client_token: Option<&'static [u8]>, /// The handle the client builds its session on, when not its own. client_auth: Option, + /// The server's session limits, when not the defaults. + server_limits: Option, } struct Pair { @@ -512,6 +519,9 @@ async fn connect(opts: Options) -> Pair { if let Some(subscribe) = opts.server_subscribe { server = server.with_subscriber(subscribe); } + if let Some(limits) = opts.server_limits { + server = server.with_limits(limits); + } let observe = client_transport.clone(); let observe_server = server_transport.clone(); @@ -906,6 +916,43 @@ async fn a_scope_change_reopens_without_letting_go(version: &'static str) { .expect("timed out"); } +/// A session holding all the announce slots it may has no room to hear its peer's claims +/// twice, so a scope change lets the old request go before the new one takes over, +/// rather than the handover running the peer past its cap. +async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static str) { + within(async { + let publisher = produce_origin(2); + let relay = produce_origin(1); + let mut limits = moq_net::session::Limits::default(); + // One route and its two claims. + limits.announces = 3; + let pair = connect(Options { + version: Some(version), + client_publish: Some(publisher.clone()), + server_subscribe: Some(relay.clone()), + server_limits: Some(limits), + ..Default::default() + }) + .await; + pair.server.auth().authorize(&grant(&["a"], &[])); + + let x = publisher.route("a/x", moq_net::origin::RouteOptions::hidden()).unwrap(); + x.announce().unwrap(); + let y = publisher.route("a/y", moq_net::origin::RouteOptions::hidden()).unwrap(); + y.announce().unwrap(); + wait_announced(&relay.consume(), "a/y", true).await; + + pair.server.auth().authorize(&grant(&["a", "b"], &[])); + kio::clock::sleep(Duration::from_secs(1)).await; + wait_announced(&relay.consume(), "a/x", true).await; + wait_announced(&relay.consume(), "a/y", true).await; + assert_eq!(pair.server_transport.close_reason(), None); + assert_eq!(pair.client_transport.close_reason(), None); + }) + .await + .expect("timed out"); +} + /// An update replaces one token's grant and leaves the other alone. async fn an_update_replaces_one_tokens_grant(version: &'static str) { within(async { From 3320df4e1c7bef41d74de9fe57dd34ce4ed36180 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:20:47 -0800 Subject: [PATCH 10/11] fix(net): a replaced request gives its announce slots to its replacement A wider replacement can need more slots than its predecessor held, so budgeting a duplicate of the old set was not enough. The predecessor now gives back its slots at the handover: the replacement gets the whole cap, and what the old one keeps stays bounded by the cap it took it under. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 18 +++++++++++------- rs/moq-net/src/lite/subscriber/routed.rs | 9 ++++++--- rs/moq-net/tests/integration/auth.rs | 7 ++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index f28699d1692..76e637bd2cb 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2217,15 +2217,19 @@ impl AnnouncePrefix { // A replacement never waits on another: the one before is let go at once. self.replacing = None; let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); - // One that never reached its loop has fed the node nothing to hand over. The - // replacement takes slots for the same claims the old one holds, so without room - // for both, the old one is let go first rather than the peer running past its cap. - let held = match &self.state { - PrefixState::Run(step) => step.run.routed.as_ref().map(routed::Routed::held), + // One that never reached its loop has fed the node nothing to hand over. One that + // did gives back its announce slots, since the replacement takes slots for the same + // claims: the replacement gets the whole cap, and what the old one keeps was bounded + // by the cap it took it under. + let replacing = match &mut self.state { + PrefixState::Run(step) => { + if let Some(routed) = &mut step.run.routed { + routed.release_slots(); + } + Some(Box::new(self)) + } _ => None, }; - let room = held.is_some_and(|held| self.subscriber.announces.available() >= held); - let replacing = room.then(|| Box::new(self)); Self { open: Default::default(), subscriber, diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 84d12729cf9..deb98a85fbc 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -302,9 +302,12 @@ impl Routed { Ok(()) } - /// How many of the session's announce slots the stream holds. - pub(super) fn held(&self) -> usize { - self.held.len() + self.echoing.len() + /// Give back the session's announce slots for what the stream holds, while a request + /// replacing it takes the same claims over. What it holds stays as bounded as the cap + /// it took it under; what it takes from now on is counted again. + pub(super) fn release_slots(&mut self) { + self.held.clear(); + self.echoing.clear(); } /// The peer sent a GOAWAY: its routes rank below every other. Idempotent. diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index c1dc455667a..3e12f3569d8 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -917,7 +917,7 @@ async fn a_scope_change_reopens_without_letting_go(version: &'static str) { } /// A session holding all the announce slots it may has no room to hear its peer's claims -/// twice, so a scope change lets the old request go before the new one takes over, +/// twice, so the old request gives its slots to the new one during a scope change, /// rather than the handover running the peer past its cap. async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static str) { within(async { @@ -941,11 +941,12 @@ async fn a_scope_change_at_the_announce_cap_keeps_the_session(version: &'static let y = publisher.route("a/y", moq_net::origin::RouteOptions::hidden()).unwrap(); y.announce().unwrap(); wait_announced(&relay.consume(), "a/y", true).await; + let mut routes = relay.consume().routes(); + kinds(&mut routes, "a/x"); pair.server.auth().authorize(&grant(&["a", "b"], &[])); kio::clock::sleep(Duration::from_secs(1)).await; - wait_announced(&relay.consume(), "a/x", true).await; - wait_announced(&relay.consume(), "a/y", true).await; + assert_eq!(kinds(&mut routes, "a/x"), [], "the handover let go of a/x"); assert_eq!(pair.server_transport.close_reason(), None); assert_eq!(pair.client_transport.close_reason(), None); }) From c8902ab87491f923f442dbcb0c341447f4ac7c57 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:39:03 -0800 Subject: [PATCH 11/11] fix(net): a handed-over request keeps its claims on a budget of its own During a scoped lite-07 subscriber's ANNOUNCE_REQUEST handover, the old request moves what it holds onto a separate counter with the session's cap, so the session's count belongs to the replacement alone. A peer can hold at most twice the cap, during one handover. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/README.md | 1 + rs/moq-net/src/lite/subscriber.rs | 7 +- rs/moq-net/src/lite/subscriber/routed.rs | 103 ++++++++++++++++++++-- rs/moq-net/src/session.rs | 5 ++ 4 files changed, 106 insertions(+), 10 deletions(-) diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 93143c1f30c..ac0b1214c4d 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -29,6 +29,7 @@ claim whose withdrawal is already known. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. 15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [moq-dev/moq#8293](https://github.com/moq-dev/moq/pull/8293) implements it. 16. ✅ Decided 2026-11-09 (maintainer, /quest-unblock on [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)), asked how a token-scoped lite-07 session certifies withdrawals so a clean finish ENDs across a relay: ✅ **the subscriber narrows its ANNOUNCE_REQUEST to its scope's single region** (where its limit and its subscribe grant meet), and reopens the request when that region changes so a widening renewal isn't lost. Floors keep their exact whole-prefix meaning, with no wire change and one SHOULD in the lite-07 draft. A multi-region scope still certifies nothing. Rejected: floors carrying regions on the wire, which reverses item 10; landing with the finish lanes accepting offline. +17. ✅ Decided 2026-11-09 (maintainer, /quest-unblock on [moq-dev/moq#8375](https://github.com/moq-dev/moq/pull/8375)), asked how item 16's reopen counts the session's announce cap while the old request keeps serving until the new one is SYNCED: ✅ **the old request gets its own budget for the handover**, moving what it holds onto a separate counter with the same cap, so the session's count belongs wholly to the new request. A peer can make the session hold at most twice the cap, and only during one handover. Rejected: dropping the old request at once on every region change, which flickers claims offline; holding a client's first request until its credential is answered and dropping the old one on later renewals. The route layer's core landed in diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 8a6e1743e59..f7b0475482f 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2222,13 +2222,12 @@ impl AnnouncePrefix { self.replacing = None; let (subscriber, prefix, snapshot) = (self.subscriber.clone(), self.prefix.clone(), self.snapshot.take()); // One that never reached its loop has fed the node nothing to hand over. One that - // did gives back its announce slots, since the replacement takes slots for the same - // claims: the replacement gets the whole cap, and what the old one keeps was bounded - // by the cap it took it under. + // did moves its announce slots onto a budget of its own, since the replacement takes + // slots for the same claims: the replacement gets the session's whole cap. let replacing = match &mut self.state { PrefixState::Run(step) => { if let Some(routed) = &mut step.run.routed { - routed.release_slots(); + routed.hand_over(); } Some(Box::new(self)) } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index bfb20156997..46436177c4d 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -304,12 +304,16 @@ impl Routed { Ok(()) } - /// Give back the session's announce slots for what the stream holds, while a request - /// replacing it takes the same claims over. What it holds stays as bounded as the cap - /// it took it under; what it takes from now on is counted again. - pub(super) fn release_slots(&mut self) { - self.held.clear(); - self.echoing.clear(); + /// Move what the stream holds onto a budget of its own with the session's cap, while a + /// request replacing it takes the same claims over on the session's. The session's + /// count is then the replacement's alone, and a peer flooding this stream fails it + /// on its own budget: at most twice the cap, during one handover. + pub(super) fn hand_over(&mut self) { + let slots = self.slots.split(); + for slot in self.held.iter_mut().chain(&mut self.echoing) { + *slot = slots.hold(); + } + self.slots = slots; } /// The peer sent a GOAWAY: its routes rank below every other. Idempotent. @@ -799,6 +803,93 @@ mod tests { assert_eq!(subscriber.announces.available(), 2); } + /// The route and claims a peer sends one stream, its Route ID 0. + fn feed( + routed: &mut super::Routed, + decoder: &mut AnnounceDecoder, + paths: &[&'static str], + ) -> Result<(), crate::Error> { + let prefix = Path::empty(); + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + routed.receive(&prefix, route, decoder)?; + for path in paths { + routed.receive(&prefix, start(path), decoder)?; + } + Ok(()) + } + + /// A request handed over keeps what it holds on a budget of its own, so the claims + /// arriving on both it and its replacement during the handover count once against the + /// session's cap: a session that fits its claims stays open. + #[kio::test] + async fn a_handover_counts_the_replacement_alone() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + // One route and three claims. + subscriber.announces = crate::session::Slots::new(4); + let prefix = Path::empty(); + let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut old_decoder = AnnounceDecoder::default(); + feed(&mut old, &mut old_decoder, &["x", "y"]).unwrap(); + + old.hand_over(); + assert_eq!( + subscriber.announces.available(), + 4, + "the session's count is the replacement's" + ); + let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut new_decoder = AnnounceDecoder::default(); + feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); + + // A claim the peer starts during the handover reaches both requests. + old.receive(&prefix, start("z"), &mut old_decoder) + .expect("the old request fits it on its own budget"); + new.receive(&prefix, start("z"), &mut new_decoder) + .expect("the session fits it"); + assert_eq!(subscriber.announces.available(), 0); + + drop(old); + assert_eq!( + subscriber.announces.available(), + 0, + "the old request held none of the session's" + ); + drop(new); + assert_eq!(subscriber.announces.available(), 4); + } + + /// A peer flooding a request handed over fails it on its own budget, the session's cap, + /// so the session holds at most twice the cap during one handover. + #[kio::test] + async fn a_handover_caps_the_old_request() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + subscriber.announces = crate::session::Slots::new(3); + let prefix = Path::empty(); + let mut old = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut old_decoder = AnnounceDecoder::default(); + feed(&mut old, &mut old_decoder, &["x"]).unwrap(); + + old.hand_over(); + let mut new = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut new_decoder = AnnounceDecoder::default(); + feed(&mut new, &mut new_decoder, &["x", "y"]).unwrap(); + assert_eq!(subscriber.announces.available(), 0); + + old.receive(&prefix, start("y"), &mut old_decoder) + .expect("the old request's budget has room for one more"); + assert!( + old.receive(&prefix, start("z"), &mut old_decoder).is_err(), + "the old request is capped on its own budget" + ); + assert!( + new.receive(&prefix, start("z"), &mut new_decoder).is_err(), + "the replacement is capped on the session's" + ); + } + /// A downstream peer raising its ROUTE_REQUEST seqno while this stream's peer /// leaves its replies unread holds one request for the identity, sent once the /// writer flushes and naming the highest seqno. diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index 707111ab18c..94935f63329 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -1097,6 +1097,11 @@ impl Slots { } } + /// A count of its own under the same cap, starting empty and reported nowhere. + pub(crate) fn split(&self) -> Self { + Self::new(self.max) + } + /// Report how many are held to `stats`, as `cap`'s peak. pub(crate) fn with_stats(mut self, stats: &crate::stats::Session, cap: crate::stats::Cap) -> Self { stats.track_held(cap, &self.live);