From 221c87c968a18f57fc9f26a02a49282aa016688c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 22:33:32 -0800 Subject: [PATCH 01/61] chore(quest): claim quest/m2/cluster-routing/routes/session.md Quest-Claim: e7a08a7c96080d03 From b755d0e9cb57f4e21cc6e42d6c59c7caa5ce9981 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:07:23 -0800 Subject: [PATCH 02/61] feat(net): present each route's claims at a stream's whole interest A floor is now one marker per route across the stream's interest. The router presents its per-region view onto it: claims inside keep their path, the most specific covering claim moves onto the interest (moved, not ended, when withdrawn), and the floor certifies only when the view knows the whole interest at one revision. Claims a stream carries uncertified stay in its heard view and show where no certified floor proves them absent; they go offline, never END, when their stream drops them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/mod.rs | 170 ++++++++++++++++------ rs/moq-net/src/route/tests/handoff.rs | 145 ++++++++++-------- rs/moq-net/src/route/tests/mod.rs | 64 ++++++-- rs/moq-net/src/route/tests/routing.rs | 46 +++++- rs/moq-net/src/route/view.rs | 202 ++++++++++++++++++++------ 5 files changed, 458 insertions(+), 169 deletions(-) diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 64ee72f116c..07f131aee0b 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -20,6 +20,7 @@ mod view; mod tests; use std::{ + borrow::Cow, cmp::Reverse, collections::{BTreeMap, BTreeSet}, fmt, @@ -30,7 +31,7 @@ use crate::Path; pub use proof::RouteKey; pub(crate) use proof::{EXPORTER_LABEL, Proof, Transcript}; -pub(crate) use view::{View, overlaps, within}; +pub(crate) use view::{View, within}; /// An opaque routing identity, declared by an origin or derived by [`Scope`]. pub(crate) type Id = u64; @@ -141,13 +142,15 @@ pub(crate) enum Reach { pub(crate) enum Out { /// ROUTE_START or ROUTE_UPDATE, ROUTE_HOLD for a held reach, or ROUTE_END for `None`. Route { to: N, id: Id, reach: Option }, - /// A batch of announcement changes for `id`, closed by its floor list, which - /// replaces the last one sent for that route. + /// A batch of announcement changes for `id`, as the neighbor's request presents + /// them, closed by ANNOUNCE_FLOOR: the revision the stream's claims for the route + /// are the origin's at, across the neighbor's whole interest, or `None` when this + /// node does not know them there at one revision. Batch { to: N, id: Id, changes: Vec<(Path, Option)>, - floors: BTreeMap, + floor: Option, }, /// ROUTE_REQUEST, toward the neighbor whose route this node would use. Request { to: N, id: Id, seqno: Seqno }, @@ -240,6 +243,9 @@ struct Peer { draining: bool, /// The prefix the neighbor asked this node for, once it opened a stream. watch: Option, + /// The prefix this node asked the neighbor for, which its floors certify; the + /// node's interest when unset. + asked: Option, } /// An identity this node serves itself. @@ -367,6 +373,7 @@ impl Router { link, draining: false, watch: None, + asked: None, }, ); // Nothing is sent or learned until a stream opens. @@ -416,6 +423,14 @@ impl Router { Ok(()) } + /// This node asked `neighbor` for announcements under `prefix`: the region its + /// floors certify. + pub fn asks(&mut self, neighbor: N, prefix: Path) -> Result<(), Error> { + let peer = self.neighbors.get_mut(&neighbor).ok_or(Error::UnknownNeighbor)?; + peer.asked = Some(prefix); + Ok(()) + } + /// `neighbor` sent GOAWAY: it ranks below every other eligible route while /// its session stays up. pub fn goaway(&mut self, neighbor: N) { @@ -632,15 +647,10 @@ impl Router { } /// A batch of announcement changes for `named` from `from`, closed by its - /// floor list, on the stream answering interest generation `interest`. - pub fn batch( - &mut self, - from: N, - named: Id, - changes: Vec<(Path, Option)>, - floors: BTreeMap, - interest: u32, - ) { + /// floor, on the stream answering interest generation `interest`. A floor + /// certifies the whole region this node asked for at that revision; `None` + /// certifies nothing, so the stream's claims stay only while it carries them. + pub fn batch(&mut self, from: N, named: Id, changes: Vec<(Path, Option)>, floor: Option, interest: u32) { let Some(peer) = self.neighbors.get(&from) else { return; }; @@ -652,6 +662,8 @@ impl Router { return; }; let scope = peer.link.scope.as_ref().map(|scope| scope.regions.clone()); + let region = peer.asked.clone().unwrap_or_else(|| self.watch.clone()); + let floors = floor.map(|rev| BTreeMap::from([(region, rev)])).unwrap_or_default(); let heard = self.heard.entry((id, from)).or_insert_with(|| Heard { raw: View::default(), view: View::default(), @@ -664,7 +676,7 @@ impl Router { } heard.raw.apply(changes, floors); heard.view = match &scope { - Some(regions) => heard.raw.restrict_all(regions), + Some(regions) => heard.raw.present_all(regions), None => heard.raw.clone(), }; let view = heard.view.clone(); @@ -1041,18 +1053,58 @@ impl Router { fn forgot_outside(&mut self, id: Id, kept: &[Path]) { #[cfg(test)] for ((key, claim), count) in self.forgot.iter_mut() { - if *key == id && !kept.iter().any(|region| overlaps(claim, region)) { + if *key == id && !kept.iter().any(|region| within(claim, region)) { *count += 1; } } } - /// The view this node shows and forwards for `id`, having selected `route`. - fn adopted(&self, id: Id, route: Option>) -> Option<&View> { + /// The view this node shows and forwards for `id`, having selected `route`: what + /// it knows certified, and each claim a certifier carries uncertified where no + /// floor it knows covers the claim. + fn adopted(&self, id: Id, route: Option>) -> Option>> { if let Some(origin) = self.origins.get(&id) { - return Some(&origin.view); + return Some(Cow::Borrowed(&origin.view)); + } + if !self.live(id, route) { + return None; + } + let merged = self.views.get(&id); + let mut loose = self.loose(id, route, merged).peekable(); + if loose.peek().is_none() { + return merged.map(Cow::Borrowed); } - self.live(id, route).then(|| self.views.get(&id)).flatten() + let mut view = merged.cloned().unwrap_or_default(); + for (claim, item, _) in loose { + view.items.entry(claim.clone()).or_insert_with(|| item.clone()); + } + Some(Cow::Owned(view)) + } + + /// The uncertified claims of `id` its certifiers carry on streams answering the + /// current interest where `merged` certifies nothing, the selected next hop's + /// first, and whether the carrier is held. + fn loose<'a>( + &'a self, + id: Id, + route: Option>, + merged: Option<&'a View>, + ) -> impl Iterator + 'a { + let mut certifiers = self.certifiers(id); + if let Some(next) = route.and_then(|route| route.next) { + certifiers.sort_by_key(|&n| n != next); + } + let routes = self.rx.get(&id); + certifiers.into_iter().flat_map(move |next| { + let held = routes.is_some_and(|routes| routes[&next].held); + self.heard + .get(&(id, next)) + .filter(|heard| heard.interest == self.interest) + .into_iter() + .flat_map(|heard| heard.view.uncertified()) + .filter(move |(claim, _)| merged.is_none_or(|merged| merged.as_of(claim).is_none())) + .map(move |(claim, item)| (claim, item, held)) + }) } /// Whether this node sends `id` to `neighbor` at all: its own identities to @@ -1135,12 +1187,13 @@ impl Router { // Recording may have tightened the feasibility distance: select again. let route = self.selected(id); let live = self.live(id, route); - let adopted = self.adopted(id, route).cloned(); - // A sender that certifies anything answers every stream, even one whose - // interest none of its regions meets, so the receiver knows it certifies - // nothing there. - let answers = adopted.as_ref().is_some_and(|view| !view.floors.is_empty()); - let mut restricted: Vec<(Path, View)> = Vec::new(); + let adopted = self.adopted(id, route).map(Cow::into_owned); + // A sender that knows anything answers every stream, even one whose interest + // none of its regions meets, so the receiver knows it certifies nothing there. + let answers = adopted + .as_ref() + .is_some_and(|view| !view.floors.is_empty() || !view.items.is_empty()); + let mut presented: Vec<(Path, View)> = Vec::new(); for &to in peers { let peer = &self.neighbors[&to]; @@ -1174,20 +1227,18 @@ impl Router { self.sent_views.remove(&(id, to)); continue; } - // Peers mostly share an interest, so restrict once per interest. - let index = match restricted.iter().position(|(interest, _)| *interest == watch) { + // Peers mostly share an interest, so present once per interest. + let index = match presented.iter().position(|(interest, _)| *interest == watch) { Some(index) => index, None => { - let view = adopted - .as_ref() - .map_or_else(View::default, |view| view.restrict(&watch)); - restricted.push((watch, view)); - restricted.len() - 1 + let view = adopted.as_ref().map_or_else(View::default, |view| view.present(&watch)); + presented.push((watch.clone(), view)); + presented.len() - 1 } }; - let view = &restricted[index].1; + let view = &presented[index].1; let changes = match self.sent_views.get(&(id, to)) { - Some(sent) if sent == view => continue, + Some(sent) if sent.items == view.items && sent.floor(&watch) == view.floor(&watch) => continue, Some(sent) => view.diff(sent), None if !answers => continue, None => view.diff(&View::default()), @@ -1197,7 +1248,7 @@ impl Router { to, id, changes, - floors: view.floors.clone(), + floor: view.floor(&watch), }); } } @@ -1255,23 +1306,39 @@ impl Router { fn show(&mut self, id: Id) { let mut shown = BTreeMap::new(); let route = self.selected(id); + let usable = route.is_some(); + let own = self.origins.contains_key(&id); + let merged = match own { + true => None, + false => self.views.get(&id), + }; + // An uncertified claim is available through a carrier that is not held. + let mut loose: BTreeMap<&Path, bool> = BTreeMap::new(); + if !own && self.live(id, route) { + for (claim, _, held) in self.loose(id, route, merged) { + *loose.entry(claim).or_insert(false) |= !held; + } + } if let Some(view) = self.adopted(id, route) { - let usable = route.is_some(); - let own = self.origins.contains_key(&id); - // A claim is available only through a certifier that is not held: - // another scope's usable route cannot serve it. + // A certified claim is available only through a certifier that is not + // held: another scope's usable route cannot serve it. let serving = match usable && !view.items.is_empty() && !own { true => self.certified(id, false), false => None, }; let current = self.current.get(&id); for (claim, item) in &view.items { - let certified = current.is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); + // A claim a stream answering the current interest carries is current. + let certified = loose.contains_key(claim) + || current.is_none_or(|regions| regions.iter().any(|region| within(claim, region))); let available = usable - && serving - .as_ref() - .is_none_or(|regions| regions.iter().any(|region| overlaps(claim, region))); - if overlaps(claim, &self.watch) && certified { + && match loose.get(claim) { + Some(&unheld) => unheld || serving.is_none(), + None => serving + .as_ref() + .is_none_or(|regions| regions.iter().any(|region| within(claim, region))), + }; + if within(claim, &self.watch) && certified { shown.insert( claim.clone(), Shown { @@ -1303,13 +1370,20 @@ impl Router { } } None => { - // A claim no routed neighbor's region covers was let go, not ended. + // A claim no routed neighbor's region covers was let go, not ended, + // and so was one the view never certified: only its stream left. let uncertified = self .certified(id, true) - .is_some_and(|regions| !regions.iter().any(|region| overlaps(claim, region))); - let kind = if !overlaps(claim, &self.watch) { + .is_some_and(|regions| !regions.iter().any(|region| within(claim, region))) + || !own && self.views.get(&id).is_none_or(|view| view.as_of(claim).is_none()); + // A claim presented at a narrower interest moved up to the claim it + // stood for, which is still shown. + let moved = shown + .iter() + .any(|(above, now)| above != claim && within(claim, above) && now.item == was.item); + let kind = if !within(claim, &self.watch) { Kind::Unwatched - } else if self.live(id, route) && !uncertified { + } else if self.live(id, route) && !uncertified && !moved { Kind::End } else { Kind::Offline diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 4c72ca7a1b6..89ad0107f78 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -411,10 +411,11 @@ fn two_scoped_uplinks() -> Sim { sim } -/// A relay forwards a key whose floors are all grant scopes, and its peer -/// moves to an interest no scope meets. The relay still answers the new -/// stream, certifying nothing, so the peer lets go of a claim it kept from the -/// scoped view instead of waiting on an answer that never comes. +/// A relay forwards a key whose floors are all grant scopes: each presents a +/// claim above its scope at the scope itself, and certifies nothing across a +/// peer's whole interest, so the peer shows both presentations uncertified. +/// When the peer moves to an interest no scope meets, each relay still answers +/// the new stream with nothing, so the peer holds no stale stream open. #[test] fn a_disjoint_interest_is_answered_with_nothing() { let (origin, z) = (0, 4); @@ -422,12 +423,23 @@ fn a_disjoint_interest_is_answered_with_nothing() { sim.announce(origin, "live/", 0, None); assert!(sim.run()); let id = qualified("local", 7); - assert_eq!(sim.shows(z, id, "live/"), Some(true)); + assert_eq!(sim.shows(z, id, "live/"), None); + assert_eq!(sim.shows(z, id, "live/a/"), Some(true)); + assert_eq!(sim.shows(z, id, "live/b/"), Some(true)); + assert!( + sim.nodes[z].views().all(|(_, view)| view.items.is_empty()), + "nothing certified" + ); sim.watch(z, "live/c"); assert!(sim.run()); - assert_eq!(sim.shows(z, id, "live/"), None); - assert_eq!(life(&sim, z, "live/"), [Kind::Start, Kind::Offline]); + assert!(sim.shown(z).is_empty()); + assert_eq!(life(&sim, z, "live/a/"), [Kind::Start, Kind::Unwatched]); + let router = &sim.nodes[z]; + assert!( + router.heard.values().all(|heard| heard.interest == router.interest), + "both relays answered" + ); sim.check().unwrap(); } @@ -484,11 +496,12 @@ fn a_lost_scope_keeps_newer_knowledge_where_still_certified() { sim.check().unwrap(); } -/// When one scoped uplink closes, its relay is starved and holds, so the third -/// relay shows that scope unavailable; once a seqno request recovers a route, -/// nothing certifies the lost scope and the third lets it go. Certifiers are -/// feasible neighbors only, so the two never keep the scope alive through each -/// other. +/// When one scoped uplink closes, its relay is starved and holds, and the +/// third relay knows that scope's claims only uncertified, through the starved +/// relay. Once the other relay's newer seqno makes the starved one infeasible, +/// nothing carries the lost scope and the third lets it go, whether or not the +/// starved relay's own requests get through. Certifiers are feasible neighbors +/// only, so the two never keep the scope alive through each other. #[test] fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { let (origin, local, x, z) = (0, 1, 2, 4); @@ -510,13 +523,17 @@ fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { sim.disconnect(x, local); sim.unannounce(origin, "live/a/x"); assert!(sim.run()); - let lost = if request { None } else { Some(false) }; if !request { // Only the ROUTE update reaches z, not the requests. while sim.deliver(x, z) {} assert!(sim.run()); } - assert_eq!(sim.shows(z, id, "live/a/x"), lost, "request {request}"); + assert_eq!(sim.shows(z, id, "live/a/x"), None, "request {request}"); + assert_eq!( + life(&sim, z, "live/a/x").last(), + Some(&Kind::Offline), + "request {request}" + ); assert_eq!(sim.shows(z, id, "live/b/z"), Some(true), "request {request}"); assert_eq!(false_ends(&sim.events), 0, "request {request}"); sim.check().unwrap(); @@ -524,16 +541,27 @@ fn a_relay_lets_go_of_a_scope_its_uplinks_lost() { } /// Every lite-07 claim covers the paths below it, so a claim above an -/// interest is inside it. +/// interest covers it, and a stream answering the interest presents it there. +/// Withdrawing it moves the presentation to the next claim up rather than +/// ending it. #[test] -fn a_claim_above_an_interest_overlaps_it() { +fn a_claim_above_an_interest_is_presented_at_it() { let (origin, client) = (0, 3); let mut sim = lagging("live/a/x"); + sim.announce(origin, "live", 0, None); sim.announce(origin, "live/a", 0, None); sim.announce(origin, "live/b", 0, None); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live/a"), Some(true)); - assert_eq!(sim.shows(client, 7, "live/b"), None); + assert_eq!(sim.shows(client, 7, "live/a/x"), Some(true)); + assert_eq!(sim.shown(client).len(), 1); + let below = sim.shown(client)[&(7, p("live/a/x"))].item; + assert_eq!(Some(below), sim.truth(7, "live/a", sim.revision(7))); + + sim.unannounce(origin, "live/a"); + assert!(sim.run()); + let above = sim.shown(client)[&(7, p("live/a/x"))].item; + assert_eq!(Some(above), sim.truth(7, "live", sim.revision(7))); + assert!(!life(&sim, client, "live/a/x").contains(&Kind::End), "moved, not ended"); sim.check().unwrap(); } @@ -560,21 +588,29 @@ fn wildcard_claims_end_per_origin_and_are_certified_below() { sim.announce(o2, "live/", 0, Some(1)); assert!(sim.run()); assert_eq!(sim.shown(client).len(), 2); + assert_eq!( + sim.shows(client, 10, "live/a/"), + Some(true), + "presented at the interest" + ); sim.hold(o1, r2); sim.unannounce(o1, "live/"); assert!(sim.run()); - assert_eq!(sim.shows(client, 10, "live/"), None); + assert_eq!(sim.shows(client, 10, "live/a/"), None); assert_eq!( - sim.shows(client, 11, "live/"), + sim.shows(client, 11, "live/a/"), Some(true), "the other origin's claim stays" ); sim.disconnect(client, r1); assert!(sim.run()); - assert_eq!(sim.shows(client, 10, "live/"), None); - assert_eq!(sim.shows(client, 11, "live/"), Some(true)); + assert_eq!(sim.shows(client, 10, "live/a/"), None); + assert_eq!(sim.shows(client, 11, "live/a/"), Some(true)); + // Each origin's claim starts, and only the withdrawn one ends. + assert_eq!(life(&sim, client, "live/a/"), [Kind::Start, Kind::Start, Kind::End]); + assert_eq!(false_ends(&sim.events), 0); } /// An origin process restarts with a fresh identity: its claims start a new @@ -1042,22 +1078,20 @@ fn sweep_scoped_handoff() { sim.check().unwrap_or_else(|err| panic!("scenario {scenario}: {err}")); let watch = sim.watching(client).clone(); let rev = sim.revision(7); - let expected: BTreeSet = CLAIMS - .into_iter() - .filter(|claim| sim.truth(7, claim, rev).is_some()) - .map(p) - .filter(|claim| { - SCOPES.iter().map(|scope| p(scope)).any(|scope| { - let region = if within(&scope, &watch) { - scope - } else if within(&watch, &scope) { - watch.clone() - } else { - return false; - }; - overlaps(claim, ®ion) - }) + // Each relay presents its scope to the client's interest. + let expected: BTreeSet = SCOPES + .iter() + .map(|scope| p(scope)) + .filter_map(|scope| { + if within(&scope, &watch) { + Some(scope) + } else if within(&watch, &scope) { + Some(watch.clone()) + } else { + None + } }) + .flat_map(|region| sim.presented(7, ®ion, rev).into_keys()) .collect(); let id = qualified("local", 7); let shown: BTreeSet = sim @@ -1212,25 +1246,23 @@ fn settled(sim: &mut Sim, label: &str) { sim.check().unwrap(); let live: Vec = (RELAYS..RELAYS + 2).map(|origin| sim.proven(origin)).collect(); for node in 0..RELAYS + 3 { - let mut expected = BTreeSet::new(); + let mut expected = BTreeMap::new(); for &id in &live { assert!( sim.route(node, id).is_some(), "{label}: node {node} has no route to {id}" ); - for claim in CLAIMS { - if overlaps(&p(claim), sim.watching(node)) && sim.truth(id, claim, sim.revision(id)).is_some() { - expected.insert((id, p(claim))); - } + for (claim, item) in sim.presented(id, sim.watching(node), sim.revision(id)) { + expected.insert((id, claim), item); } } - let shown: BTreeSet<(Id, Path)> = sim.shown(node).keys().cloned().collect(); - assert_eq!(shown, expected, "{label}: node {node}"); + let shown: BTreeSet<&(Id, Path)> = sim.shown(node).keys().collect(); + assert_eq!(shown, expected.keys().collect(), "{label}: node {node}"); for ((id, claim), shown) in sim.shown(node) { assert!(shown.available, "{label}: node {node} {id} {claim}"); assert_eq!( - sim.truth(*id, claim.as_str(), sim.revision(*id)), - Some(shown.item), + expected.get(&(*id, claim.clone())), + Some(&shown.item), "{label}: node {node} {id} {claim}" ); } @@ -1324,14 +1356,13 @@ fn a_restarted_route_sends_a_complete_snapshot() { sim.check().unwrap(); } -/// A claim covering both interests stays shown when a client moves to a -/// disjoint interest: what it knows of that claim is current as of the old -/// interest, which carried every change to it until the switch. A withdrawal -/// still queued on the old stream is discarded with it and arrives on the -/// new stream's snapshot instead: the claim ends once, never revives, and no -/// END is false. +/// A claim covering two disjoint interests is presented at each one's own +/// path, so a client moving between them stops watching the old presentation +/// at once. A withdrawal still queued on the old stream is discarded with it, +/// and the new stream's snapshot never shows the withdrawn claim: nothing +/// revives and no END is false. #[test] -fn a_covering_claim_survives_a_disjoint_interest_until_the_new_snapshot() { +fn a_covering_claim_moves_with_a_disjoint_interest() { let (origin, relay, client) = (0, 1, 2); let mut sim = Sim::new(3); sim.leaf(origin); @@ -1342,19 +1373,19 @@ fn a_covering_claim_survives_a_disjoint_interest_until_the_new_snapshot() { sim.originate(origin, 7, 0); sim.announce(origin, "live", 0, None); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), Some(true)); + assert_eq!(sim.shows(client, 7, "live/a"), Some(true)); sim.hold(relay, client); sim.unannounce(origin, "live"); assert!(sim.run()); sim.watch(client, "live/b"); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), Some(true), "lags like any snapshot"); + assert!(sim.shown(client).is_empty(), "nothing answers the new interest yet"); sim.release(relay, client); assert!(sim.run()); - assert_eq!(sim.shows(client, 7, "live"), None); - assert_eq!(life(&sim, client, "live"), [Kind::Start, Kind::End]); + assert!(sim.shown(client).is_empty()); + assert_eq!(life(&sim, client, "live/a"), [Kind::Start, Kind::Unwatched]); assert_eq!(revivals(&sim.events), 0); assert_eq!(false_ends(&sim.events), 0); sim.check().unwrap(); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 7c46f61cfd5..93d8fc9b5c3 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -97,7 +97,7 @@ enum Msg { Batch { id: Id, changes: Vec<(Path, Option)>, - floors: BTreeMap, + floor: Option, interest: u32, }, Request { @@ -387,9 +387,9 @@ impl Sim { Msg::Batch { id, changes, - floors, + floor, interest, - } => router.batch(from, id, changes, floors, interest), + } => router.batch(from, id, changes, floor, interest), Msg::Request { id, seqno } => router.request(from, id, seqno), } self.stats.delivered += 1; @@ -468,12 +468,7 @@ impl Sim { self.stats.route_msgs += 1; (to, Msg::Route { id, reach }) } - Out::Batch { - to, - id, - changes, - floors, - } => { + Out::Batch { to, id, changes, floor } => { self.stats.claim_msgs += 1; self.stats.claim_items += changes.len(); let interest = self.nodes[to].interest(); @@ -482,7 +477,7 @@ impl Sim { Msg::Batch { id, changes, - floors, + floor, interest, }, ) @@ -526,7 +521,7 @@ impl Sim { let origin = self.source(event.id).unwrap_or(event.id); let since = event.item.since; let withdrawn = self - .truth(origin, event.claim.as_str(), self.revision(origin)) + .covering(origin, &event.claim, self.revision(origin)) .is_none_or(|item| item.since != since); self.events.push(SimEvent { node, @@ -618,6 +613,39 @@ impl Sim { .and_then(|change| change.item) } + /// The origin's most specific claim at or above `claim` as of revision `rev`: + /// what a stream answering `claim` presents there. + pub fn covering(&self, id: Id, claim: &Path, rev: Rev) -> Option { + let history = self.history.get(&id)?; + let ancestors: BTreeSet<&Path> = history + .iter() + .map(|change| &change.claim) + .filter(|known| within(claim, known)) + .collect(); + ancestors + .into_iter() + .rev() + .filter_map(|known| self.truth(id, known.as_str(), rev)) + .next() + } + + /// The origin's claims as of revision `rev` as a stream answering `prefix` + /// presents them. + pub fn presented(&self, id: Id, prefix: &Path, rev: Rev) -> BTreeMap { + let claims: BTreeSet<&Path> = (self.history.get(&id).into_iter().flatten()) + .map(|change| &change.claim) + .filter(|claim| within(claim, prefix)) + .collect(); + let mut out: BTreeMap = claims + .into_iter() + .filter_map(|claim| Some((claim.clone(), self.truth(id, claim.as_str(), rev)?))) + .collect(); + if let Some(item) = self.covering(id, prefix, rev) { + out.entry(prefix.clone()).or_insert(item); + } + out + } + pub fn revision(&self, id: Id) -> Rev { self.history .get(&id) @@ -635,10 +663,11 @@ impl Sim { continue; }; checked.insert((node, key)); - let claims: BTreeSet<&Path> = self.history[&id].iter().map(|change| &change.claim).collect(); - if let Some(claim) = view.items.keys().find(|claim| !claims.contains(claim)) { - return Err(format!("node {node} holds {key}'s {claim} with no origin history")); - } + // A floor's region presents the most specific claim covering it. + let claims: BTreeSet<&Path> = (self.history[&id].iter().map(|change| &change.claim)) + .chain(view.items.keys()) + .chain(view.floors.keys()) + .collect(); for claim in claims { let Some(rev) = view.as_of(claim) else { if view.items.contains_key(claim) { @@ -647,7 +676,10 @@ impl Sim { continue; }; let known = view.items.get(claim).copied(); - let truth = self.truth(id, claim.as_str(), rev); + let truth = match view.floors.contains_key(claim) { + true => self.covering(id, claim, rev), + false => self.truth(id, claim.as_str(), rev), + }; if known != truth { return Err(format!( "node {node} holds {key}'s {claim} as {known:?} at revision {rev}, where the origin had {truth:?}" diff --git a/rs/moq-net/src/route/tests/routing.rs b/rs/moq-net/src/route/tests/routing.rs index 01316c3c763..cbb7e9af7b9 100644 --- a/rs/moq-net/src/route/tests/routing.rs +++ b/rs/moq-net/src/route/tests/routing.rs @@ -844,7 +844,6 @@ fn an_overflowing_route_neither_certifies_nor_holds() { down_only: false, })) }; - let floors = |region: &str| BTreeMap::from([(p(region), 1)]); let item = Item { seed: 0, epoch: None, @@ -859,10 +858,11 @@ fn an_overflowing_route_neither_certifies_nor_holds() { let mut router: Router = Router::new(1); router.connect(near, Link::default()).unwrap(); router.connect(far, far_link.clone()).unwrap(); + router.asks(near, p("live/a")).unwrap(); router.route(near, 7, finite(1)); - router.batch(near, 7, vec![(p("live/a/x"), Some(item))], floors("live/a"), 0); + router.batch(near, 7, vec![(p("live/a/x"), Some(item))], Some(1), 0); router.route(far, 7, finite(1)); - router.batch(far, 7, vec![(p("live/b/y"), Some(item))], floors(""), 0); + router.batch(far, 7, vec![(p("live/b/y"), Some(item))], Some(1), 0); assert_eq!(router.selected(7).and_then(|route| route.next), Some(near)); assert_eq!(router.shows(7, &p("live/a/x")), Some(true)); assert_eq!( @@ -874,9 +874,47 @@ fn an_overflowing_route_neither_certifies_nor_holds() { let mut router: Router = Router::new(1); router.connect(far, far_link).unwrap(); router.route(far, 7, finite(1)); - router.batch(far, 7, vec![(p("live/x"), Some(item))], floors(""), 0); + router.batch(far, 7, vec![(p("live/x"), Some(item))], Some(1), 0); router.route(far, 7, Some(Reach::Held)); assert!(router.selected(7).is_none()); assert!(!router.grounded(7), "an unreachable held route recovers nothing"); assert_eq!(router.shows(7, &p("live/x")), None); } + +/// A fresh claim arrives only on a stream that certifies nothing, while an +/// older floor on another stream proves it absent. The merge rule trusts the +/// floor, so the claim stays hidden until the certifying neighbor refreshes +/// past it or lets go: that is the bound on how late an uncertified claim +/// shows. +#[test] +fn a_fresh_uncertified_claim_waits_on_an_older_floor() { + let finite = |metric| { + Some(Reach::Finite(Advert { + seqno: 0, + metric, + down_only: false, + })) + }; + let item = Item { + seed: 0, + epoch: None, + since: 2, + }; + let (certifier, carrier) = (1, 2); + for refresh in [true, false] { + let mut router: Router = Router::new(1); + router.connect(certifier, Link::default()).unwrap(); + router.connect(carrier, Link::default()).unwrap(); + router.route(certifier, 7, finite(1)); + router.batch(certifier, 7, Vec::new(), Some(1), 0); + router.route(carrier, 7, finite(1)); + router.batch(carrier, 7, vec![(p("live/x"), Some(item))], None, 0); + assert_eq!(router.shows(7, &p("live/x")), None, "the older floor hides it"); + + match refresh { + true => router.batch(certifier, 7, vec![(p("live/x"), Some(item))], Some(2), 0), + false => router.disconnect(certifier), + } + assert_eq!(router.shows(7, &p("live/x")), Some(true), "refresh {refresh}"); + } +} diff --git a/rs/moq-net/src/route/view.rs b/rs/moq-net/src/route/view.rs index e11b44a6d88..b57288be05d 100644 --- a/rs/moq-net/src/route/view.rs +++ b/rs/moq-net/src/route/view.rs @@ -11,18 +11,14 @@ pub(crate) fn within(path: &Path, prefix: &Path) -> bool { path.has_prefix(prefix) } -/// Whether a claim on `claim` can concern a path under `region`. -/// -/// Every claim covers its own path and everything below it, so a claim meets a -/// region when either lies within the other. -pub(crate) fn overlaps(claim: &Path, region: &Path) -> bool { - within(claim, region) || within(region, claim) -} - /// What a node knows of one origin's claims. /// -/// Each region's floor certifies that the claims overlapping it are exactly the -/// origin's claims there as of that revision. +/// Each region's floor certifies that the claims within it are exactly the +/// origin's claims there as of that revision, as a stream answering that region +/// presents them: every claim inside the region at its own path, and the most +/// specific one above it at the region itself (see [`View::present`]). A claim +/// within no floor is uncertified: some stream carries it, and nothing proves it +/// current. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct View { pub floors: BTreeMap, @@ -39,32 +35,70 @@ impl Default for View { } impl View { - /// The view as a stream to a peer interested in `prefix` carries it. + /// The view as a stream answering `prefix` presents it. /// - /// A region meeting the interest is narrowed to whichever of the two lies - /// within the other, so every region a stream carries lies within its - /// interest. - pub fn restrict(&self, prefix: &Path) -> Self { - let mut floors: BTreeMap = BTreeMap::new(); - for (region, &floor) in &self.floors { - let narrowed = if within(region, prefix) { - region - } else if within(prefix, region) { - prefix - } else { - continue; - }; - let entry = floors.entry(narrowed.clone()).or_insert(floor); - *entry = (*entry).max(floor); + /// Every claim inside the prefix keeps its path, and of the claims above it only + /// the most specific remains, presented at the prefix itself: a receiver can tell + /// apart no claims that cover the whole prefix alike. A region meeting the prefix + /// is narrowed to whichever of the two lies within the other. + /// + /// The newest floor over the prefix certifies what covers it there: a claim + /// between the two, else the one that floor presents at its own region, else + /// none. A claim above that floor is older knowledge, or uncertified, so it is + /// not presented. + pub fn present(&self, prefix: &Path) -> Self { + let mut out = Self { + floors: narrow(&self.floors, prefix), + items: BTreeMap::new(), + }; + let mut covering: Option<(&Path, &A)> = None; + for (claim, item) in &self.items { + if within(claim, prefix) { + out.items.insert(claim.clone(), item.clone()); + } + if within(prefix, claim) && covering.is_none_or(|(best, _)| claim.len() > best.len()) { + covering = Some((claim, item)); + } } - // A claim stays only while a kept floor still certifies it. - let items = self - .items - .iter() - .filter(|(claim, _)| floors.keys().any(|region| overlaps(claim, region))) - .map(|(claim, item)| (claim.clone(), item.clone())) - .collect(); - Self { floors, items } + if let Some(&floor) = out.floors.get(prefix) { + let certifier = self + .floors + .iter() + .filter(|&(region, &rev)| rev == floor && within(prefix, region)) + .map(|(region, _)| region) + .max_by_key(|region| region.len()); + if certifier.is_some_and(|certifier| covering.is_some_and(|(claim, _)| !within(claim, certifier))) { + covering = None; + } + } + if let Some((_, item)) = covering { + out.items.entry(prefix.clone()).or_insert_with(|| item.clone()); + } + out + } + + /// The view presented to the union of `regions`. + pub fn present_all(&self, regions: &[Path]) -> Self { + let mut out = Self::default(); + for region in regions { + let presented = self.present(region); + out.items.extend(presented.items); + for (region, floor) in presented.floors { + let entry = out.floors.entry(region).or_insert(floor); + *entry = (*entry).max(floor); + } + } + out + } + + /// What the view certifies within `prefix`: the view as presented there, less + /// the claims no kept floor certifies. + pub fn restrict(&self, prefix: &Path) -> Self { + let mut out = self.present(prefix); + let floors = &out.floors; + out.items + .retain(|claim, _| floors.keys().any(|region| within(claim, region))); + out } /// The view restricted to the union of `regions`. @@ -76,21 +110,40 @@ impl View { out } - /// The newest revision this view knows `claim` at, if any region covers it. + /// The newest revision this view knows `claim` at, if a region certifies it. pub fn as_of(&self, claim: &Path) -> Option { self.floors .iter() - .filter(|(region, _)| overlaps(claim, region)) + .filter(|(region, _)| within(claim, region)) .map(|(_, floor)| *floor) .max() } + /// The claims no floor certifies. + pub fn uncertified(&self) -> impl Iterator { + self.items.iter().filter(|(claim, _)| self.as_of(claim).is_none()) + } + + /// The single revision the view certifies all of `prefix` at, if one floor + /// covers it and nothing within it is known at another revision. + pub fn floor(&self, prefix: &Path) -> Option { + match self.floors.iter().next() { + Some((region, &rev)) if self.floors.len() == 1 && region == prefix => Some(rev), + _ => None, + } + } + /// Take `other`'s state for every claim it knows at a newer revision, then /// raise the floors and drop any region a wider one at least as new covers. + /// An uncertified claim of `other` is known at no revision, so it is not taken. pub fn merge(&mut self, other: &Self) { let claims: BTreeSet = self.items.keys().chain(other.items.keys()).cloned().collect(); for claim in claims { - if other.as_of(&claim) > self.as_of(&claim) { + let (theirs, ours) = (other.as_of(&claim), self.as_of(&claim)); + // At one revision, a claim exactly on a wider region outranks the claim + // above it that a floor on this path presented here. + let exact = theirs == ours && self.floors.contains_key(&claim) && !other.floors.contains_key(&claim); + if theirs > ours || exact { match other.items.get(&claim) { Some(item) => self.items.insert(claim, item.clone()), None => self.items.remove(&claim), @@ -123,7 +176,7 @@ impl View { if Some(floor) > newest && !kept.floors.contains_key(region) { kept.floors.insert(region.clone(), floor); for (claim, item) in &self.items { - if overlaps(claim, region) { + if within(claim, region) { kept.items.insert(claim.clone(), item.clone()); } } @@ -150,8 +203,7 @@ impl View { out } - /// Apply a batch: its changes, then its floor list, which replaces the - /// last one. + /// Apply a batch: its changes, then its floors, which replace the last. pub fn apply(&mut self, changes: Vec<(Path, Option)>, floors: BTreeMap) { for (claim, item) in changes { match item { @@ -163,6 +215,24 @@ impl View { } } +/// `floors` narrowed to `prefix`: a region meeting it becomes whichever of the two +/// lies within the other, keeping the newest floor where two narrow alike. +fn narrow(floors: &BTreeMap, prefix: &Path) -> BTreeMap { + let mut out: BTreeMap = BTreeMap::new(); + for (region, &floor) in floors { + let narrowed = if within(region, prefix) { + region + } else if within(prefix, region) { + prefix + } else { + continue; + }; + let entry = out.entry(narrowed.clone()).or_insert(floor); + *entry = (*entry).max(floor); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -179,10 +249,54 @@ mod tests { } #[test] - fn restrict_narrows_regions_to_the_interest() { - let v = view(&[("", 5)], &[("live/a", 1), ("live", 2), ("other/x", 3)]); - let r = v.restrict(&p("live/a")); - assert_eq!(r, view(&[("live/a", 5)], &[("live/a", 1), ("live", 2)])); + fn present_narrows_regions_and_moves_the_covering_claim_onto_the_interest() { + let v = view(&[("", 5)], &[("live/a/x", 1), ("live", 2), ("other/x", 3)]); + let r = v.present(&p("live/a")); + assert_eq!(r, view(&[("live/a", 5)], &[("live/a/x", 1), ("live/a", 2)])); + assert_eq!(r.floor(&p("live/a")), Some(5)); + } + + #[test] + fn present_keeps_only_the_most_specific_covering_claim() { + let v = view(&[("", 5)], &[("", 9), ("live", 10), ("live/a", 1)]); + assert_eq!(v.present(&p("live/a/x")), view(&[("live/a/x", 5)], &[("live/a/x", 1)])); + // A claim at the interest itself is the most specific there is. + let v = view(&[("", 5)], &[("live", 10), ("live/a", 1)]); + assert_eq!(v.present(&p("live/a")), view(&[("live/a", 5)], &[("live/a", 1)])); + } + + #[test] + fn restrict_moves_a_certified_covering_claim_and_drops_an_uncertified_one() { + let v = view(&[("", 5)], &[("live/a/x", 1), ("live", 2)]); + assert_eq!( + v.restrict(&p("live/a")), + view(&[("live/a", 5)], &[("live/a", 2), ("live/a/x", 1)]) + ); + // The floor proves no claim covers its region, so the claim above is stale. + let v = view(&[("live/a", 5)], &[("live/a/x", 1), ("live", 2)]); + assert_eq!(v.restrict(&p("live/a/y")), view(&[("live/a/y", 5)], &[])); + } + + #[test] + fn a_wider_floor_at_one_revision_replaces_a_claim_moved_onto_a_narrower_one() { + let mut a = view(&[("live/b/z", 5)], &[("live/b/z", 1)]); + a.merge(&view(&[("live/b", 5)], &[("live/b", 1)])); + assert_eq!(a, view(&[("live/b", 5)], &[("live/b", 1)])); + } + + #[test] + fn a_floor_certifies_only_what_lies_within_it() { + let v = view(&[("live/a", 5)], &[("live/a", 1), ("live/a/x", 2), ("live", 3)]); + assert_eq!(v.as_of(&p("live/a")), Some(5)); + assert_eq!(v.as_of(&p("live/a/x")), Some(5)); + assert_eq!(v.as_of(&p("live")), None); + assert_eq!( + v.uncertified().map(|(claim, _)| claim.as_str()).collect::>(), + ["live"] + ); + // Two floors certify no single revision. + let v = view(&[("live", 5), ("live/a", 6)], &[]); + assert_eq!(v.floor(&p("live")), None); } #[test] From 62250733f3226ed2469bfafc3b771e32a0ed32e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:25:21 -0800 Subject: [PATCH 03/61] feat(net): one route node per origin, fed the routes it serves itself The origin keeps a route layer node its lite-07 sessions will share, created on first use with a driver task that announces every route the origin serves itself. A route the route layer selected names its origin node, which a reader can filter on and which ranks routes by the rendezvous hash of the path and that node. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 41 ++++ rs/moq-net/src/model/mod.rs | 1 + rs/moq-net/src/model/origin.rs | 163 ++++++++++++- rs/moq-net/src/route/mod.rs | 14 +- rs/moq-net/src/route/node.rs | 411 +++++++++++++++++++++++++++++++++ 5 files changed, 623 insertions(+), 7 deletions(-) create mode 100644 rs/moq-net/src/route/node.rs diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 35b6955bf14..3cd7c038974 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -543,6 +543,36 @@ impl Permit { self.limit.as_ref().is_none_or(|limit| limit.matches(path)) } + /// Whether this allows every path at or beneath `prefix`, judged pattern by + /// pattern, so a `false` may be spurious but a `true` never is. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn covers_subtree(&self, prefix: &str) -> bool { + let Ok(subtree) = moq_pattern::Pattern::subtree(prefix) else { + return false; + }; + let covers = |patterns: &Option| patterns.as_ref().is_none_or(|patterns| patterns.contains(&subtree)); + covers(&self.granted) && covers(&self.limit) + } + + /// The prefixes this confines a peer to, or `None` when it allows every path: + /// the heads of the grant's patterns, narrowed by the limit's. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn regions(&self) -> Option> { + if self.covers_subtree("") { + return None; + } + let heads = |patterns: &Option| patterns.as_ref().map(crate::model::interest_prefixes); + let regions = match (heads(&self.granted), heads(&self.limit)) { + (Some(granted), Some(limit)) => granted + .iter() + .flat_map(|a| limit.iter().filter_map(move |b| narrower(a, b))) + .collect(), + (Some(regions), None) | (None, Some(regions)) => regions, + (None, None) => return None, + }; + Some(regions) + } + /// Whether this allows everything `other` does, judged part by part, so a `false` /// may be spurious but a `true` never is. pub(crate) fn covers(&self, other: &Self) -> bool { @@ -555,6 +585,17 @@ impl Permit { } } +/// Whichever of two prefixes lies within the other, or `None` when they are disjoint. +fn narrower(a: &crate::Path, b: &crate::Path) -> Option { + if a.has_prefix(b) { + Some(a.clone()) + } else if b.has_prefix(a) { + Some(b.clone()) + } else { + None + } +} + /// The tokens this side presents and the grant they earn, across every session built on it. /// /// A session made without one creates its own, returned by `Session::auth`, which ends diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index b010421a7b5..d80baef7c0f 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -56,6 +56,7 @@ pub use timed::Timed; pub mod origin { pub use super::follow::Following; pub(crate) use super::origin_impl::HiddenWithheld; + pub(crate) use super::origin_impl::NodeFilter; #[cfg(feature = "async")] pub(crate) use super::origin_impl::HiddenWithholding; #[cfg(feature = "async")] diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 7949c6ef0e0..ccedcb9f912 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -517,6 +517,12 @@ pub struct RouteInfo { /// Where the route entered this origin; see [`Self::source`]. Never forwarded. pub(crate) source: Source, + + /// The origin node that serves the route, for a route the lite-07 route layer + /// selected: a request resolved through it names that node to the next hop. + /// `None` for a route this node serves itself, published here or learned from a + /// peer that names no node. + pub(crate) node: Option, } impl Default for RouteInfo { @@ -529,6 +535,7 @@ impl Default for RouteInfo { auth: None, via: Hop::UNKNOWN, source: Source::Local, + node: None, } } } @@ -612,6 +619,18 @@ impl RouteInfo { self } + /// The origin node a route-layer route serves; see [`Self::node`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_node(mut self, node: Option) -> Self { + self.node = node; + self + } + + /// The origin node serving this route, when the route layer selected it. + pub(crate) fn node(&self) -> Option { + self.node + } + /// Whether this route passed through an anonymous hop. /// /// True when the chain holds a 0 anywhere, including Lite03 hop-count @@ -794,9 +813,11 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { entry.cost, !entry.local, entry.hops.len(), - match entry.link == Link::Local && entry.prefix == *path { - true => (0, 0), - false => ( + match (entry.node, entry.link == Link::Local && entry.prefix == *path) { + // Every node that knows the claim ranks its origin node alike. + (Some(node), _) => (fnv_key(path.as_str(), node.to_le_bytes()), 0), + (None, true) => (0, 0), + (None, false) => ( instance_key(path.as_str(), &entry.instance), chain_key(path.as_str(), &entry.hops), ), @@ -808,9 +829,9 @@ fn route_order(path: &Path, entry: &RouteEntry) -> RouteOrder { /// The key [`route_order`] ranks an entry by: lower wins. type RouteOrder = (Reverse, bool, Cost, bool, usize, (u64, u64), Reverse); -/// The `(hops, cost, source, instance, epoch, auth)` metadata an announce cursor delivers +/// The `(hops, cost, source, instance, epoch, auth, node)` metadata an announce cursor delivers /// alongside a prefix. -type RouteMeta = (Hops, Cost, Source, Instance, u64, Option); +type RouteMeta = (Hops, Cost, Source, Instance, u64, Option, Option); /// The earlier of two optional deadlines. fn earliest(a: Option, b: Option) -> Option { @@ -1289,6 +1310,7 @@ impl OriginConsumerState { instance: Some(meta.3), epoch: meta.4, auth: meta.5, + node: meta.6, }, }; let event = match kind { @@ -1328,6 +1350,8 @@ struct RouteEntry { local: bool, /// The link the entry arrived on, from the handle that inserted it. link: Link, + /// The origin node serving the route; see [`RouteInfo::node`]. + node: Option, /// The queue requests under this route are served from, when the announcer /// serves content on demand (a prefix [`Route`]). `None` for an advertise-only /// announcement ([`Producer::announce`]) and for a local broadcast. @@ -1402,18 +1426,20 @@ impl RouteEntry { self.instance.clone(), self.epoch, self.auth.clone(), + self.node, ) } /// Whether [`Self::meta`] would equal `meta`, without building it. fn delivers(&self, meta: &RouteMeta) -> bool { - let (hops, cost, source, instance, epoch, auth) = meta; + let (hops, cost, source, instance, epoch, auth, node) = meta; *hops == self.hops && *cost == self.cost && *source == self.entered() && *instance == self.instance && *epoch == self.epoch && *auth == self.auth + && *node == self.node } /// Whether a request for `path` can be served through this entry. A served @@ -1520,6 +1546,31 @@ struct Horizon { /// forwards them ([`Consumer::forwardable`]). Selection then falls back to a /// shorter route instead of withdrawing a prefix an in-cap path still reaches. max_hops: Option, + /// Which origin nodes' routes this reader sees ([`Consumer::with_node`]). + node: NodeFilter, +} + +/// Which origin nodes' routes a reader sees: a request naming a node resolves only +/// through that node's routes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub(crate) enum NodeFilter { + /// Every route. + #[default] + Any, + /// Only the routes this node serves itself. + Own, + /// Only the routes the route layer selected toward this origin node. + Is(u64), +} + +impl NodeFilter { + fn admits(self, node: Option) -> bool { + match self { + Self::Any => true, + Self::Own => node.is_none(), + Self::Is(id) => node == Some(id), + } + } } impl Horizon { @@ -1528,6 +1579,7 @@ impl Horizon { self.admits_link(entry.link) && entry.visible_to(self.exclude) && self.max_hops.is_none_or(|max| entry.hops.len() < max.get()) + && self.node.admits(entry.node) } /// Whether routes arriving on `link` may be observed through this horizon. @@ -2297,6 +2349,25 @@ impl Producer { self } + /// The rank of the link this handle is given to; see [`Self::with_preference`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn preference(&self) -> Preference { + self.preference + } + + /// The link this handle's routes arrive on; see [`Self::with_link`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn link(&self) -> Link { + self.link + } + + /// The route layer node every lite-07 session on this origin shares; see + /// [`Consumer::route_node`]. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn route_node(&self) -> crate::route::Node { + self.consume().route_node() + } + /// This origin's construction config. pub fn config(&self) -> Config { let shared = self.shared.lock(); @@ -2885,6 +2956,7 @@ impl Announcing { via, local: self.local, link: self.link, + node: route.node, server: serving.server.clone(), source: serving.source.clone(), advertised: serving.advertised, @@ -3329,6 +3401,7 @@ impl AnnounceProducer { entry.cost = route.cost; entry.auth = route.auth.clone(); entry.via = route.via; + entry.node = route.node; entry.advertised = true; let claim = entry.claim.clone(); if advertising { @@ -3380,6 +3453,7 @@ impl AnnounceProducer { auth: entry.auth.clone(), via: entry.via, source: entry.entered(), + node: entry.node, }) } @@ -3478,6 +3552,38 @@ impl Drop for AnnounceProducer { /// Drives origin lifecycle work and cache expiration with caller-supplied time. /// +/// Feeds a route layer node the routes this node serves itself, best per prefix, +/// for as long as the origin lives. +struct OwnClaims { + routes: Routes, + node: crate::route::Node, +} + +impl kio::Task for OwnClaims { + type Output = (); + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + while let Some(event) = ready!(self.routes.poll_next(waiter)) { + match event { + RouteEvent::Start(announce) + | RouteEvent::Update(announce) + | RouteEvent::Restart(announce) + | RouteEvent::Takeover { announce, .. } => { + let route = announce.route; + let claim = route.instance.map(|instance| crate::route::Claim { + instance, + epoch: route.epoch, + cost: route.cost.value(), + }); + self.node.own(announce.prefix, claim); + } + RouteEvent::End(announce) => self.node.own(announce.prefix, None), + } + } + Poll::Ready(()) + } +} + /// Returned by [`Producer::new`]. Poll on external activity or at the deadline /// it returns, supplying nondecreasing instants. RouteInfo changes, track serving, linger, /// failover, and teardown run here; the route table and announce cursors update @@ -5748,6 +5854,10 @@ struct OriginState { // The sessions withholding this origin's hidden routes from their peer. hidden_withheld: HiddenWithheld, + + // The route layer node this origin's lite-07 sessions share; see + // [`Consumer::route_node`]. + route_node: Option, } impl OriginState { @@ -5766,6 +5876,7 @@ impl OriginState { listeners: Vec::new(), next_listener: 0, hidden_withheld: HiddenWithheld::default(), + route_node: None, } } @@ -6951,6 +7062,46 @@ impl Consumer { self } + /// The route layer node every lite-07 session on this origin shares, created on + /// first use along with the task that feeds it the routes this node serves + /// itself: everything in the table the route layer did not select. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn route_node(&self) -> crate::route::Node { + let mut shared = self.shared.lock(); + if let Some(node) = &shared.route_node { + return node.clone(); + } + let node = crate::route::Node::random(); + shared.route_node = Some(node.clone()); + drop(shared); + // The whole table, whatever this handle's scope. + let own = Self { + scope: OriginScope::default(), + root: Path::default(), + horizon: Horizon { + node: NodeFilter::Own, + ..Horizon::default() + }, + covering: false, + ..self.clone() + } + .with_hidden(true); + self.tasks.push(OwnClaims { + routes: own.routes(), + node: node.clone(), + }); + node + } + + /// A clone that sees only the routes `node` admits: a request naming an origin + /// node resolves only through that node's routes, and the route layer's own + /// claims are the routes this node serves itself. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn with_node(mut self, node: NodeFilter) -> Self { + self.horizon.node = node; + self + } + /// A clone whose announcements skip the routes this origin cannot forward: a /// chain already at [`Config::max_hops`] has no room for our hop. Sessions whose /// wire carries the hop path apply this to the routes they advertise. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 07f131aee0b..6365df7e286 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -13,6 +13,7 @@ #![cfg_attr(not(test), allow(dead_code, reason = "the lite-07 session does not drive it yet"))] +mod node; mod proof; mod view; @@ -29,6 +30,7 @@ use std::{ use crate::Path; +pub(crate) use node::{Claim, Node}; pub use proof::RouteKey; pub(crate) use proof::{EXPORTER_LABEL, Proof, Transcript}; pub(crate) use view::{View, within}; @@ -298,6 +300,8 @@ pub(crate) struct Router { heard: BTreeMap<(Id, N), Heard>, sent_views: BTreeMap<(Id, N), View>, shown: BTreeMap<(Id, Path), Shown>, + /// The identities recomputed since [`Self::changed`] last took them. + changed: BTreeSet, /// Mints a fresh identity on rotation. rng: u64, /// Per claim this node has shown, how many times it forgot that claim. @@ -340,6 +344,7 @@ impl Router { heard: BTreeMap::new(), sent_views: BTreeMap::new(), shown: BTreeMap::new(), + changed: BTreeSet::new(), rng: seed | 1, #[cfg(test)] forgot: BTreeMap::new(), @@ -350,6 +355,12 @@ impl Router { } } + /// Take the identities recomputed since the last call: their selection or + /// shown claims may have changed. + pub fn changed(&mut self) -> BTreeSet { + std::mem::take(&mut self.changed) + } + /// Take the messages queued for neighbors, in order. pub fn drain(&mut self) -> Vec> { std::mem::take(&mut self.out) @@ -922,7 +933,7 @@ impl Router { /// The identity this node names for `id` in a request to `next`: the one /// `next` advertised, if this node qualified it. - fn name(&self, next: N, id: Id) -> Id { + pub fn name(&self, next: N, id: Id) -> Id { self.filed.get(&(id, next)).copied().unwrap_or(id) } @@ -1154,6 +1165,7 @@ impl Router { { self.touched += ids.len(); } + self.changed.extend(&ids); for id in ids { self.prune(id); self.advertise(id, &peers); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs new file mode 100644 index 00000000000..b151608c50c --- /dev/null +++ b/rs/moq-net/src/route/node.rs @@ -0,0 +1,411 @@ +//! One origin node's router, shared by every lite-07 announce stream on its origin. +//! +//! Each announce stream is its own neighbor key, one per direction: a stream this +//! node asked for (it receives routes and claims, and sends ROUTE_REQUEST) and a +//! stream it answers (it sends routes and claims, and receives ROUTE_REQUEST and +//! the ROUTE_END echo). Keying by stream lets a replacement stream open before +//! the old one closes, so no route drops in between. +//! +//! The node does no I/O. A stream's session feeds it what the peer sent and takes +//! [`Output`]s from its queue: wire messages to send, and the shown claims it +//! serves into the origin as the next hop of their selected route. A claim moves +//! to another stream's session when its identity's selected route does, and the +//! old session lets go only once the new one serves it, so the origin always +//! holds a route for it. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + sync::{Arc, Mutex, MutexGuard}, +}; + +use crate::{Instance, Path}; + +use rand::RngExt; + +use super::{Error, Id, Link, Metric, Out, QUALIFIED, Reach, Rev, Router, Seqno}; + +/// One announce stream in one direction. +pub(crate) type Key = u64; + +/// A claim's attributes on the route layer. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Claim { + pub instance: Instance, + pub epoch: u64, + /// The origin's Announce Cost. + pub cost: u64, +} + +/// A shown claim as the origin serves it through the selected route. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Served { + pub claim: Claim, + /// The selected route's metric: a path through the claim costs the Announce + /// Cost plus this. + pub metric: Metric, + /// Whether a usable route serves it; a held one leaves it shown but unavailable. + pub available: bool, +} + +/// What a stream's session acts on, in order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Output { + /// Send ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD, or for `None` ROUTE_END. + Route { id: Id, reach: Option }, + /// Send these claim changes for the route, closed by ANNOUNCE_FLOOR. + Batch { + id: Id, + changes: Vec<(Path, Option)>, + floor: Option, + }, + /// Send ROUTE_REQUEST, naming the identity as the peer advertised it. + Request { id: Id, seqno: Seqno }, + /// Echo the peer's ROUTE_END for the identity it named `id`. + Echo { id: Id }, + /// Serve `claim` of `id` into the origin through this stream, or update how it + /// is served. A request forwarded through it names `named`, the identity as + /// the peer advertised it. Confirm with [`Stream::serving`]. + Serve { + id: Id, + named: Id, + claim: Path, + served: Served, + }, + /// Stop serving `claim` of `id` through this stream. + Unserve { id: Id, claim: Path }, +} + +/// The node every lite-07 session of one origin shares. +#[derive(Clone)] +pub(crate) struct Node { + state: Arc>, +} + +impl std::fmt::Debug for Node { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Node").finish_non_exhaustive() + } +} + +struct State { + router: Router, + next_key: Key, + queues: HashMap>, + /// Each shown claim of another identity, and the streams serving it. + homes: BTreeMap<(Id, Path), Home>, +} + +/// Where one shown claim is served. +#[derive(Default)] +struct Home { + /// The stream it is served through, and what it was last sent. + key: Option<(Key, Served)>, + /// Streams that served it before and let go once `key` confirms. + leaving: BTreeSet, + /// Whether `key` confirmed it serves the claim. + confirmed: bool, +} + +impl Node { + /// A node declaring a fresh identity, `seed` keying its rotation. + pub fn new(id: Id, seed: u64) -> Self { + debug_assert!(id != 0 && id & QUALIFIED == 0, "a declared identity is non-zero and unqualified"); + let mut router = Router::new(seed); + router.originate(id, 0); + Self { + state: Arc::new(Mutex::new(State { + router, + next_key: 1, + queues: HashMap::new(), + homes: BTreeMap::new(), + })), + } + } + + /// A node with a random identity. + pub fn random() -> Self { + let mut rng = kio::clock::rng(); + let id = rng.random_range(1..QUALIFIED); + Self::new(id, rng.random()) + } + + fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|poison| poison.into_inner()) + } + + /// The identity this node declares, which a rotation replaces. + pub fn declared(&self) -> Id { + self.lock().router.declared().expect("a node always declares an identity") + } + + /// This node's own claim at `path` changed: `None` withdraws it. + pub fn own(&self, path: Path, claim: Option) { + let mut state = self.lock(); + let changed = match claim { + Some(claim) => { + let current = state.router.declared().and_then(|id| state.router.view(id)?.items.get(&path).cloned()); + current.as_ref() != Some(&claim) && state.router.announce(path, claim).is_ok() + } + None => state.router.unannounce(path).is_ok(), + }; + if changed { + state.flush(); + } + } + + /// Register a stream this node asked the peer for under `prefix`. + pub fn receive(&self, link: Link, prefix: Path) -> Result { + self.register(link, |router, key| router.asks(key, prefix)) + } + + /// Register a stream answering the peer's request for `prefix`. + pub fn answer(&self, link: Link, prefix: Path) -> Result { + self.register(link, |router, key| router.peer_interest(key, prefix)) + } + + fn register( + &self, + link: Link, + open: impl FnOnce(&mut Router, Key) -> Result<(), Error>, + ) -> Result { + let mut state = self.lock(); + let key = state.next_key; + state.next_key += 1; + state.router.connect(key, link)?; + let queue = kio::Queue::new(); + state.queues.insert(key, queue.clone()); + if let Err(err) = open(&mut state.router, key) { + state.router.disconnect(key); + state.queues.remove(&key); + return Err(err); + } + // The node's own set of sessions changed. + let _ = state.router.bump(); + state.flush(); + Ok(Stream { + node: self.clone(), + key, + queue, + }) + } + + #[cfg(test)] + pub(crate) fn shown(&self) -> BTreeMap<(Id, Path), (Claim, bool)> { + (self.lock().router.shown().iter()) + .map(|(key, shown)| (key.clone(), (shown.item.clone(), shown.available))) + .collect() + } +} + +/// One announce stream's key on the node. Dropping it disconnects the stream. +pub(crate) struct Stream { + node: Node, + key: Key, + queue: kio::Queue, +} + +impl std::fmt::Debug for Stream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Stream").field("key", &self.key).finish_non_exhaustive() + } +} + +impl Stream { + /// The next output for this stream's session. + pub fn poll_output(&self, waiter: &kio::Waiter) -> std::task::Poll { + match self.queue.poll_pop(waiter) { + std::task::Poll::Ready(Ok(output)) => std::task::Poll::Ready(output), + // The queue closes only once the stream dropped. + _ => std::task::Poll::Pending, + } + } + + /// The next output already queued, without waiting. + pub fn try_output(&self) -> Option { + self.queue.try_pop().ok().flatten() + } + + fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { + let mut state = self.node.lock(); + let out = f(&mut state.router, self.key); + state.flush(); + out + } + + /// ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD (`Held`), or ROUTE_END (`None`) for + /// the identity the peer names `named`. + pub fn route(&self, named: Id, reach: Option) { + self.with(|router, key| router.route(key, named, reach)); + } + + /// A batch of claim changes for the route the peer names `named`, closed by + /// its floor. + pub fn batch(&self, named: Id, changes: Vec<(Path, Option)>, floor: Option) { + self.with(|router, key| { + let interest = router.interest(); + router.batch(key, named, changes, floor, interest) + }); + } + + /// The peer sent ROUTE_REQUEST for `id`, as this node advertised it. + pub fn request(&self, id: Id, seqno: Seqno) { + self.with(|router, key| router.request(key, id, seqno)); + } + + /// The peer echoed this node's ROUTE_END for `id`. + pub fn echo(&self, id: Id) { + self.with(|router, key| router.echo(key, id)); + } + + /// The peer sent GOAWAY: its routes rank below every other. + pub fn goaway(&self) { + self.with(|router, key| router.goaway(key)); + } + + /// This stream's session now serves `claim` of `id`, so wherever it was served + /// before lets go. + pub fn serving(&self, id: Id, claim: &Path) { + let mut state = self.node.lock(); + let State { homes, queues, .. } = &mut *state; + let Some(home) = homes.get_mut(&(id, claim.clone())) else { + return; + }; + if home.key.as_ref().is_none_or(|(key, _)| *key != self.key) || home.confirmed { + return; + } + home.confirmed = true; + for key in std::mem::take(&mut home.leaving) { + push(queues, key, Output::Unserve { id, claim: claim.clone() }); + } + } +} + +impl Drop for Stream { + fn drop(&mut self) { + self.queue.close(); + let mut state = self.node.lock(); + state.queues.remove(&self.key); + state.router.disconnect(self.key); + for home in state.homes.values_mut() { + home.leaving.remove(&self.key); + if home.key.as_ref().is_some_and(|(key, _)| *key == self.key) { + // Served nowhere until the router moves it: nothing is left here to wait for. + home.key = None; + home.confirmed = false; + } + } + let _ = state.router.bump(); + state.flush(); + } +} + +fn push(queues: &HashMap>, key: Key, output: Output) { + if let Some(queue) = queues.get(&key) { + let _ = queue.try_push(output); + } +} + +impl State { + /// Route the router's messages and shown claims to the streams. + fn flush(&mut self) { + for out in self.router.drain() { + let (key, output) = match out { + Out::Route { to, id, reach } => (to, Output::Route { id, reach }), + Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), + Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Echo { to, id } => (to, Output::Echo { id }), + }; + push(&self.queues, key, output); + } + // The shown claims below are read whole; the events only say which changed. + self.router.events(); + let changed = self.router.changed(); + let declared = self.router.declared(); + for id in changed { + if Some(id) != declared { + self.home(id); + } + } + } + + /// Serve each shown claim of `id` through the stream its selected route + /// goes through, keeping it where it is while no route is selected. + fn home(&mut self, id: Id) { + let selected = self.router.selected(id); + let claims: Vec<(Path, Claim, bool)> = (self.router.shown().range((id, Path::empty())..)) + .take_while(|((shown, _), _)| *shown == id) + .map(|((_, claim), shown)| (claim.clone(), shown.item.clone(), shown.available)) + .collect(); + let mut gone: BTreeSet = self + .homes + .range((id, Path::empty())..) + .take_while(|((home, _), _)| *home == id) + .map(|((_, claim), _)| claim.clone()) + .collect(); + for (claim, item, available) in claims { + gone.remove(&claim); + let home = self.homes.entry((id, claim.clone())).or_default(); + let current = home.key.as_ref().map(|(key, served)| (*key, served.metric)); + let Some((key, metric)) = selected + .and_then(|route| Some((route.next?, route.metric))) + .or(current) + else { + continue; + }; + let served = Served { + claim: item, + metric, + available, + }; + match &home.key { + Some((at, sent)) if *at == key && *sent == served => continue, + Some((at, _)) if *at == key => {} + Some((at, _)) => { + if home.confirmed { + home.leaving.insert(*at); + } + home.confirmed = false; + } + None => home.confirmed = false, + } + home.key = Some((key, served.clone())); + let named = self.router.name(key, id); + push( + &self.queues, + key, + Output::Serve { + id, + named, + claim, + served, + }, + ); + } + for claim in gone { + let Some(home) = self.homes.remove(&(id, claim.clone())) else { + continue; + }; + for key in home.key.map(|(key, _)| key).into_iter().chain(home.leaving) { + push(&self.queues, key, Output::Unserve { id, claim: claim.clone() }); + } + } + // Nothing left to recover: let the identity go. + let shows = (self.router.shown().range((id, Path::empty())..).next()).is_some_and(|((shown, _), _)| *shown == id); + if !self.router.holds(id) && !shows { + self.router.collect(id); + } + } +} + +/// The lite-06 hop a router identity travels as, so a relay translating its routes +/// for a lite-06 peer recognizes them coming back. +pub(crate) fn hop(id: Id) -> crate::Hop { + // FNV-1a over the identity, folded into the 53 bits a hop id keeps. + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in id.to_le_bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + let id = (hash & ((1 << 53) - 1)).max(1); + crate::Hop::new(id).expect("a non-zero hop id") +} From 97419faf5f090d5830d6fb093672fa10416dee3b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:26:22 -0800 Subject: [PATCH 04/61] test(net): the route node serves claims through their route's stream Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index b151608c50c..93918d36f48 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -409,3 +409,139 @@ pub(crate) fn hop(id: Id) -> crate::Hop { let id = (hash & ((1 << 53) - 1)).max(1); crate::Hop::new(id).expect("a non-zero hop id") } + +#[cfg(test)] +mod tests { + use super::*; + use crate::route::Advert; + + fn p(path: &'static str) -> Path { + Path::from_static(path) + } + + fn claim(instance: &Instance, cost: u64) -> Claim { + Claim { + instance: instance.clone(), + epoch: 0, + cost, + } + } + + fn outputs(stream: &Stream) -> Vec { + std::iter::from_fn(|| stream.try_output()).collect() + } + + /// Deliver everything `from` would send on the wire to the stream receiving it, + /// returning what else it queued. + fn pump(from: &Stream, to: &Stream) -> Vec { + let mut rest = Vec::new(); + for output in outputs(from) { + match output { + Output::Route { id, reach } => to.route(id, reach), + Output::Batch { id, changes, floor } => to.batch(id, changes, floor), + other => rest.push(other), + } + } + rest + } + + /// The serve commands among a stream's outputs. + fn served(outputs: Vec) -> Vec { + (outputs.into_iter()) + .filter(|output| matches!(output, Output::Serve { .. } | Output::Unserve { .. })) + .collect() + } + + #[test] + fn a_claim_is_served_through_the_stream_its_route_arrived_on() { + let instance = Instance::random(); + let (origin, relay) = (Node::new(1, 1), Node::new(2, 2)); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 5))); + pump(&answer, &receive); + assert_eq!( + served(outputs(&receive)), + [Output::Serve { + id: 1, + named: 1, + claim: p("live/x"), + served: Served { + claim: claim(&instance, 5), + metric: 1, + available: true, + }, + }] + ); + + origin.own(p("live/x"), None); + pump(&answer, &receive); + assert_eq!( + served(outputs(&receive)), + [Output::Unserve { + id: 1, + claim: p("live/x") + }] + ); + } + + #[test] + fn a_claim_moves_to_a_cheaper_stream_once_that_one_serves_it() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let far = relay + .receive( + Link { + cost: 3, + ..Link::default() + }, + Path::empty(), + ) + .unwrap(); + let far_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0))); + pump(&far_answer, &far); + assert!(matches!(served(outputs(&far))[..], [Output::Serve { .. }])); + far.serving(1, &p("live/x")); + + let near = relay.receive(Link::default(), Path::empty()).unwrap(); + let near_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + pump(&far_answer, &far); + pump(&near_answer, &near); + assert!(matches!(&served(outputs(&near))[..], [Output::Serve { served, .. }] if served.metric == 1)); + assert!(served(outputs(&far)).is_empty(), "the old stream serves until the new one does"); + near.serving(1, &p("live/x")); + assert_eq!( + served(outputs(&far)), + [Output::Unserve { + id: 1, + claim: p("live/x") + }] + ); + } + + #[test] + fn a_held_route_keeps_serving_where_it_was_but_unavailable() { + let instance = Instance::random(); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + receive.route(1, Some(Reach::Finite(advert))); + receive.batch(1, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + assert!(matches!(served(outputs(&receive))[..], [Output::Serve { .. }])); + receive.serving(1, &p("live/x")); + + receive.route(1, Some(Reach::Held)); + assert!(matches!( + &served(outputs(&receive))[..], + [Output::Serve { served, .. }] if !served.available + )); + receive.route(1, None); + assert!(matches!(served(outputs(&receive))[..], [Output::Unserve { .. }])); + } +} From 30a653f15874161d994828360993aef61d370b2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 23:53:59 -0800 Subject: [PATCH 05/61] feat(relay)!: remove cluster.id and refuse zero-cost links Every relay now takes a random Hop ID on each start: `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id` are gone and refused as unknown settings, matching the route layer's per-process node identity. A cluster link costs at least 1, the price of an unpriced link. A peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is a configuration error, as is a LAN-advertised `cluster.node` whose cost is 0 or malformed. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 19 ++- doc/bin/relay/config.md | 1 - rs/moq-cli/MIGRATE.md | 10 ++ rs/moq-cli/src/args.rs | 4 +- rs/moq-relay/MIGRATE.md | 14 +++ rs/moq-relay/benches/session.rs | 1 - rs/moq-relay/src/cache.rs | 6 +- rs/moq-relay/src/cluster.rs | 109 +++++++----------- rs/moq-relay/src/config.rs | 20 +--- rs/moq-relay/src/settings.rs | 3 - .../tests/integration/cluster_unknown.rs | 10 +- rs/moq-relay/tests/integration/embed.rs | 8 +- .../tests/integration/released_cli.rs | 1 - .../tests/integration/upstream_links.rs | 53 +++++---- 14 files changed, 122 insertions(+), 137 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index c0f6fc071f2..e1bead6ae27 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -153,9 +153,10 @@ connect = [{ url = "tls://core-a.internal:4443/", upstream = true }] ## Link costs Add `?cost=N` to a peer URL to route by price instead of hop count. An unpriced -link costs 1, which reproduces plain hop counting. Each relay adds the price of -the link an announcement arrived on before forwarding it, so a route's cost is -the sum of what it crossed. +link costs 1, which reproduces plain hop counting, and 1 is also the minimum: a +cost of 0 is refused. Each relay adds the price of the link an announcement +arrived on before forwarding it, so a route's cost is the sum of what it +crossed. Routing prefers the longest covering prefix, then a fully identified hop list over one holding an anonymous hop (0), then the lowest cost, then the shortest @@ -167,7 +168,7 @@ restarts takes over from its lingering old session at once. ```toml [cluster] -connect = ["https://sibling.same-dc/?cost=0", "https://us-east.example.com/?cost=10"] +connect = ["https://sibling.same-dc/?cost=1", "https://us-east.example.com/?cost=10"] ``` The same policy reads as an object, the only form that accepts `token` and @@ -176,7 +177,7 @@ The same policy reads as an object, the only form that accepts `token` and ```toml [cluster] connect = [ - { url = "https://sibling.same-dc/", cost = 0 }, + { url = "https://sibling.same-dc/", cost = 1 }, { url = "https://us-east.example.com/", cost = 10, token = "PEER_JWT" }, ] ``` @@ -253,11 +254,9 @@ decides which relays receive the cluster token. ## Identity Each relay has a Hop ID: the value it adds to a route's hop list for loop -detection and shortest-path routing. It is random on every start, which is fine -for loop detection but makes a restarted relay look like a new node. Set -`cluster.id` to a stable non-zero 64-bit integer to pin it: below 2^53 if older -browser clients decode it, and below 2^62 to keep loop detection over moq-lite-06 -and older links, which carry a larger id as 0. +detection and shortest-path routing. It is random on every start, so a +restarted relay is a new node to the rest of the mesh, and there is no setting +to pin it. ## Failure detection diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 20b76f970c7..25afa723fda 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -305,7 +305,6 @@ See [Authentication](/bin/relay/auth). connect = ["https://us-east.example.com/?cost=10"] # Peers to dial. ?cost prices the link, or use {url, cost, egress, token, upstream} objects. node = "https://us-west.example.com/" # This relay's own URL. token = "cluster.jwt" # JWT for dials without an inline ?jwt=. -id = 12345 # Stable Hop ID across restarts. route_key = "route.key" # Route key; relays sharing the file fail over as one to scoped peers. max_hops = 32 # Longest accepted/forwarded hop path (1 to 32). diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index e29b62468e9..3070c7399d2 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,16 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `--cluster-id` is removed + +Remove `--cluster-id` and `MOQ_CLUSTER_ID`: `moq` refuses both as unknown settings. +The Hop ID is random on every run and nothing pins it. + +### Zero-cost cluster links are refused + +A `--cluster-connect` or `--cluster-api-url` peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is refused, as is a `--cluster-node` carrying `?cost=0` under `--cluster-lan`. +A link costs at least 1, the price of an unpriced link: use `?cost=1` for a same-datacenter sibling. + ### The iroh transport is removed ([#7345](https://github.com/moq-dev/moq/pull/7345)) The `iroh` Cargo feature and the `--iroh-*` flags are gone, and a `MOQ_IROH_*` variable stops startup like any other unknown `MOQ_*` variable. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 3888d4bccdb..b366ff3357c 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -555,7 +555,7 @@ impl MoqSide { /// The cluster this process publishes and subscribes on, with the driver that /// must run for as long as it serves. Built once; the origin is its origin, whose - /// Hop ID is random per run unless `--cluster-id` names the node. + /// Hop ID is random per run. pub fn cluster(&self) -> anyhow::Result<(moq_relay::cluster::Cluster, moq_relay::cluster::Driver)> { moq_relay::cluster::Cluster::new(moq_relay::cluster::Options::new(self.cluster.clone())) } @@ -2682,7 +2682,7 @@ mod tests { ("--cluster-api-url", "https://api.example/peers", "--cluster-api-url"), ("--cluster-node", "https://self.example", "--cluster-node"), ("--cluster-token", "cluster.jwt", "--cluster-token"), - ("--cluster-id", "1", "--cluster-id"), + ("--cluster-max-hops", "8", "--cluster-max-hops"), ("--cluster-tier", "internal", "--cluster-tier"), ] { let cli = Invocation::try_parse_from(["moq", flag, value, "auth", "generate"]).unwrap(); diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 6f06755853d..3823718c8af 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,20 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `cluster.id` is removed + +Remove `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id`; in Rust, drop `cluster::Config::id`. +A relay that still sets one stops at startup: the flag, variable, and key are refused as unknown settings. +Each relay picks a random Hop ID on every start and nothing pins it, so a restarted relay is a new node to the mesh. +`Cluster::id()` still reports the one in use. + +### Zero-cost cluster links are refused + +A link costs at least 1, the price of an unpriced link, since routing needs every link to cost something. +A peer priced at 0 (`?cost=0` on its URL, or an object `cost` or `egress` of 0) stops startup in `cluster.connect` and rejects a `cluster.api.url` update, which keeps the last good list. +With LAN discovery on, a `cluster.node` whose `?cost=` is 0 or malformed stops startup, and a LAN peer advertising one is skipped. +Price a same-datacenter sibling at 1, and raise the other links' costs if it must stay relatively cheaper. + ### Shutdown reports deadline cuts separately from end reports `shutdown::Observer::drained()` now returns `shutdown::Drained` instead of `()`. diff --git a/rs/moq-relay/benches/session.rs b/rs/moq-relay/benches/session.rs index bfa7b429a41..c8a7e0923de 100644 --- a/rs/moq-relay/benches/session.rs +++ b/rs/moq-relay/benches/session.rs @@ -76,7 +76,6 @@ impl Backend for Relay { let cache = config.init().unwrap(); let pool = cache.pool.clone(); let mut options = cluster::Options::default(); - options.config.id = Some(id); // The shared sweeps include more announcements than a default ordinary client. options.session.announces = options.peer.announces; let mut stats = stats::Config::default(); diff --git a/rs/moq-relay/src/cache.rs b/rs/moq-relay/src/cache.rs index 2cf25f2ad5c..281a101fd4e 100644 --- a/rs/moq-relay/src/cache.rs +++ b/rs/moq-relay/src/cache.rs @@ -337,12 +337,12 @@ mod tests { async fn setup_failure_stops_governor() { let before = spawned(); - // `--cluster-id 0` is rejected, so the cache is dropped before it is attached. + // `--cluster-max-hops 0` is rejected, so the cache is dropped before it is attached. let cluster = crate::cluster::Config { - id: Some(0), + max_hops: Some(0), ..Default::default() }; - assert!(attach(&governed(), cluster).is_err(), "cluster id 0 is rejected"); + assert!(attach(&governed(), cluster).is_err(), "a hop limit of 0 is rejected"); settle().await; assert_eq!(spawned(), before, "a failed setup should not strand the governor"); diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 86edbb4178f..7e6ff25831e 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -287,6 +287,10 @@ impl DialTarget { "cluster peer sets token alongside a URL ?jwt=; use one or the other" ); } + anyhow::ensure!( + peer.cost != Some(0) && peer.egress != Some(0), + "cluster peer sets a cost or egress of 0; a link costs at least 1, the price of an unpriced link" + ); let cost = peer.cost.or(take_cost(&mut url)?); // An unpriced link costs 1 (see `moq_net::Client::with_cost`), so `egress` // alone is symmetric only at that price. @@ -722,23 +726,6 @@ impl Drop for Held { #[serde(default, deny_unknown_fields)] #[non_exhaustive] pub struct Config { - /// Fixed origin (hop) id for this relay, identifying it in the hop chains - /// carried on each broadcast for loop detection and shortest-path routing. - /// - /// Unset (the default) picks a fresh random id on every start. Set it to give - /// a node a stable identity across restarts. Must be non-zero; 0 errors at - /// startup. Keep it below 2^53 for compatibility with older `@moq/lite` JS - /// clients, which decode hop ids as a `u53` and reject anything larger. A - /// moq-lite-06 or older link carries an id at or above 2^62 as 0, losing loop - /// detection through this relay on that link. - #[usage( - name = "cluster-id", - long = "cluster-id", - env = "MOQ_CLUSTER_ID", - setting = "cluster.id" - )] - pub id: Option, - /// Route key file: the Ed25519 key this relay proves to every moq-lite-07 peer, /// which files the routes this relay sends under it when this relay's grant is scoped. /// @@ -770,11 +757,12 @@ pub struct Config { /// /// A `?cost=N` query param (or object `cost`) prices the link (moq-lite-06+): /// every announcement crossing it adds `N` to its route cost, so routing - /// prefers cheap paths over short ones. Use `0` for a same-datacenter sibling - /// and something large for a metered backbone; an unpriced link costs 1, - /// which reproduces plain hop counting. The param is consumed by this relay, - /// not sent to the peer. Object `egress` defaults to `cost`; anything else is - /// rejected until asymmetric routing lands. + /// prefers cheap paths over short ones. The minimum and the price of an + /// unpriced link is 1, which reproduces plain hop counting: use it for a + /// same-datacenter sibling and something large for a metered backbone. A + /// cost of 0 is refused. The param is consumed by this relay, not sent to the + /// peer. Object `egress` defaults to `cost`; anything else is rejected until + /// asymmetric routing lands. #[usage( name = "cluster-connect", long = "cluster-connect", @@ -1379,8 +1367,9 @@ impl Cluster { /// and stops publishing stats. A stats publisher panic ends the driver like /// its other work. Needs no Tokio runtime. /// - /// Errors if `config.id` is set to 0. An unset id picks a fresh random origin. - /// Errors too if the `peer.quic` receive window does not fit a QUIC varint, + /// The origin takes a fresh random Hop ID on every call. + /// + /// Errors if the `peer.quic` receive window does not fit a QUIC varint, /// `config.tier` is not a valid stats tier, or the stats config is invalid. pub fn new(options: Options) -> anyhow::Result<(Self, Driver)> { let Options { @@ -1390,10 +1379,6 @@ impl Cluster { peer, stats, } = options; - let id = match config.id { - Some(id) => Hop::new(id).context("--cluster-id must be non-zero")?, - None => Hop::random(), - }; let deprecated = config.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Reject a repeated static identity with conflicting policy up front, @@ -1406,7 +1391,7 @@ impl Cluster { crate::configured_tier(config.tier.clone()).context("invalid --cluster-tier")?; peer.quic.validate()?; let mut work = kio::Tasks::new(); - let mut origin_config = origin::Config::new(id); + let mut origin_config = origin::Config::new(Hop::random()); origin_config.route_key = Some(match &config.route_key { Some(path) => (&moq_crypto::sign::SecretKey::load_or_generate(path) .with_context(|| format!("--cluster-route-key {}", path.display()))?) @@ -1432,7 +1417,7 @@ impl Cluster { work.push(Child::new("stats publisher", publisher.run())); let stats = stats_producer.registry().clone(); let nodes = crate::nodes::Nodes::default(); - tracing::info!(hop_id = %origin.hop(), configured = config.id.is_some(), "cluster initialized"); + tracing::info!(hop_id = %origin.hop(), "cluster initialized"); let cluster = Cluster { session_limits: session.resolve(), peer_limits: peer.resolve(), @@ -2316,6 +2301,8 @@ async fn lan_discovery( Only `?cost=` may be advertised; pass credentials with `--cluster-token` instead.", published.join(", ") ); + // Every LAN peer skips an advertised cost that is malformed or 0, so refuse it here. + take_cost(&mut url.clone()).context("invalid --cluster-node")?; config = config.with_node(url); } if let Some(secret) = secret { @@ -2330,7 +2317,9 @@ async fn lan_discovery( /// The param is dial-side configuration, not something the peer reads off the /// URL (it rides SETUP instead), so it is stripped before connecting. An /// unparseable value is an error rather than a silent default: a mispriced link -/// skews routing for every broadcast crossing it. +/// skews routing for every broadcast crossing it. So is 0: route selection needs +/// every link to cost something, or a route could match its own feasibility +/// distance. fn take_cost(url: &mut Url) -> anyhow::Result> { let Some(value) = url .query_pairs() @@ -2341,6 +2330,10 @@ fn take_cost(url: &mut Url) -> anyhow::Result> { let cost: u64 = value .parse() .with_context(|| format!("invalid cost {value:?} on cluster peer URL"))?; + anyhow::ensure!( + cost > 0, + "cost 0 on cluster peer URL; a link costs at least 1, the price of an unpriced link" + ); let remaining: Vec<(String, String)> = url .query_pairs() @@ -2770,8 +2763,8 @@ mod tests { /// silent default. #[test] fn cost_param_is_consumed() { - let mut url = Url::parse("https://peer.example/?jwt=abc&cost=0").unwrap(); - assert_eq!(take_cost(&mut url).unwrap(), Some(0)); + let mut url = Url::parse("https://peer.example/?jwt=abc&cost=1").unwrap(); + assert_eq!(take_cost(&mut url).unwrap(), Some(1)); assert_eq!(url.as_str(), "https://peer.example/?jwt=abc"); let mut url = Url::parse("https://peer.example/").unwrap(); @@ -3356,18 +3349,6 @@ mod tests { assert!(commands.is_empty(), "an invalid list must not reach the dial map"); } - #[tokio::test] - async fn cluster_id_sets_origin() { - for id in [42, u64::MAX] { - let cluster = new_cluster(Config { - id: Some(id), - ..Default::default() - }) - .expect("valid id"); - assert_eq!(cluster.origin.hop().id(), id); - } - } - /// Cache settings land on the one origin serving and stats share. A handle /// cloned at construction stays on that origin. #[tokio::test] @@ -3380,22 +3361,16 @@ mod tests { let cache = cache.init().expect("cache"); let pool = cache.pool.clone(); - let cluster = Cluster::spawn( - Options::new(Config { - id: Some(42), - ..Default::default() - }) - .with_cache(cache) - .with_stats(crate::stats::Config { + let cluster = Cluster::spawn(Options::new(Config::default()).with_cache(cache).with_stats( + crate::stats::Config { enabled: true, node: Some("test".to_string()), ..Default::default() - }), - ) + }, + )) .expect("cluster"); let origin = cluster.origin.clone(); - assert_eq!(origin.hop().id(), 42); assert_eq!(origin.config().cache_duration, duration); assert_eq!(origin.config().pool.expiry(), Some(duration)); @@ -3423,17 +3398,21 @@ mod tests { .unwrap(); } - /// The reserved `cluster.id` 0 is rejected, since it cannot be excluded for loop - /// detection. + /// A link costs at least 1, so a static peer priced at 0 stops startup, in the + /// URL or the object form. #[test] - fn cluster_id_zero_errors() { - let err = new_cluster(Config { - id: Some(0), - ..Default::default() - }) - .err() - .expect("should error"); - assert!(format!("{err}").contains("--cluster-id"), "got: {err}"); + fn zero_cost_peer_errors() { + for peer in [ + Peer::new("https://peer.example/?cost=0"), + Peer::new("https://peer.example/").with_cost(0), + Peer::new("https://peer.example/").with_egress(0), + ] { + let config = Config { + connect: vec![peer.clone()], + ..Default::default() + }; + assert!(new_cluster(config).is_err(), "{peer:?} was accepted"); + } } #[tokio::test] diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index df0c538b970..33cf5f1ee56 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1136,12 +1136,13 @@ ws = false /// An embedder can merge its relay fragment without parsing the binary CLI again. #[test] fn merge_into_relay_fragment() { - if !test_env::Env::without(&["MOQ_CLUSTER_ID"]).isolate() { + if !test_env::Env::without(&["MOQ_CLUSTER_MAX_HOPS"]).isolate() { return; } let (parsed, cli) = - Cli::parse_from_with_settings(&[std::ffi::OsStr::new("--cluster-id"), std::ffi::OsStr::new("9")]).unwrap(); - let file = toml::from_str::("[cluster]\nid = 7\n").unwrap(); + Cli::parse_from_with_settings(&[std::ffi::OsStr::new("--cluster-max-hops"), std::ffi::OsStr::new("9")]) + .unwrap(); + let file = toml::from_str::("[cluster]\nmax_hops = 7\n").unwrap(); let source = moq_tokio::cli::FileSource { path: std::path::Path::new("relay.toml"), value: &file, @@ -1150,8 +1151,8 @@ ws = false config .merge_into(&cli, &usage_rs::config::EnvLayer::from_process(), Some(source)) .unwrap(); - assert_eq!(config.cluster.id, Some(9)); - assert_eq!(config.source("cluster.id"), Some("--cluster-id")); + assert_eq!(config.cluster.max_hops, Some(9)); + assert_eq!(config.source("cluster.max_hops"), Some("--cluster-max-hops")); } /// One source a merged setting is read from, and what it must end up as. @@ -1174,7 +1175,6 @@ ws = false "MOQ_CACHE_DURATION", "MOQ_CONNECT_TIMEOUT", "MOQ_CLUSTER_TIER", - "MOQ_CLUSTER_ID", "MOQ_CLUSTER_MAX_HOPS", "MOQ_QUIC_RECEIVE_WINDOW", "MOQ_CONNECT_TLS_SYSTEM_ROOTS", @@ -1288,14 +1288,6 @@ ws = false check: |config| assert_eq!(config.cluster.tier.as_deref(), Some("region")), source: "tier", }, - Case { - name: "optional-int-from-file", - toml: "[cluster]\nid = 12345\n", - cli: &[], - key: "cluster.id", - check: |config| assert_eq!(config.cluster.id, Some(12345)), - source: "id", - }, Case { name: "hop-limit-from-file", toml: "[cluster]\nmax_hops = 8\n", diff --git a/rs/moq-relay/src/settings.rs b/rs/moq-relay/src/settings.rs index 3b798f655e9..7ab93dc6d04 100644 --- a/rs/moq-relay/src/settings.rs +++ b/rs/moq-relay/src/settings.rs @@ -71,9 +71,6 @@ struct Runtime { #[derive(usage_rs::Config)] #[usage(prefix = "cluster")] struct Cluster { - #[usage(env = "MOQ_CLUSTER_ID", cli("--cluster-id"))] - id: Option, - #[usage(env = "MOQ_CLUSTER_ROUTE_KEY", cli("--cluster-route-key"))] route_key: Option, diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 813e6f068f2..1045ab7944f 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -28,7 +28,7 @@ struct Node { task: tokio::task::JoinHandle<()>, } -async fn spawn_relay(id: u64, connect: Vec, cluster_version: Option) -> Node { +async fn spawn_relay(connect: Vec, cluster_version: Option) -> Node { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut config = Config::default(); @@ -38,7 +38,6 @@ async fn spawn_relay(id: u64, connect: Vec, cluster_version: Option) -> Self { - let a = spawn_relay(11, vec![], cluster_version).await; - let b = spawn_relay(12, vec![format!("tcp://127.0.0.1:{}", a.port)], cluster_version).await; + let a = spawn_relay(vec![], cluster_version).await; + let b = spawn_relay(vec![format!("tcp://127.0.0.1:{}", a.port)], cluster_version).await; let c = spawn_relay( - 13, vec![ format!("tcp://127.0.0.1:{}", a.port), format!("tcp://127.0.0.1:{}", b.port), @@ -118,7 +116,7 @@ impl Mesh { cluster_version, ) .await; - let d = spawn_relay(14, vec![format!("tcp://127.0.0.1:{}", c.port)], cluster_version).await; + let d = spawn_relay(vec![format!("tcp://127.0.0.1:{}", c.port)], cluster_version).await; // One client for every poll: building one loads the system trust roots. let http = reqwest::Client::new(); for (node, dialed) in [(&b, 1), (&c, 2), (&d, 1)] { diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 98580bc7f07..88d65297604 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -557,19 +557,19 @@ struct EmbeddedConfig { fn embedded_cli_merges_only_relay_settings() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("relay.toml"); - std::fs::write(&path, "[cluster]\nid = 7\n[stats]\nenabled = false\n").unwrap(); + std::fs::write(&path, "[cluster]\nmax_hops = 7\n[stats]\nenabled = false\n").unwrap(); let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[ path.as_os_str(), std::ffi::OsStr::new("--worker-name"), std::ffi::OsStr::new("recorder"), - std::ffi::OsStr::new("--cluster-id"), + std::ffi::OsStr::new("--cluster-max-hops"), std::ffi::OsStr::new("9"), ]) .expect("parse embedding CLI"); parsed.relay.merge(&cli).unwrap(); assert_eq!(parsed.worker_name.as_deref(), Some("recorder")); - assert_eq!(parsed.relay.cluster.id, Some(9)); - assert_eq!(parsed.relay.source("cluster.id"), Some("--cluster-id")); + assert_eq!(parsed.relay.cluster.max_hops, Some(9)); + assert_eq!(parsed.relay.source("cluster.max_hops"), Some("--cluster-max-hops")); assert!(!parsed.relay.stats.enabled); assert!(parsed.relay.source("stats.enabled").unwrap().contains("enabled")); // The explicit file value equals the runtime default but still has file provenance. diff --git a/rs/moq-relay/tests/integration/released_cli.rs b/rs/moq-relay/tests/integration/released_cli.rs index 38a9707554b..262f9aa3acd 100644 --- a/rs/moq-relay/tests/integration/released_cli.rs +++ b/rs/moq-relay/tests/integration/released_cli.rs @@ -54,7 +54,6 @@ const RELEASED: &[(&str, Option<&str>)] = &[ ("client-version", Some("MOQ_CLIENT_VERSION")), ("cluster-connect", Some("MOQ_CLUSTER_CONNECT")), ("cluster-connect-api", Some("MOQ_CLUSTER_CONNECT_API")), - ("cluster-id", Some("MOQ_CLUSTER_ID")), ("cluster-node", Some("MOQ_CLUSTER_NODE")), ("cluster-tier", Some("MOQ_CLUSTER_TIER")), ("cluster-token", Some("MOQ_CLUSTER_TOKEN")), diff --git a/rs/moq-relay/tests/integration/upstream_links.rs b/rs/moq-relay/tests/integration/upstream_links.rs index 71b97ec486a..8f640d90c89 100644 --- a/rs/moq-relay/tests/integration/upstream_links.rs +++ b/rs/moq-relay/tests/integration/upstream_links.rs @@ -52,9 +52,10 @@ where .expect("test thread panicked"); } -/// One relay: its origin, listen port, and every task it runs. Dropping it -/// aborts them all, closing its sessions like a crash. +/// One relay: its Hop ID, origin, listen port, and every task it runs. Dropping +/// it aborts them all, closing its sessions like a crash. struct Node { + id: u64, origin: origin::Producer, port: u16, _tasks: tokio::task::JoinSet<()>, @@ -82,7 +83,7 @@ fn grant(path: &str) -> Grant { const LITE_07: &str = "moq-lite-07-wip"; -async fn node(id: u64, connect: Vec) -> Node { +async fn node(connect: Vec) -> Node { let _ = rind_rustls::default_provider().unwrap().install_default(); let mut tasks = tokio::task::JoinSet::new(); @@ -106,9 +107,9 @@ async fn node(id: u64, connect: Vec) -> Node { client.version = vec![LITE_07.parse().expect("lite-07")]; let mut config = cluster::Config::default(); - config.id = Some(id); config.connect = connect; let cluster = crate::spawn_cluster(cluster::Options::new(config)).with_connect(client, Default::default()); + let id = cluster.id(); let (auth, mut admissions) = auth::Auth::embedded(format!("node-{id}")); tasks.spawn(async move { @@ -135,6 +136,7 @@ async fn node(id: u64, connect: Vec) -> Node { }); Node { + id, origin: cluster.origin.clone(), port, _tasks: tasks, @@ -241,12 +243,6 @@ fn has(routes: &BTreeMap>, paths: &[&str]) -> bool { paths.iter().all(|path| routes.contains_key(*path)) } -const R: u64 = 1; -const C1: u64 = 11; -const C2: u64 = 12; -const E1: u64 = 21; -const E2: u64 = 22; - /// Two regions: two edges whose links to two cores are upstream, and one relay /// that also serves clients. Both edges send a path to the same core, an edge's /// broadcast reaches the far relay, and losing a core moves only its paths. @@ -256,13 +252,15 @@ fn tiered_regions() { } async fn tiered_regions_inner() { - let r = node(R, vec![]).await; - let c1 = node(C1, vec![r.dial(false, false)]).await; - let c2 = node(C2, vec![r.dial(false, false)]).await; - let e1 = node(E1, vec![c1.dial(true, false), c2.dial(true, false)]).await; - let e2 = node(E2, vec![c1.dial(true, false), c2.dial(true, false)]).await; - - let paths: Vec = (0..8).map(|i| format!("live/{i}")).collect(); + let r = node(vec![]).await; + let c1 = node(vec![r.dial(false, false)]).await; + let c2 = node(vec![r.dial(false, false)]).await; + let e1 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; + let e2 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; + + // Each path's core is a hash over the random Hop IDs, so enough paths that both + // cores get some: all 32 on one core is a 2^-31 chance. + let paths: Vec = (0..32).map(|i| format!("live/{i}")).collect(); let paths: Vec<&str> = paths.iter().map(String::as_str).collect(); let _published: Vec<_> = paths.iter().map(|path| publish(&r.origin, path)).collect(); routes_when("both cores learn the far region", &c1.origin, |routes| { @@ -298,21 +296,22 @@ async fn tiered_regions_inner() { // Losing a core moves only its paths. let on = |core_id: u64| -> Vec<&str> { paths.iter().copied().filter(|p| core(&at_e1, p) == core_id).collect() }; - let (on_c1, on_c2) = (on(C1), on(C2)); + let (on_c1, on_c2) = (on(c1.id), on(c2.id)); assert!( !on_c1.is_empty() && !on_c2.is_empty(), "paths all on one core: {at_e1:?}" ); + let lost = c1.id; drop(c1); let after = routes_when("edge 1 drops the lost core", &e1.origin, |routes| { - has(routes, &paths) && routes.values().all(|hops| !hops.contains(&C1)) + has(routes, &paths) && routes.values().all(|hops| !hops.contains(&lost)) }) .await; for path in &on_c2 { assert_eq!(after[*path], at_e1[*path], "{path} moved off the surviving core"); } for path in &on_c1 { - assert_eq!(after[*path], [R, C2], "{path} did not fail over"); + assert_eq!(after[*path], [r.id, c2.id], "{path} did not fail over"); } } @@ -324,11 +323,11 @@ fn edges_never_transit_between_cores() { } async fn edges_never_transit_between_cores_inner() { - let c1 = node(C1, vec![]).await; - let e1 = node(E1, vec![c1.dial(true, false)]).await; + let c1 = node(vec![]).await; + let e1 = node(vec![c1.dial(true, false)]).await; // The core dials this edge, which still treats it as upstream. - let c2 = node(C2, vec![e1.dial(false, true)]).await; - let e2 = node(E2, vec![c1.dial(true, false), c2.dial(true, false)]).await; + let c2 = node(vec![e1.dial(false, true)]).await; + let e2 = node(vec![c1.dial(true, false), c2.dial(true, false)]).await; let _core = publish(&c1.origin, "live/c1"); routes_when("edge 1 learns core 1", &e1.origin, |routes| has(routes, &["live/c1"])).await; @@ -358,9 +357,9 @@ fn uplink_transits_both_ways() { } async fn uplink_transits_both_ways_inner() { - let cdn = node(31, vec![]).await; - let uplink = node(32, vec![cdn.dial(true, false)]).await; - let mesh = node(33, vec![uplink.dial(false, false)]).await; + let cdn = node(vec![]).await; + let uplink = node(vec![cdn.dial(true, false)]).await; + let mesh = node(vec![uplink.dial(false, false)]).await; let _mesh = publish(&mesh.origin, "mesh/cam"); let _cdn = publish(&cdn.origin, "cdn/cam"); From 8059e163e41020904abd5c1a0f847ce3bf43c669 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 00:34:58 -0800 Subject: [PATCH 06/61] feat(net)!: lite-07 names a claim's route instead of its hop chain ANNOUNCE_START and ANNOUNCE_TAKEOVER carry the Route ID, ANNOUNCE_UPDATE only the Announce Cost, and ANNOUNCE_RESTART no hops; hop bases are gone. ANNOUNCE_OK has no body and SETUP no Hop parameter on lite-07, a Cost of 0 is refused, and SUBSCRIBE, FETCH, TRACK, TRACKS, and DATAGRAMS carry the Node ID. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/announce.rs | 6 +- rs/moq-net/src/coding/writer.rs | 1 + rs/moq-net/src/fuzz.rs | 29 +- rs/moq-net/src/lite/announce.rs | 352 +++++++++++++------- rs/moq-net/src/lite/compress.rs | 525 ++++++++---------------------- rs/moq-net/src/lite/datagram.rs | 9 + rs/moq-net/src/lite/fetch.rs | 18 + rs/moq-net/src/lite/instance.rs | 17 + rs/moq-net/src/lite/publisher.rs | 54 ++- rs/moq-net/src/lite/setup.rs | 98 ++++-- rs/moq-net/src/lite/subscribe.rs | 18 + rs/moq-net/src/lite/subscriber.rs | 36 +- rs/moq-net/src/lite/track.rs | 29 ++ rs/moq-net/src/lite/tracks.rs | 16 +- rs/moq-net/src/lite/version.rs | 14 +- rs/moq-net/src/path/mod.rs | 1 + rs/moq-net/src/test_interop.rs | 1 + 17 files changed, 639 insertions(+), 585 deletions(-) diff --git a/rs/moq-net/benches/announce.rs b/rs/moq-net/benches/announce.rs index c30a7bbe082..f567100d78b 100644 --- a/rs/moq-net/benches/announce.rs +++ b/rs/moq-net/benches/announce.rs @@ -9,8 +9,10 @@ //! //! - health: `/private/channel_N/stream-health-` from a few origins behind //! a shared pair of relays, the shape that motivated compression. -//! - unique: random single-segment paths and random two-hop chains, where nothing -//! is shared and every START pays the four zero base/keep bytes. +//! - unique: random single-segment paths and random two-hop chains, where no path +//! is shared and every START pays the two zero base/keep bytes. +//! +//! The hop chains travel only on lite-06; lite-07 names each claim's route instead. //! //! The byte counts print once before the timings. //! diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 01bb04c8da2..71591b52578 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -509,6 +509,7 @@ mod tests { .buffer(&lite::Track { broadcast: crate::Path::from_static("test"), instance: None, + node: 0, track: "video".into(), }) .unwrap(); diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index bc08a7261ef..a3c8d81b274 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -359,6 +359,9 @@ pub fn lite_wire(data: &[u8]) -> bool { } /// One announcement on an announce stream, resolved: what the application sees. +/// +/// The hop chain travels only on lite-06: lite-07 names the claim's route instead, so +/// its writer drops the chain and its reader decodes an empty one. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Announced { /// ANNOUNCE_START: a route's suffix and wire hop chain, taking the next id. @@ -397,19 +400,20 @@ impl AnnounceWriter { pub fn write(&mut self, announced: &Announced, data: &mut Vec) { let msg = match announced { Announced::Start(suffix, hops) => { - let (_, suffix, hops) = self.encoder.start(suffix.clone(), hops.clone()); + let (_, suffix) = self.encoder.start(suffix.clone()); // A fixed instance, so the bytes stay pinned and match what `js/net` writes. lite::AnnounceBroadcast::Active { + route: 0, instance: self.version.has_instance().then(|| "a".parse().unwrap()), epoch: 0, suffix, - hops, + hops: self.chain(hops), cost: Default::default(), } } Announced::Update(id, hops) => lite::AnnounceBroadcast::Update { id: *id, - hops: self.encoder.update(*id, hops.clone()), + hops: self.chain(hops), cost: Default::default(), }, Announced::End(id) => { @@ -420,6 +424,14 @@ impl AnnounceWriter { msg.encode(&mut Encoder::new(data, self.version.into()), self.version) .expect("could not encode an announcement"); } + + /// The hop chain as this version carries it: none on lite-07. + fn chain(&self, hops: &Hops) -> Hops { + match self.version.has_routes() { + true => Hops::default(), + false => hops.clone(), + } + } } /// Mark `track` as a copy received from a remote publisher, as a subscriber session does, for @@ -574,12 +586,10 @@ pub fn decode_announces(data: &[u8], compress: bool) -> Vec { let mut resolved = Vec::new(); while let Ok(msg) = lite::AnnounceBroadcast::decode(&mut data, version) { let announced = match msg { - lite::AnnounceBroadcast::Active { suffix, hops, .. } => decoder - .start(suffix, hops) - .map(|(suffix, hops)| Announced::Start(suffix, hops)), - lite::AnnounceBroadcast::Update { id, hops, .. } => { - decoder.update(id, hops).map(|(_, hops)| Announced::Update(id, hops)) + lite::AnnounceBroadcast::Active { suffix, hops, .. } => { + decoder.start(suffix).map(|suffix| Announced::Start(suffix, hops)) } + lite::AnnounceBroadcast::Update { id, hops, .. } => decoder.get(id).map(|_| Announced::Update(id, hops)), lite::AnnounceBroadcast::EndedId { id } => decoder.end(id).map(|_| Announced::End(id)), _ => continue, }; @@ -682,6 +692,7 @@ impl LiteSample { Self::Subscribe => lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 3, broadcast: Path::from_static("room/alice"), track: "video".into(), @@ -880,6 +891,7 @@ impl Default for Messages { lite_subscribe: lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 7, broadcast: Path::from_static("room/alice"), track: "video".into(), @@ -1062,6 +1074,7 @@ pub fn encode_subscribe(broadcast: &Path, track: &str, ietf: bool, out: &mut Vec id: 0, broadcast: broadcast.clone(), instance: None, + node: 0, track: track.into(), priority: 0, max_delay: std::time::Duration::ZERO, diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index 8e9d42770b4..baa005bbb97 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -59,20 +59,23 @@ pub fn update_supported(version: Version) -> bool { /// repeating the path. Older versions send a single `ANNOUNCE_BROADCAST` /// message that retracts by path (`Ended`). /// -/// The path and hop chain are as they appear on the wire: on lite-07 they may name a -/// base announcement, resolved against the stream's history by -/// [`AnnounceDecoder`](super::AnnounceDecoder). +/// The path is as it appears on the wire: on lite-07 it may name a base announcement, +/// resolved against the stream's history by [`AnnounceDecoder`](super::AnnounceDecoder). +/// +/// Lite-07 names the route each claim travels under in place of a hop chain, so `hops` +/// must be empty there and `route` must be 0 on every earlier version. #[derive(Clone, Debug, PartialEq, Eq)] pub enum AnnounceBroadcast { /// ANNOUNCE_START (lite-06) / active (older): a broadcast is now available. - /// Carries the path suffix, (lite-07+) the instance and epoch it serves, the hop - /// chain, and (lite-06+) the static route cost, and assigns the next announce id. - /// The instance and epoch change only with a `Restart`. + /// Carries the path suffix, (lite-07+) the Route ID and the instance and epoch it + /// serves, (before lite-07) the hop chain, and (lite-06+) the cost, and assigns the + /// next announce id. The instance and epoch change only with a `Restart`. Active { suffix: PathRef, + route: u64, instance: Option, epoch: u64, - hops: HopsRef, + hops: Hops, cost: Cost, }, /// Pre-lite-06: a broadcast is no longer available, retracted by path. @@ -81,9 +84,8 @@ pub enum AnnounceBroadcast { /// announce id. The id is retired; referencing it again is a protocol violation. EndedId { id: u64 }, /// ANNOUNCE_UPDATE (lite-06+): atomically replace the metadata of the announcement - /// with this id (e.g. a new hop chain after a relay failover, or a route whose cost - /// moved). The id stays live. - Update { id: u64, hops: HopsRef, cost: Cost }, + /// with this id: its cost, and before lite-07 its hop chain. The id stays live. + Update { id: u64, hops: Hops, cost: Cost }, /// ANNOUNCE_RESTART (lite-07+): the announcement with this id now serves the /// instance and epoch given. Another instance is another stream, so whatever the /// receiver resolved under its prefix is stale; a higher epoch of the same one @@ -92,7 +94,6 @@ pub enum AnnounceBroadcast { id: u64, instance: Option, epoch: u64, - hops: HopsRef, cost: Cost, }, /// ANNOUNCE_TAKEOVER (lite-07+): an `Active` that continues what the live @@ -101,9 +102,9 @@ pub enum AnnounceBroadcast { Takeover { predecessor: u64, suffix: PathRef, + route: u64, instance: Option, epoch: u64, - hops: HopsRef, cost: Cost, }, /// An unknown lite-06+ announce type. The length-prefixed body was skipped so @@ -216,55 +217,37 @@ impl<'a> Decode<'a, Version> for PathRef { } } -/// A hop chain as it travels: `literal` leading hops, followed by the last `keep` -/// entries of a base announcement's chain. -/// -/// `base` counts back like [`PathRef::base`], independently of it. Only lite-07 -/// carries a base; every other version is `literal` alone. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct HopsRef { - pub base: u64, - pub literal: Hops, - pub keep: u64, +/// Encode the Route ID a claim travels under, which only lite-07 carries. +fn encode_route_id(w: &mut Encoder<'_>, version: Version, route: u64) -> Result<(), EncodeError> { + match version.has_routes() { + true => w.varint(route), + false if route != 0 => Err(EncodeError::InvalidState), + false => Ok(()), + } } -impl HopsRef { - /// The whole chain, with no base. - pub fn literal(literal: Hops) -> Self { - Self { - base: 0, - literal, - keep: 0, - } +/// Decode the Route ID a claim travels under, 0 on a version without one. +fn decode_route_id(r: &mut Decoder<'_>, version: Version) -> Result { + match version.has_routes() { + true => r.varint(), + false => Ok(0), } } -impl Encode for HopsRef { - fn encode(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { - if !version.has_announce_compression() { - if self.base != 0 || self.keep != 0 { - return Err(EncodeError::Version); - } - return self.literal.encode(w, version); - } - w.varint(self.base)?; - self.literal.encode(w, version)?; - w.varint(self.keep) +/// Encode a lite-06+ announcement's hop chain, which lite-07 replaces with its Route ID. +fn encode_chain(w: &mut Encoder<'_>, version: Version, hops: &Hops) -> Result<(), EncodeError> { + match version.has_routes() { + true if !hops.is_empty() => Err(EncodeError::InvalidState), + true => Ok(()), + false => hops.encode(w, version), } } -impl<'a> Decode<'a, Version> for HopsRef { - fn decode(buf: &mut Decoder<'a>, version: Version) -> Result { - if !version.has_announce_compression() { - return Ok(Self::literal(Hops::decode(buf, version)?)); - } - let base = buf.varint()?; - let literal = Hops::decode(buf, version)?; - let keep = buf.varint()?; - if base == 0 && keep != 0 { - return Err(DecodeError::InvalidValue); - } - Ok(Self { base, literal, keep }) +/// Decode a lite-06+ announcement's hop chain, empty on lite-07. +fn decode_chain(r: &mut Decoder<'_>, version: Version) -> Result { + match version.has_routes() { + true => Ok(Hops::default()), + false => Hops::decode(r, version), } } @@ -337,46 +320,46 @@ impl Encode for AnnounceBroadcast { match self { Self::Active { suffix, + route, instance, epoch, hops, cost, } => { suffix.encode(w, version)?; + encode_route_id(w, version, *route)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; + encode_chain(w, version, hops)?; cost.encode(w, version)?; } Self::EndedId { id } => w.varint(*id)?, Self::Update { id, hops, cost } => { w.varint(*id)?; - hops.encode(w, version)?; + encode_chain(w, version, hops)?; cost.encode(w, version)?; } Self::Restart { id, instance, epoch, - hops, cost, } => { w.varint(*id)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; cost.encode(w, version)?; } Self::Takeover { predecessor, suffix, + route, instance, epoch, - hops, cost, } => { w.varint(*predecessor)?; suffix.encode(w, version)?; + encode_route_id(w, version, *route)?; super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - hops.encode(w, version)?; cost.encode(w, version)?; } Self::Synced => {} @@ -407,14 +390,19 @@ impl Encode for AnnounceBroadcast { let prefix = w.prefix_varint(); match self { // The cost is a lite-06 addition, so it is simply not on the wire here. - Self::Active { suffix, hops, .. } => { - // Bases are a lite-07 addition; PathRef and HopsRef refuse one here. - if suffix.base != 0 || hops.base != 0 { + Self::Active { + suffix, route, hops, .. + } => { + // Bases are a lite-07 addition; PathRef refuses one here. + if suffix.base != 0 { return Err(EncodeError::Version); } + if *route != 0 { + return Err(EncodeError::InvalidState); + } AnnounceStatus::Active.encode(w, version)?; suffix.rest.encode(w, version)?; - encode_hops(w, version, &hops.literal)?; + encode_hops(w, version, hops)?; } Self::Ended { suffix, hops } => { AnnounceStatus::Ended.encode(w, version)?; @@ -456,12 +444,14 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { Ok(match typ { ANNOUNCE_START => { let suffix = PathRef::decode(&mut body, version)?; + let route = decode_route_id(&mut body, version)?; let (instance, epoch) = super::instance::decode_route(&mut body, version)?; Self::Active { suffix, + route, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, + hops: decode_chain(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } @@ -469,7 +459,7 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { ANNOUNCE_END => Self::EndedId { id: body.varint()? }, ANNOUNCE_UPDATE => Self::Update { id: body.varint()?, - hops: HopsRef::decode(&mut body, version)?, + hops: decode_chain(&mut body, version)?, cost: Cost::decode(&mut body, version)?, }, ANNOUNCE_RESTART if version.has_announce_restart() => { @@ -479,20 +469,20 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { id, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } ANNOUNCE_TAKEOVER if version.has_announce_takeover() => { let predecessor = body.varint()?; let suffix = PathRef::decode(&mut body, version)?; + let route = decode_route_id(&mut body, version)?; let (instance, epoch) = super::instance::decode_route(&mut body, version)?; Self::Takeover { predecessor, suffix, + route, instance, epoch, - hops: HopsRef::decode(&mut body, version)?, cost: Cost::decode(&mut body, version)?, } } @@ -561,9 +551,10 @@ impl AnnounceBroadcast { Ok(match status { AnnounceStatus::Active => Self::Active { suffix: PathRef::literal(suffix), + route: 0, instance: None, epoch: 0, - hops: HopsRef::literal(hops), + hops, cost: Cost::UNKNOWN, }, AnnounceStatus::Ended => Self::Ended { suffix, hops }, @@ -574,9 +565,10 @@ impl AnnounceBroadcast { // invalid value there. AnnounceStatus::Update if update_supported(version) => Self::Active { suffix: PathRef::literal(suffix), + route: 0, instance: None, epoch: 0, - hops: HopsRef::literal(hops), + hops, cost: Cost::UNKNOWN, }, AnnounceStatus::Update => return Err(DecodeError::InvalidValue), @@ -817,8 +809,10 @@ impl<'a> Message<'a> for AnnounceInit { /// /// `origin` is the responder's session origin id. In Lite05 the publisher no /// longer stamps it onto each Announce's hop chain; the subscriber appends it on -/// receipt instead. Lite05/06 carry `active`, the number of starts immediately -/// following; lite07 marks the initial set with SYNCED instead. +/// receipt instead. Lite05/06 carry it and `active`, the number of starts immediately +/// following. Lite07 has an empty body: its claims name their route instead of a hop +/// chain, and SYNCED marks the initial set, so neither field is sent and both decode +/// as unknown. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct AnnounceOk { pub origin: Hop, @@ -831,9 +825,15 @@ impl<'a> Message<'a> for AnnounceOk { if !version.has_announce_ok() { return Err(DecodeError::Version); } + if version.has_routes() { + return Ok(Self { + origin: Hop::UNKNOWN, + active: 0, + }); + } let origin = Hop::decode(r, version)?; - let active = if version.has_synced() { 0 } else { r.varint()? }; + let active = r.varint()?; Ok(Self { origin, active }) } @@ -841,12 +841,12 @@ impl<'a> Message<'a> for AnnounceOk { if !version.has_announce_ok() { return Err(EncodeError::Version); } + if version.has_routes() { + return Ok(()); + } self.origin.encode(w, version)?; - if !version.has_synced() { - w.varint(self.active)?; - } - Ok(()) + w.varint(self.active) } } @@ -880,8 +880,10 @@ mod tests { assert!(matches!(decode(&wire, usize::MAX), Err(DecodeError::InvalidValue))); } + /// Lite-07's ANNOUNCE_OK is empty: SYNCED replaces the count, and routes replace the + /// Hop ID. Lite-06 keeps both. #[test] - fn synced_replaces_only_lite07s_count() { + fn announce_ok_is_empty_on_lite07() { for version in [Version::Lite06, Version::Lite07] { let mut bytes = Vec::new(); AnnounceOk { @@ -895,10 +897,25 @@ mod tests { if version == Version::Lite06 { vec![2, 1, 7] } else { - vec![1, 1] + vec![0] } ); } + assert_eq!( + crate::coding::decode_buf(&mut &[0u8][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).unwrap(), + AnnounceOk { + origin: Hop::UNKNOWN, + active: 0, + } + ); + // A body where the draft has none is malformed. + assert!( + crate::coding::decode_buf(&mut &[1u8, 1][..], Version::Lite07, |r, v| AnnounceOk::decode(r, v)).is_err() + ); + } + + #[test] + fn synced_marks_lite07s_initial_set() { let mut bytes = Vec::new(); AnnounceBroadcast::Synced .encode(&mut Encoder::new(&mut bytes, Version::Lite07.into()), Version::Lite07) @@ -918,10 +935,11 @@ mod tests { // Encode a normal Active, then flip its status byte (1 -> 2). let mut buf = Vec::new(); AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("foo/bar")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default(), } .encode(&mut Encoder::new(&mut buf, version.into()), version) @@ -1010,10 +1028,11 @@ mod tests { let mut hops = Hops::new(); hops.push(Hop::new(7).unwrap()).unwrap(); let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::literal(hops.clone()), + hops: hops.clone(), cost: Cost::UNKNOWN, }; assert_eq!(broadcast_round_trip(&msg, Version::Lite05), msg); @@ -1034,10 +1053,11 @@ mod tests { let cost = Cost::new(12); let active = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::literal(hops.clone()), + hops: hops.clone(), cost, }; assert_eq!(broadcast_round_trip(&active, Version::Lite06), active); @@ -1045,22 +1065,18 @@ mod tests { let ended = AnnounceBroadcast::EndedId { id: 3 }; assert_eq!(broadcast_round_trip(&ended, Version::Lite06), ended); - let update = AnnounceBroadcast::Update { - id: 3, - hops: HopsRef::literal(hops), - cost, - }; + let update = AnnounceBroadcast::Update { id: 3, hops, cost }; assert_eq!(broadcast_round_trip(&update, Version::Lite06), update); } - // Lite07 carries both bases as they travel; the codec resolves nothing. + // Lite07 carries the path base as it travels, and the route in place of a hop chain; + // the codec resolves nothing. #[test] fn announce_broadcast_round_trip_on_lite07() { - let mut hops = Hops::new(); - hops.push(Hop::new(7).unwrap()).unwrap(); let cost = Cost::new(12); let active = AnnounceBroadcast::Active { + route: 4, instance: Some("a".parse().unwrap()), epoch: 5, suffix: PathRef { @@ -1068,22 +1084,14 @@ mod tests { keep: 3, rest: Path::from_static("cam"), }, - hops: HopsRef { - base: 1, - literal: hops.clone(), - keep: 2, - }, + hops: Hops::default(), cost, }; assert_eq!(broadcast_round_trip(&active, Version::Lite07), active); let update = AnnounceBroadcast::Update { id: 3, - hops: HopsRef { - base: 4, - literal: hops.clone(), - keep: 1, - }, + hops: Hops::default(), cost, }; assert_eq!(broadcast_round_trip(&update, Version::Lite07), update); @@ -1093,11 +1101,6 @@ mod tests { id: 3, instance: Some(Instance::random()), epoch, - hops: HopsRef { - base: 4, - literal: hops.clone(), - keep: 1, - }, cost, }; assert_eq!(broadcast_round_trip(&restart, Version::Lite07), restart); @@ -1109,28 +1112,127 @@ mod tests { keep: 1, rest: Path::from_static("p"), }, + route: 0, instance: Some(Instance::random()), epoch, - hops: HopsRef { - base: 1, - literal: hops.clone(), - keep: 1, - }, cost, }; assert_eq!(broadcast_round_trip(&takeover, Version::Lite07), takeover); } } + /// The draft's field order, byte for byte: ANNOUNCE_START is the path base, keep, and + /// rest, the Route ID, the instance and epoch, and the cost; ANNOUNCE_UPDATE is the + /// Announce ID and the cost; ANNOUNCE_TAKEOVER leads with its predecessor. + #[test] + fn lite07_announce_layout() { + let encode = |msg: AnnounceBroadcast| { + let mut buf = Vec::new(); + msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07) + .unwrap(); + buf + }; + let suffix = PathRef { + base: 1, + keep: 2, + rest: Path::from_static("c"), + }; + assert_eq!( + encode(AnnounceBroadcast::Active { + suffix: suffix.clone(), + route: 3, + instance: Some("a".parse().unwrap()), + epoch: 5, + hops: Hops::default(), + cost: Cost::new(6), + }), + [0, 9, 1, 2, 1, b'c', 3, 1, b'a', 5, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Update { + id: 7, + hops: Hops::default(), + cost: Cost::new(6), + }), + [2, 2, 7, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Restart { + id: 7, + instance: Some("a".parse().unwrap()), + epoch: 5, + cost: Cost::new(6), + }), + [3, 5, 7, 1, b'a', 5, 6] + ); + assert_eq!( + encode(AnnounceBroadcast::Takeover { + predecessor: 9, + suffix, + route: 3, + instance: Some("a".parse().unwrap()), + epoch: 5, + cost: Cost::new(6), + }), + [5, 10, 9, 1, 2, 1, b'c', 3, 1, b'a', 5, 6] + ); + } + + /// Lite-07 names a claim's route instead of its hop chain, so a chain is refused there. + #[test] + fn lite07_refuses_a_hop_chain() { + let mut hops = Hops::new(); + hops.push(Hop::new(7).unwrap()).unwrap(); + let refused = [ + AnnounceBroadcast::Active { + route: 0, + instance: Some("a".parse().unwrap()), + epoch: 0, + suffix: PathRef::literal(Path::from_static("cam")), + hops: hops.clone(), + cost: Cost::default(), + }, + AnnounceBroadcast::Update { + id: 0, + hops, + cost: Cost::default(), + }, + ]; + for msg in refused { + let mut buf = Vec::new(); + let res = msg.encode(&mut Encoder::new(&mut buf, Version::Lite07.into()), Version::Lite07); + assert!(matches!(res, Err(EncodeError::InvalidState)), "{msg:?}"); + } + } + + /// Only lite-07 carries a Route ID: a non-zero one has nowhere to go earlier, and an + /// earlier start decodes as route 0. + #[test] + fn a_route_id_needs_lite07() { + let msg = AnnounceBroadcast::Active { + route: 1, + instance: None, + epoch: 0, + suffix: PathRef::literal(Path::from_static("cam")), + hops: Hops::default(), + cost: Cost::default(), + }; + for version in [Version::Lite05, Version::Lite06] { + let mut buf = Vec::new(); + let res = msg.encode(&mut Encoder::new(&mut buf, version.into()), version); + assert!(matches!(res, Err(EncodeError::InvalidState)), "{version:?}"); + } + } + // Only lite-07 carries ANNOUNCE_TAKEOVER: older versions send a start instead. #[test] fn announce_takeover_needs_lite07() { let takeover = AnnounceBroadcast::Takeover { predecessor: 0, suffix: PathRef::literal(Path::from_static("p")), + route: 0, instance: Some(Instance::random()), epoch: 0, - hops: HopsRef::default(), cost: Cost::default(), }; assert!(matches!( @@ -1150,7 +1252,6 @@ mod tests { id: 1, instance: Some("a".parse().unwrap()), epoch: 0, - hops: HopsRef::default(), cost: Cost::default(), }; let mut buf = Vec::new(); @@ -1181,22 +1282,21 @@ mod tests { // A keep copies from a base, so one without a base is malformed. #[test] fn a_keep_without_a_base_is_rejected() { - for (path_keep, hop_keep) in [(1u8, 0u8), (0, 1)] { - // Path base, path keep, empty rest, instance "a", epoch, hop base, no hops, hop keep, cost. - let body = [0, path_keep, 0, 1, b'a', 0, 0, 0, hop_keep, 0]; - let mut buf = vec![ANNOUNCE_START as u8, body.len() as u8]; - buf.extend_from_slice(&body); - assert!(matches!( - crate::coding::decode_buf(&mut &buf[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)), - Err(DecodeError::InvalidValue) - )); - } + // Path base, path keep, empty rest, route, instance "a", epoch, cost. + let body = [0, 1, 0, 0, 1, b'a', 0, 0]; + let mut buf = vec![ANNOUNCE_START as u8, body.len() as u8]; + buf.extend_from_slice(&body); + assert!(matches!( + crate::coding::decode_buf(&mut &buf[..], Version::Lite07, |r, v| AnnounceBroadcast::decode(r, v)), + Err(DecodeError::InvalidValue) + )); } // Only lite-07 has room for a base. #[test] fn a_base_needs_lite07() { let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef { @@ -1204,7 +1304,7 @@ mod tests { keep: 1, rest: Path::from_static("cam"), }, - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default(), }; for version in [Version::Lite05, Version::Lite06] { @@ -1228,7 +1328,7 @@ mod tests { assert!(matches!( AnnounceBroadcast::Update { id: 1, - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::default() } .encode(&mut Encoder::new(&mut buf, Version::Lite05.into()), Version::Lite05), @@ -1251,20 +1351,22 @@ mod tests { #[test] fn route_cost_is_dropped_before_lite06() { let msg = AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::new(9), }; let got = broadcast_round_trip(&msg, Version::Lite05); assert_eq!( got, AnnounceBroadcast::Active { + route: 0, instance: None, epoch: 0, suffix: PathRef::literal(Path::from_static("room/cam")), - hops: HopsRef::default(), + hops: Hops::default(), cost: Cost::UNKNOWN, } ); diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index 1ab2bfeb1ea..3c015d9c2d3 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -1,57 +1,49 @@ -//! Lite-07 announce compression: an ANNOUNCE_START may copy the head of a live -//! announcement's path, and an ANNOUNCE_START or ANNOUNCE_UPDATE the tail of a live -//! announcement's hop chain, naming that base by its distance back from the stream's -//! next unassigned Announce ID. +//! Lite-07 announce compression: an ANNOUNCE_START or ANNOUNCE_TAKEOVER may copy the +//! head of a live announcement's path, naming that base by its distance back from the +//! stream's next unassigned Announce ID. //! //! The codec in `announce.rs` stays stateless and carries bases as they travel. The //! decoder here resolves them against the stream's live announcements, which the //! subscriber has to track anyway, and the encoder picks them. -use std::{ - borrow::Borrow, - collections::{BTreeSet, HashMap}, - hash::Hash, -}; +use std::collections::{BTreeSet, HashMap}; use crate::{ - Error, Hop, Hops, Path, + Error, Path, coding::{Form, varint}, }; -use super::{HopsRef, PathRef, Version}; - -/// A live announcement, as the peer holds it: its wire suffix and wire hop chain -/// (the one excluding ANNOUNCE_OK's Hop ID). -#[derive(Clone, Debug)] -struct Entry { - suffix: Path, - hops: Hops, -} +use super::{PathRef, Version}; -/// The live announcements on one stream by Announce ID, which is all a base can name. +/// The live announcements on one stream by Announce ID, each with its wire suffix, +/// which is all a base can name. #[derive(Debug, Default)] struct Live { next: u64, - entries: HashMap, + suffixes: HashMap, } impl Live { + /// The suffix of the live announcement with this id. + fn suffix(&self, id: u64) -> Result<&Path, Error> { + self.suffixes.get(&id).ok_or(Error::ProtocolViolation) + } + /// The live announcement `distance` back from the next id, or `None` for 0. - fn base(&self, distance: u64) -> Result, Error> { + fn base(&self, distance: u64) -> Result, Error> { if distance == 0 { return Ok(None); } let id = self.next.checked_sub(distance).ok_or(Error::ProtocolViolation)?; - let entry = self.entries.get(&id).ok_or(Error::ProtocolViolation)?; - Ok(Some(entry)) + self.suffix(id).map(Some) } - fn resolve_path(&self, path: PathRef) -> Result { + fn resolve(&self, path: PathRef) -> Result { let Some(base) = self.base(path.base)? else { return Ok(path.rest); }; let keep = usize::try_from(path.keep).map_err(|_| Error::ProtocolViolation)?; - let head = head(&base.suffix, keep).ok_or(Error::ProtocolViolation)?; + let head = head(base, keep).ok_or(Error::ProtocolViolation)?; let suffix = match (head.is_empty(), path.rest.is_empty()) { (true, _) => Ok(path.rest), (false, true) => Path::new(head), @@ -63,20 +55,6 @@ impl Live { } Ok(suffix) } - - fn resolve_hops(&self, hops: HopsRef) -> Result { - let Some(base) = self.base(hops.base)? else { - return Ok(hops.literal); - }; - let keep = usize::try_from(hops.keep).map_err(|_| Error::ProtocolViolation)?; - let tail = base.hops.as_slice(); - let start = tail.len().checked_sub(keep).ok_or(Error::ProtocolViolation)?; - let mut out = hops.literal; - for hop in &tail[start..] { - out.push(*hop).map_err(|_| Error::ProtocolViolation)?; - } - Ok(out) - } } /// The first `keep` segments of `path`, or `None` if it has fewer. @@ -100,84 +78,56 @@ pub(crate) struct AnnounceDecoder { } impl AnnounceDecoder { - /// An ANNOUNCE_START: resolve it and assign it the next id. - pub fn start(&mut self, suffix: PathRef, hops: HopsRef) -> Result<(Path, Hops), Error> { - let suffix = self.live.resolve_path(suffix)?; - let hops = self.live.resolve_hops(hops)?; + /// An ANNOUNCE_START: resolve its suffix and assign it the next id. + pub fn start(&mut self, suffix: PathRef) -> Result { + let suffix = self.live.resolve(suffix)?; let id = self.live.next; self.live.next += 1; - self.live.entries.insert( - id, - Entry { - suffix: suffix.clone(), - hops: hops.clone(), - }, - ); - Ok((suffix, hops)) + self.live.suffixes.insert(id, suffix.clone()); + Ok(suffix) } /// An ANNOUNCE_TAKEOVER: resolve it like a start against the live `predecessor`, - /// returning the predecessor's suffix too. One at the predecessor's own suffix - /// retires the predecessor, since a suffix holds one advertisement. - pub fn takeover(&mut self, predecessor: u64, suffix: PathRef, hops: HopsRef) -> Result<(Path, Path, Hops), Error> { - let previous = self - .live - .entries - .get(&predecessor) - .ok_or(Error::ProtocolViolation)? - .suffix - .clone(); - let (suffix, hops) = self.start(suffix, hops)?; + /// returning the predecessor's suffix, then its own. One at the predecessor's own + /// suffix retires the predecessor, since a suffix holds one advertisement. + pub fn takeover(&mut self, predecessor: u64, suffix: PathRef) -> Result<(Path, Path), Error> { + let previous = self.live.suffix(predecessor)?.clone(); + let suffix = self.start(suffix)?; if suffix == previous { - self.live.entries.remove(&predecessor); + self.live.suffixes.remove(&predecessor); } - Ok((previous, suffix, hops)) + Ok((previous, suffix)) } - /// An ANNOUNCE_UPDATE: resolve its chain before replacing the old one, which it - /// may itself be based on. - pub fn update(&mut self, id: u64, hops: HopsRef) -> Result<(Path, Hops), Error> { - let hops = self.live.resolve_hops(hops)?; - let entry = self.live.entries.get_mut(&id).ok_or(Error::ProtocolViolation)?; - entry.hops = hops.clone(); - Ok((entry.suffix.clone(), hops)) + /// The suffix of the live id an ANNOUNCE_UPDATE or ANNOUNCE_RESTART names, which + /// stays live. + pub fn get(&self, id: u64) -> Result { + self.live.suffix(id).cloned() } /// An ANNOUNCE_END: retire the id, returning its suffix. pub fn end(&mut self, id: u64) -> Result { - let entry = self.live.entries.remove(&id).ok_or(Error::ProtocolViolation)?; - Ok(entry.suffix) + self.live.suffixes.remove(&id).ok_or(Error::ProtocolViolation) } } -/// The live ids under a run of keys: path segments from the head, or hop ids from -/// the tail. Every node holds the ids of the entries beneath it, so the entry sharing -/// the longest run with a query is found by walking the query down once, at a cost -/// that grows with its length rather than with the number of entries. -#[derive(Debug)] -struct Trie { - children: HashMap>, +/// The live ids under a run of leading path segments. Every node holds the ids of the +/// entries beneath it, so the entry sharing the longest head with a query is found by +/// walking the query down once, at a cost that grows with its length rather than with +/// the number of entries. +#[derive(Debug, Default)] +struct Trie { + children: HashMap, } #[derive(Debug)] -struct Node { +struct Node { ids: BTreeSet, - children: HashMap>, + children: HashMap, } -impl Default for Trie { - fn default() -> Self { - Self { - children: HashMap::new(), - } - } -} - -impl Trie { - fn insert<'a, Q>(&mut self, keys: impl Iterator, id: u64) - where - Q: ?Sized + ToOwned + 'a, - { +impl Trie { + fn insert<'a>(&mut self, keys: impl Iterator, id: u64) { let mut children = &mut self.children; for key in keys { let node = children.entry(key.to_owned()).or_insert_with(|| Node { @@ -189,19 +139,11 @@ impl Trie { } } - fn remove<'a, Q>(&mut self, keys: impl Iterator, id: u64) - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn remove<'a>(&mut self, keys: impl Iterator, id: u64) { Self::remove_from(&mut self.children, keys, id); } - fn remove_from<'a, Q>(children: &mut HashMap>, mut keys: impl Iterator, id: u64) - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn remove_from<'a>(children: &mut HashMap, mut keys: impl Iterator, id: u64) { let Some(key) = keys.next() else { return; }; @@ -219,11 +161,7 @@ impl Trie { /// How many leading keys the query shares with its closest entry, and that entry's /// id, preferring the newest (the smallest distance to encode). - fn longest<'a, Q>(&self, keys: impl Iterator) -> Option<(usize, u64)> - where - Q: ?Sized + Hash + Eq + 'a, - K: Borrow, - { + fn longest<'a>(&self, keys: impl Iterator) -> Option<(usize, u64)> { let mut children = &self.children; let mut best = None; for (depth, key) in keys.enumerate() { @@ -242,20 +180,13 @@ impl Trie { /// versions that have them. Only lite-07 keeps the live set, since nothing earlier /// can name a base. /// -/// One trie over the live paths by segment, and one over the live hop chains from the -/// last hop back, find the longest shared head and tail. A base is used only when it -/// encodes smaller than the literal. +/// A trie over the live paths by segment finds the longest shared head. A base is +/// used only when it encodes smaller than the literal. #[derive(Debug)] pub(crate) struct AnnounceEncoder { version: Version, live: Live, - heads: Trie, - tails: Trie, -} - -/// A hop chain from the last hop back. -fn reversed(hops: &Hops) -> impl Iterator { - hops.as_slice().iter().rev() + heads: Trie, } impl AnnounceEncoder { @@ -264,60 +195,39 @@ impl AnnounceEncoder { version, live: Live::default(), heads: Trie::default(), - tails: Trie::default(), } } - /// An ANNOUNCE_START: its id (on versions that assign one) and its wire form. - pub fn start(&mut self, suffix: Path, hops: Hops) -> (Option, PathRef, HopsRef) { + /// An ANNOUNCE_START: its id (on versions that assign one) and its wire suffix. + pub fn start(&mut self, suffix: Path) -> (Option, PathRef) { if !self.version.has_announce_id() { - return (None, PathRef::literal(suffix), HopsRef::literal(hops)); + return (None, PathRef::literal(suffix)); } let id = self.live.next; self.live.next += 1; if !self.version.has_announce_compression() { - return (Some(id), PathRef::literal(suffix), HopsRef::literal(hops)); + return (Some(id), PathRef::literal(suffix)); } // Distances count back from the id this START takes. let path = self.path_ref(&suffix, id); - let chain = self.hops_ref(&hops, id); self.heads.insert(suffix.parts(), id); - self.tails.insert(reversed(&hops), id); - self.live.entries.insert(id, Entry { suffix, hops }); - (Some(id), path, chain) + self.live.suffixes.insert(id, suffix); + (Some(id), path) } /// An ANNOUNCE_TAKEOVER of the live `predecessor`: like [`Self::start`], retiring /// the predecessor when it holds the same suffix. - pub fn takeover(&mut self, predecessor: u64, suffix: Path, hops: Hops) -> (u64, PathRef, HopsRef) { - let replaces = self - .live - .entries - .get(&predecessor) - .is_some_and(|entry| entry.suffix == suffix); - let (id, path, chain) = self.start(suffix, hops); + pub fn takeover(&mut self, predecessor: u64, suffix: Path) -> (u64, PathRef) { + let replaces = self.live.suffixes.get(&predecessor) == Some(&suffix); + let (id, path) = self.start(suffix); let id = id.expect("a takeover is only sent on a version with announce ids"); if replaces { self.end(predecessor); } - (id, path, chain) - } - - /// An ANNOUNCE_UPDATE for a live `id`: its wire hop chain. - pub fn update(&mut self, id: u64, hops: Hops) -> HopsRef { - if !self.version.has_announce_compression() { - return HopsRef::literal(hops); - } - // The decoder resolves against the chain this replaces, so it is a valid base. - let chain = self.hops_ref(&hops, self.live.next); - let entry = self.live.entries.get_mut(&id).expect("update for a live id"); - self.tails.remove(reversed(&entry.hops), id); - self.tails.insert(reversed(&hops), id); - entry.hops = hops; - chain + (id, path) } /// An ANNOUNCE_END for a live `id`. @@ -325,9 +235,8 @@ impl AnnounceEncoder { if !self.version.has_announce_compression() { return; } - let entry = self.live.entries.remove(&id).expect("end for a live id"); - self.heads.remove(entry.suffix.parts(), id); - self.tails.remove(reversed(&entry.hops), id); + let suffix = self.live.suffixes.remove(&id).expect("end for a live id"); + self.heads.remove(suffix.parts(), id); } /// The wire form of `suffix`, sized from its parts so only the winner is built. @@ -350,35 +259,6 @@ impl AnnounceEncoder { false => literal(), } } - - /// The wire form of `hops`, sized from its parts so only the winner is built. - fn hops_ref(&self, hops: &Hops, next: u64) -> HopsRef { - let form: Form = self.version.into(); - let literal = || HopsRef::literal(hops.clone()); - let Some((keep, id)) = self.tails.longest(reversed(hops)) else { - return literal(); - }; - let head = &hops.as_slice()[..hops.len() - keep]; - - let base = next - id; - let keep = keep as u64; - let chain = |hops: &[Hop]| { - varint_size(hops.len() as u64, form) - + hops - .iter() - .map(|hop| varint_size(hop.wire_id(form), form)) - .sum::() - }; - let size = varint_size(base, form) + chain(head) + varint_size(keep, form); - match size < varint_size(0, form) * 2 + chain(hops.as_slice()) { - true => HopsRef { - base, - literal: Hops::try_from(head.to_vec()).expect("a prefix of a valid chain is valid"), - keep, - }, - false => literal(), - } - } } /// The bytes `value` takes as a varint in `form`. @@ -399,10 +279,6 @@ mod tests { const VERSION: Version = Version::Lite07; - fn hops(ids: &[u64]) -> Hops { - Hops::try_from(ids.iter().map(|id| Hop::from_wire(*id)).collect::>()).unwrap() - } - fn path(base: u64, keep: u64, rest: &str) -> PathRef { PathRef { base, @@ -411,85 +287,52 @@ mod tests { } } - fn chain(base: u64, literal: &[u64], keep: u64) -> HopsRef { - HopsRef { - base, - literal: hops(literal), - keep, - } - } - - fn literal(suffix: &str, ids: &[u64]) -> (PathRef, HopsRef) { - (path(0, 0, suffix), chain(0, ids, 0)) + fn literal(suffix: &str) -> PathRef { + path(0, 0, suffix) } #[test] - fn resolves_a_path_head_and_hop_tail() { + fn resolves_a_path_head() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("pid/private/channel_1/health-1", &[1, 2, 3]); - decoder.start(p, h).unwrap(); + decoder.start(literal("pid/private/channel_1/health-1")).unwrap(); - // Three leading segments of the latest START, and its last two hops. - let (suffix, chain) = decoder.start(path(1, 3, "health-2"), chain(1, &[7], 2)).unwrap(); + // Three leading segments of the latest START. + let suffix = decoder.start(path(1, 3, "health-2")).unwrap(); assert_eq!(suffix.as_str(), "pid/private/channel_1/health-2"); - assert_eq!(chain, hops(&[7, 2, 3])); } #[test] - fn keeps_the_whole_path_and_chain() { + fn keeps_the_whole_path() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a/b", &[1, 2]); - decoder.start(p, h).unwrap(); - // A full keep with an empty rest names a route above the base's: not a - // duplicate, since the rest extends it. - let (suffix, chain) = decoder.start(path(1, 2, "c"), chain(1, &[], 2)).unwrap(); + decoder.start(literal("a/b")).unwrap(); + // A full keep with a rest names a route beneath the base's: not a duplicate, + // since the rest extends it. + let suffix = decoder.start(path(1, 2, "c")).unwrap(); + assert_eq!(suffix.as_str(), "a/b/c"); + // And a full keep with an empty rest copies it whole. + let suffix = decoder.start(path(1, 3, "")).unwrap(); assert_eq!(suffix.as_str(), "a/b/c"); - assert_eq!(chain, hops(&[1, 2])); - // An update copying its own full chain. - let (suffix, chain) = decoder.update(0, chain_ref(2, 2)).unwrap(); - assert_eq!(suffix.as_str(), "a/b"); - assert_eq!(chain, hops(&[1, 2])); - } - - fn chain_ref(base: u64, keep: u64) -> HopsRef { - chain(base, &[], keep) } #[test] fn a_new_stream_has_no_base() { let mut decoder = AnnounceDecoder::default(); - assert!(matches!( - decoder.start(path(1, 0, "a"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); - let mut decoder = AnnounceDecoder::default(); - assert!(matches!( - decoder.start(path(0, 0, "a"), chain(1, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(1, 0, "a")), Err(Error::ProtocolViolation))); } #[test] fn rejects_a_retired_or_unassigned_base() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a", &[1]); - decoder.start(p, h).unwrap(); - let (p, h) = literal("b", &[2]); - decoder.start(p, h).unwrap(); + decoder.start(literal("a")).unwrap(); + decoder.start(literal("b")).unwrap(); decoder.end(0).unwrap(); // Id 0 is two back, and retired. - assert!(matches!( - decoder.start(path(2, 1, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(2, 1, "x")), Err(Error::ProtocolViolation))); // Three back was never assigned. - assert!(matches!( - decoder.start(path(3, 0, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(3, 0, "x")), Err(Error::ProtocolViolation))); // Id 1 is still live, so reusing it works after the END. - let (suffix, _) = decoder.start(path(1, 1, "x"), chain(1, &[], 1)).unwrap(); + let suffix = decoder.start(path(1, 1, "x")).unwrap(); assert_eq!(suffix.as_str(), "b/x"); } @@ -498,27 +341,22 @@ mod tests { #[test] fn takeover_assigns_an_id_and_retires_one_in_place() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("pool", &[1]); - decoder.start(p, h).unwrap(); + decoder.start(literal("pool")).unwrap(); - let (previous, suffix, chain) = decoder.takeover(0, path(1, 1, "p"), chain_ref(1, 1)).unwrap(); + let (previous, suffix) = decoder.takeover(0, path(1, 1, "p")).unwrap(); assert_eq!((previous.as_str(), suffix.as_str()), ("pool", "pool/p")); - assert_eq!(chain, hops(&[1])); - decoder.update(0, chain_ref(0, 0)).unwrap(); + assert_eq!(decoder.get(0).unwrap().as_str(), "pool"); // Id 2 at id 1's own suffix, copied from id 1 itself, which then retires. - let (previous, suffix, _) = decoder.takeover(1, path(1, 2, ""), chain_ref(1, 1)).unwrap(); + let (previous, suffix) = decoder.takeover(1, path(1, 2, "")).unwrap(); assert_eq!((previous.as_str(), suffix.as_str()), ("pool/p", "pool/p")); + assert!(matches!(decoder.get(1), Err(Error::ProtocolViolation))); assert!(matches!( - decoder.update(1, chain_ref(0, 0)), - Err(Error::ProtocolViolation) - )); - assert!(matches!( - decoder.takeover(1, path(0, 0, "pool/p/q"), chain_ref(0, 0)), + decoder.takeover(1, literal("pool/p/q")), Err(Error::ProtocolViolation) )); assert!(matches!( - decoder.takeover(9, path(0, 0, "pool/p/q"), chain_ref(0, 0)), + decoder.takeover(9, literal("pool/p/q")), Err(Error::ProtocolViolation) )); assert_eq!(decoder.end(2).unwrap().as_str(), "pool/p"); @@ -528,54 +366,17 @@ mod tests { #[test] fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a/b", &[1]); - decoder.start(p, h).unwrap(); - assert!(matches!( - decoder.start(path(1, 3, "x"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); - assert!(matches!( - decoder.update(0, chain_ref(1, 2)), - Err(Error::ProtocolViolation) - )); - } - - #[test] - fn rejects_a_resolved_chain_breaking_the_hop_rules() { - let mut decoder = AnnounceDecoder::default(); - let (p, h) = literal("a", &[1, 2]); - decoder.start(p, h).unwrap(); - // A literal hop the kept tail repeats. - assert!(matches!( - decoder.start(path(0, 0, "b"), chain(1, &[2], 1)), - Err(Error::ProtocolViolation) - )); - - // Anonymous hops may repeat across the two halves. - let (p, h) = literal("c", &[0]); - decoder.start(p, h).unwrap(); - let (_, resolved) = decoder.start(path(0, 0, "d"), chain(1, &[0], 1)).unwrap(); - assert_eq!(resolved, hops(&[0, 0])); - - // And the resolved chain is still capped. - // MAX_HOPS entries. - let full: Vec = (1..=32).collect(); - let (p, h) = literal("e", &full); - decoder.start(p, h).unwrap(); - assert!(matches!( - decoder.start(path(0, 0, "f"), chain(1, &[100], 32)), - Err(Error::ProtocolViolation) - )); + decoder.start(literal("a/b")).unwrap(); + assert!(matches!(decoder.start(path(1, 3, "x")), Err(Error::ProtocolViolation))); } #[test] fn rejects_a_resolved_path_too_deep() { let mut decoder = AnnounceDecoder::default(); let deep = vec!["x"; Path::MAX_PARTS].join("/"); - let (p, h) = literal(&deep, &[]); - decoder.start(p, h).unwrap(); + decoder.start(literal(&deep)).unwrap(); assert!(matches!( - decoder.start(path(1, Path::MAX_PARTS as u64, "y"), chain(0, &[], 0)), + decoder.start(path(1, Path::MAX_PARTS as u64, "y")), Err(Error::ProtocolViolation) )); } @@ -584,26 +385,20 @@ mod tests { fn rejects_a_resolved_path_too_long() { let mut decoder = AnnounceDecoder::default(); let head = "x".repeat(Path::MAX_BYTES - 2); - let (p, h) = literal(&head, &[]); - decoder.start(p, h).unwrap(); + decoder.start(literal(&head)).unwrap(); // `head/y` is exactly the limit, and one byte more is refused. - let (suffix, _) = decoder.start(path(1, 1, "y"), chain(0, &[], 0)).unwrap(); + let suffix = decoder.start(path(1, 1, "y")).unwrap(); assert_eq!(suffix.len(), Path::MAX_BYTES); - assert!(matches!( - decoder.start(path(1, 1, "yz"), chain(0, &[], 0)), - Err(Error::ProtocolViolation) - )); + assert!(matches!(decoder.start(path(1, 1, "yz")), Err(Error::ProtocolViolation))); } - /// Encode a message with `encoder`, decode it with `decoder`, and check that the - /// resolved values match the input. - fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str, ids: &[u64]) -> (u64, usize) { - let (id, wire_path, wire_hops) = encoder.start(Path::new(suffix).unwrap(), hops(ids)); - let size = wire_path.encode_bytes(VERSION).unwrap().len() + wire_hops.encode_bytes(VERSION).unwrap().len(); - let (got_path, got_hops) = decoder.start(wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), suffix); - assert_eq!(got_hops, hops(ids)); + /// Encode a START with `encoder`, decode it with `decoder`, and check that the + /// resolved suffix matches the input. Returns its id and wire size. + fn start(encoder: &mut AnnounceEncoder, decoder: &mut AnnounceDecoder, suffix: &str) -> (u64, usize) { + let (id, wire) = encoder.start(Path::new(suffix).unwrap()); + let size = wire.encode_bytes(VERSION).unwrap().len(); + assert_eq!(decoder.start(wire).unwrap().as_str(), suffix); (id.unwrap(), size) } @@ -612,22 +407,17 @@ mod tests { fn encoder_takeover_decodes_to_its_input() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - let (claim, _) = start(&mut encoder, &mut decoder, "pool", &[1, 2]); + let (claim, _) = start(&mut encoder, &mut decoder, "pool"); let mut predecessor = claim; - for (suffix, ids) in [ - ("pool/p", &[3, 1, 2][..]), - ("pool/p", &[4, 1, 2]), - ("pool/p/q", &[4, 1, 2]), - ] { - let (id, wire_path, wire_hops) = encoder.takeover(predecessor, Path::new(suffix).unwrap(), hops(ids)); - let (_, got_path, got_hops) = decoder.takeover(predecessor, wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), suffix); - assert_eq!(got_hops, hops(ids)); + for suffix in ["pool/p", "pool/p", "pool/p/q"] { + let (id, wire) = encoder.takeover(predecessor, Path::new(suffix).unwrap()); + let (_, got) = decoder.takeover(predecessor, wire).unwrap(); + assert_eq!(got.as_str(), suffix); predecessor = id; } // Both sides retired the id replaced in place and kept the rest. - let (id, wire_path, wire_hops) = encoder.start(Path::new("other").unwrap(), hops(&[1, 2])); - decoder.start(wire_path, wire_hops).unwrap(); + let (id, wire) = encoder.start(Path::new("other").unwrap()); + decoder.start(wire).unwrap(); assert_eq!(id, Some(4)); assert!(matches!(decoder.end(1), Err(Error::ProtocolViolation))); decoder.end(claim).unwrap(); @@ -640,13 +430,10 @@ mod tests { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - // Several origins behind a shared pair of relays. - let relays = [1u64 << 40, 1u64 << 41]; let mut ids = Vec::new(); for n in 0..40u64 { - let origin = 1000 + n % 4; let suffix = format!("pid{}/private/channel_{}/stream-health-{}", n % 4, n % 7, n); - let (id, _) = start(&mut encoder, &mut decoder, &suffix, &[origin, relays[0], relays[1]]); + let (id, _) = start(&mut encoder, &mut decoder, &suffix); ids.push(id); // Retire every third one so bases come and go. if n % 3 == 0 { @@ -656,103 +443,57 @@ mod tests { } } - // Updates resolve against the chain they replace. - for id in ids.iter().copied().take(5) { - let chain = hops(&[2000, relays[1]]); - let wire = encoder.update(id, chain.clone()); - let (_, got) = decoder.update(id, wire).unwrap(); - assert_eq!(got, chain); - } - - // And a later START still matches what the encoder remembers. - start( - &mut encoder, - &mut decoder, - "pid0/private/channel_0/x", - &[1000, relays[0], relays[1]], - ); + // A later START still matches what the encoder remembers. + start(&mut encoder, &mut decoder, "pid0/private/channel_0/x"); } #[test] - fn encoder_picks_the_longest_shared_head_and_tail() { + fn encoder_picks_the_longest_shared_head() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "a/b/c", &[1u64 << 50, 1u64 << 51]); + start(&mut encoder, &mut decoder, "a/b/c"); // Shares a byte prefix with `a/b/d`, but no second segment. - start(&mut encoder, &mut decoder, "a/b-z", &[3]); - - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("a/b/d"), hops(&[5, 1u64 << 51])); - assert_eq!(wire_path, path(2, 2, "d")); - assert_eq!(wire_hops, chain(2, &[5], 1)); - } + start(&mut encoder, &mut decoder, "a/b-z"); - /// Hop 100 is 2 bytes as a QUIC varint and 1 byte as leading-ones. Sharing that - /// single hop from 128 announcements back costs 4 bytes in both forms (the base - /// is 2 bytes either way). The literal chain is 5 bytes in QUIC and 4 in - /// leading-ones, so QUIC would compress and leading-ones ties. A tie stays literal. - #[test] - fn a_tie_stays_literal_where_hop_100_sizes_differ() { - let quic = Form::Quic; - let leading = Form::LeadingOnes; - assert_eq!(varint_size(100, quic), 2); - assert_eq!(varint_size(100, leading), 1); - - let base = 128; - let compressed = |form| varint_size(base, form) + varint_size(0, form) + varint_size(1, form); - let literal = |form| varint_size(0, form) * 2 + varint_size(1, form) + varint_size(100, form); - assert!(compressed(quic) < literal(quic)); - assert_eq!(compressed(leading), literal(leading)); - - let mut encoder = AnnounceEncoder::new(VERSION); - let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "origin", &[100]); - for n in 1..128 { - let suffix = format!("f{n}"); - start(&mut encoder, &mut decoder, &suffix, &[1]); - } - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("again"), hops(&[100])); - assert_eq!(wire_hops, HopsRef::literal(hops(&[100]))); - let (got_path, got_hops) = decoder.start(wire_path, wire_hops).unwrap(); - assert_eq!(got_path.as_str(), "again"); - assert_eq!(got_hops, hops(&[100])); + let (_, wire) = encoder.start(Path::from_static("a/b/d")); + assert_eq!(wire, path(2, 2, "d")); } #[test] fn encoder_sends_literal_when_nothing_is_shared() { let mut encoder = AnnounceEncoder::new(VERSION); let mut decoder = AnnounceDecoder::default(); - start(&mut encoder, &mut decoder, "a", &[1]); - let (_, wire_path, wire_hops) = encoder.start(Path::from_static("b"), hops(&[2])); - assert_eq!(wire_path, PathRef::literal(Path::from_static("b"))); - assert_eq!(wire_hops, HopsRef::literal(hops(&[2]))); + start(&mut encoder, &mut decoder, "a"); + let (_, wire) = encoder.start(Path::from_static("b")); + assert_eq!(wire, PathRef::literal(Path::from_static("b"))); } #[test] fn lite06_assigns_ids_without_bases() { let mut encoder = AnnounceEncoder::new(Version::Lite06); - let (id, _, _) = encoder.start(Path::from_static("a/b"), hops(&[1])); + let (id, _) = encoder.start(Path::from_static("a/b")); assert_eq!(id, Some(0)); - let (id, wire_path, wire_hops) = encoder.start(Path::from_static("a/c"), hops(&[1])); + let (id, wire) = encoder.start(Path::from_static("a/c")); assert_eq!(id, Some(1)); - assert_eq!(wire_path, PathRef::literal(Path::from_static("a/c"))); - assert_eq!(wire_hops, HopsRef::literal(hops(&[1]))); + assert_eq!(wire, PathRef::literal(Path::from_static("a/c"))); encoder.end(0); - assert_eq!(encoder.update(1, hops(&[1])), HopsRef::literal(hops(&[1]))); } /// A compressed stream pinned byte for byte: `js/net/src/lite/announce.test.ts` /// decodes the same hex, so a codec change on either side breaks both. - const GOLDEN: &str = "001800000a726f6f6d2f612f63616d01610000029111a2220000000f0102036d69630161000101b33301000208000101c044440100010101000e020101620161000201c055550100"; + const GOLDEN: &str = + "001200000a726f6f6d2f612f63616d0001610000000b0102036d69630001610000020200000101010009020101620001610000"; #[cfg(feature = "async")] fn golden() -> Vec { use crate::fuzz::Announced; + // Lite-07 names a route instead of a hop chain, so none travels. vec![ - Announced::Start(Path::from_static("room/a/cam"), hops(&[0x1111, 0x2222])), - Announced::Start(Path::from_static("room/a/mic"), hops(&[0x3333, 0x2222])), - Announced::Update(0, hops(&[0x4444, 0x2222])), + Announced::Start(Path::from_static("room/a/cam"), crate::Hops::new()), + Announced::Start(Path::from_static("room/a/mic"), crate::Hops::new()), + Announced::Update(0, crate::Hops::new()), Announced::End(1), - Announced::Start(Path::from_static("room/b"), hops(&[0x5555, 0x2222])), + Announced::Start(Path::from_static("room/b"), crate::Hops::new()), ] } @@ -767,7 +508,7 @@ mod tests { /// The literal lite-07 stream `js/net` writes for the same announcements, which /// never picks a base. - const JS_LITERAL: &str = "001800000a726f6f6d2f612f63616d01610000029111a2220000001800000a726f6f6d2f612f6d69630161000002b333a2220000020a000002c04444a22200000101010015000006726f6f6d2f620161000002c05555a2220000"; + const JS_LITERAL: &str = "001200000a726f6f6d2f612f63616d0001610000001200000a726f6f6d2f612f6d6963000161000002020000010101000e000006726f6f6d2f620001610000"; #[test] #[cfg(feature = "async")] diff --git a/rs/moq-net/src/lite/datagram.rs b/rs/moq-net/src/lite/datagram.rs index d5fd1bb0e71..349f6c3d870 100644 --- a/rs/moq-net/src/lite/datagram.rs +++ b/rs/moq-net/src/lite/datagram.rs @@ -117,6 +117,9 @@ pub struct Datagrams<'a> { pub broadcast: Path, /// The instance the subscriber expects; see [`crate::origin::Route::instance`]. pub instance: Option, + /// The Node ID of the origin that serves the request, or 0 for whichever the + /// publisher selects. + pub node: u64, pub track: Cow<'a, str>, /// The subscriber's priority. pub priority: i8, @@ -129,6 +132,7 @@ impl Datagrams<'_> { id: self.id, broadcast: self.broadcast, instance: self.instance, + node: self.node, track: Cow::Owned(self.track.into_owned()), priority: self.priority, } @@ -145,12 +149,14 @@ impl<'a> Message<'a> for Datagrams<'a> { let id = r.varint()?; let broadcast = Path::decode(r, version)?; let instance = super::instance::decode_pin(r, version)?; + let node = super::instance::decode_node(r, version)?; let track = Cow::Borrowed(r.str()?); let priority = super::priority::from_wire(r.u8()?); Ok(Self { id, broadcast, instance, + node, track, priority, }) @@ -163,6 +169,7 @@ impl<'a> Message<'a> for Datagrams<'a> { w.varint(self.id)?; self.broadcast.encode(w, version)?; super::instance::encode_pin(w, version, self.instance.as_ref())?; + super::instance::encode_node(w, version, self.node)?; w.string(&self.track)?; w.u8(super::priority::to_wire(self.priority)); Ok(()) @@ -362,6 +369,7 @@ mod test { id: 7, broadcast: Path::from_static("room/alice"), instance: None, + node: 0x7a3f, track: "audio".into(), priority: -3, }; @@ -369,6 +377,7 @@ mod test { let (decoded, _) = Datagrams::decode_slice(&buf, Version::Lite07).unwrap(); assert_eq!(decoded.id, 7); assert_eq!(decoded.broadcast, request.broadcast); + assert_eq!(decoded.node, 0x7a3f); assert_eq!(decoded.track, "audio"); assert_eq!(decoded.priority, -3); assert!(matches!( diff --git a/rs/moq-net/src/lite/fetch.rs b/rs/moq-net/src/lite/fetch.rs index 575136726a6..288655be66a 100644 --- a/rs/moq-net/src/lite/fetch.rs +++ b/rs/moq-net/src/lite/fetch.rs @@ -16,6 +16,9 @@ pub struct Fetch<'a> { /// The instance the subscriber expects; see [`crate::origin::RouteInfo::instance`]. /// Lite07+ only. pub instance: Option, + /// The Node ID of the origin that serves the request, or 0 for whichever the + /// publisher selects. Lite07+ only. + pub node: u64, pub track: Cow<'a, str>, pub priority: i8, /// Reader freshness budget. Lite07+ only; omission uses the publisher's max age alone. @@ -34,6 +37,7 @@ impl Fetch<'_> { Fetch { broadcast: self.broadcast, instance: self.instance, + node: self.node, track: Cow::Owned(self.track.into_owned()), priority: self.priority, max_delay: self.max_delay, @@ -56,6 +60,7 @@ impl<'a> Message<'a> for Fetch<'a> { let broadcast = Path::decode(r, version)?; let instance = super::instance::decode_pin(r, version)?; + let node = super::instance::decode_node(r, version)?; let track = Cow::Borrowed(r.str()?); let priority = super::priority::from_wire(r.u8()?); let max_delay = match version.has_fetch_max_delay() { @@ -76,6 +81,7 @@ impl<'a> Message<'a> for Fetch<'a> { Ok(Self { broadcast, instance, + node, track, priority, max_delay, @@ -95,6 +101,7 @@ impl<'a> Message<'a> for Fetch<'a> { self.broadcast.encode(w, version)?; super::instance::encode_pin(w, version, self.instance.as_ref())?; + super::instance::encode_node(w, version, self.node)?; w.string(&self.track)?; w.u8(super::priority::to_wire(self.priority)); match version.has_fetch_max_delay() { @@ -128,6 +135,7 @@ mod test { fn fetch_sample() -> Fetch<'static> { Fetch { instance: None, + node: 0, broadcast: Path::from_static("room"), track: Cow::Borrowed("video"), priority: 3, @@ -171,6 +179,16 @@ mod test { .into_owned() } + /// Only lite-07 names the origin node; older versions decode as letting the + /// publisher select. + #[test] + fn fetch_node_roundtrips_only_on_lite07() { + let mut msg = fetch_sample(); + msg.node = 0x7a3f; + assert_eq!(fetch_roundtrip(Version::Lite07, &msg).node, 0x7a3f); + assert_eq!(fetch_roundtrip(Version::Lite06, &msg).node, 0); + } + #[test] fn fetch_roundtrips() { for version in [Version::Lite03, Version::Lite04, Version::Lite05] { diff --git a/rs/moq-net/src/lite/instance.rs b/rs/moq-net/src/lite/instance.rs index 9d46a3179f0..336e2357c73 100644 --- a/rs/moq-net/src/lite/instance.rs +++ b/rs/moq-net/src/lite/instance.rs @@ -29,6 +29,23 @@ pub(super) fn decode_pin(r: &mut Decoder<'_>, version: Version) -> Result { } /// Retained state, summed over the node, for budgets and leak checks. +#[cfg(test)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct Retained { pub items: usize, @@ -473,6 +479,7 @@ impl Router { self.update(ids); } + #[cfg(test)] /// Ask every neighbor for announcements under `prefix` instead, returning /// the new interest generation. Each neighbor answers on a new stream, and a /// batch from an older one is discarded. @@ -537,11 +544,13 @@ impl Router { .map(|(id, _)| *id) } + #[cfg(test)] /// The declared identity's seqno. pub fn seqno(&self) -> Option { self.origins.values().find(|origin| origin.declared).map(|o| o.seqno) } + #[cfg(test)] /// The declared identity's claim revision. pub fn revision(&self) -> Option { let origin = self.origins.values().find(|origin| origin.declared)?; @@ -849,16 +858,19 @@ impl Router { .collect() } + #[cfg(test)] /// The route this node would offer an upstream link for `id`. pub fn export(&self, id: Id) -> Option> { self.selected(id).filter(|route| !route.down_only) } + #[cfg(test)] /// Whether this node holds a reachable route to `id` but none is feasible. pub fn starved(&self, id: Id) -> bool { self.selected(id).is_none() && self.infeasible(id).next().is_some() } + #[cfg(test)] /// Whether this node, with no usable route to `id`, holds a held route it /// could recover through. pub fn grounded(&self, id: Id) -> bool { @@ -870,21 +882,25 @@ impl Router { self.rx.contains_key(&id) } + #[cfg(test)] /// Whether this node keeps a feasibility distance for `id`. pub fn remembers(&self, id: Id) -> bool { self.fd.contains_key(&id) } + #[cfg(test)] /// How many identities this node holds routes to. pub fn identities(&self) -> usize { self.rx.len() } + #[cfg(test)] /// How many qualified keys this node remembers the forwarded name of. pub fn aliases(&self) -> usize { self.filed.len() } + #[cfg(test)] /// How many region floors this node keeps for `id`. pub fn floors(&self, id: Id) -> usize { self.views.get(&id).map_or(0, |view| view.floors.len()) @@ -918,11 +934,13 @@ impl Router { &self.shown } + #[cfg(test)] /// Whether this node shows `claim` from `id`, and whether it is available. pub fn shows(&self, id: Id, claim: &Path) -> Option { self.shown.get(&(id, claim.clone())).map(|shown| shown.available) } + #[cfg(test)] pub fn retained(&self) -> Retained { let mut out = Retained::default(); for (id, view) in &self.views { diff --git a/rs/moq-net/src/route/proof.rs b/rs/moq-net/src/route/proof.rs index 7bc6fec6772..bdba760f977 100644 --- a/rs/moq-net/src/route/proof.rs +++ b/rs/moq-net/src/route/proof.rs @@ -76,6 +76,7 @@ pub(crate) struct Proof { impl Proof { /// Its encoded size; any other is malformed. + #[cfg(test)] pub const LEN: usize = KEY_LEN + SIGNATURE_LEN; pub fn to_bytes(self) -> Vec { diff --git a/rs/moq-net/tests/integration/support/harness.rs b/rs/moq-net/tests/integration/support/harness.rs index baac0218ad5..2fb68611a8d 100644 --- a/rs/moq-net/tests/integration/support/harness.rs +++ b/rs/moq-net/tests/integration/support/harness.rs @@ -186,7 +186,7 @@ pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { client = client.with_implementation(implementation); } if let Some(publish) = &opts.client_publish { - client = client.with_publisher(&publish.clone().with_proxy(opts.proxy)); + client = client.with_publisher(publish.clone().with_proxy(opts.proxy)); } let client_discovery = opts .client_subscribe @@ -202,7 +202,7 @@ pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { .with_versions(opts.version.into()) .with_extensions(opts.server_extensions); if let Some(publish) = &opts.server_publish { - server = server.with_publisher(&publish.clone().with_proxy(opts.proxy)); + server = server.with_publisher(publish.clone().with_proxy(opts.proxy)); } if let Some(subscribe) = opts.server_subscribe { server = server.with_subscriber(subscribe); From 917341a336ca2f7966da8e3a3f584d6ead0410b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:41:46 -0800 Subject: [PATCH 13/61] fix(net): drop a lite-06 route carrying this node's route identity A relay translating a moq-lite-07 claim for a moq-lite-06 peer starts the chain with the claiming node's route identity hop. When that chain came back to the claiming node over lite-06, nothing in it named the node's own origin hop, so it was filed as one of the node's own claims again and kept the broadcast circling after its publisher left. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 4 ++ rs/moq-net/src/model/origin.rs | 9 ++++ rs/moq-net/tests/integration/mesh_withdraw.rs | 44 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 18e5fecb62d..b932e1df108 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -865,6 +865,10 @@ impl Subscriber { } None => hops.contains(&self.self_origin), }; + // A claim of this node's that a moq-lite-07 relay translated for a lite-06 peer + // starts with its route identity's hop instead of this origin's. Filed again as + // an own claim, it would keep itself alive once its source left. + let reflected = reflected || self.origin.route_hop().is_some_and(|hop| hops.contains(&hop)); if reflected { tracing::debug!(route = %self.log_path(path), "dropping reflected announce"); return None; diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index bef12aef14b..9902ca030f9 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2388,6 +2388,15 @@ impl Producer { self.consume().route_node() } + /// The lite-06 hop this node's route identity travels as, once a moq-lite-07 + /// session created the route node: a relay translating this node's claims for a + /// lite-06 peer starts their chain with it. `None` while no session has, which + /// leaves the node uncreated. + pub(crate) fn route_hop(&self) -> Option { + let node = self.shared.lock().route_node.clone()?; + Some(crate::route::node::hop(node.declared())) + } + /// This origin's construction config. pub fn config(&self) -> Config { let shared = self.shared.lock(); diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 21bc17b1fcc..2c6a1be509b 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -635,3 +635,47 @@ async fn withdrawal_counts(capped: bool) -> (usize, usize, usize) { } (hunted, revived, extra) } + +/// A relay translating a moq-lite-07 claim for a lite-06 peer starts its chain with +/// the claiming node's route identity, not that node's hop, so the claim coming +/// back to that node over lite-06 is a reflection all the same. Filed as one of its +/// own claims again, the broadcast would outlive its publisher, carried around the +/// ring by itself. +#[kio::test] +async fn mixed_version_ring_withdraw_retracts() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let a = produce_origin(1); + let b = produce_origin(2); + let c = produce_origin(3); + let _ab = peer(lite07, &a, &b).await; + let _bc = peer(lite06, &b, &c).await; + // C publishes to A but hears nothing from it directly, so its only route to A's + // broadcast is B's translation. + let mut options = MockConnectOptions::new(lite06); + options.client_publish = Some(c.clone().with_link(origin::Link::Peer).consume().with_hidden(true)); + options.server_subscribe = Some(a.clone().with_link(origin::Link::Peer)); + let _ca = connect_mock(options).await; + + let mut at_a = watch(a.consume().routes()); + let mut at_c = watch(c.consume().routes()); + let broadcast = a.route("room/x", origin::RouteOptions::hidden()).unwrap(); + broadcast.announce().unwrap(); + kio::clock::sim::idle().await; + assert_eq!(drain(&mut at_a).get("room/x"), Some(&vec![Kind::Start])); + assert_eq!(drain(&mut at_c).get("room/x"), Some(&vec![Kind::Start])); + + drop(broadcast); + // A reflected claim circles the ring at one instant forever, so a regression never + // settles here and fails on the runner's timeout instead. + kio::clock::sim::idle().await; + for (relay, watched) in [("a", &mut at_a), ("c", &mut at_c)] { + let updates = drain(watched); + let kinds = updates.get("room/x"); + assert_eq!( + kinds.and_then(|kinds| kinds.last()), + Some(&Kind::End), + "{relay}: {kinds:?}" + ); + } +} From 28bb2b91f55ed66182a90bf2fae819508cbc3b34 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:41:51 -0800 Subject: [PATCH 14/61] docs: lite-07 routes, proxy opt-in, and lite-06 translation Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 21 +++++++++++++++++---- doc/lib/rs/moq-net.md | 1 + rs/moq-net/MIGRATE.md | 10 ++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index e1bead6ae27..11beeb67d9f 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -13,8 +13,20 @@ upstream bandwidth. Each broadcast carries the list of relays it passed through. That hop list catches loops and picks the shortest route, and every relay breaks ties the same way so the cluster converges instead of flapping. Both wire protocols -carry it: natively on moq-lite, and via the [cluster extension](/draft/moq-cluster) -on moq-transport. +carry it: natively on moq-lite up to moq-lite-06, and via the +[cluster extension](/draft/moq-cluster) on moq-transport. + +moq-lite-07-wip carries no hop list. Each publishing node declares a random +routing identity, and relays run a distance-vector route to it, loop-free by +Babel's feasibility condition, so every claim of one origin shares that route. +The first relay a request reaches picks the origin node, by a hash of the path +and the node's identity that every relay computes alike, and names it on the +request so later hops follow their own route to that node. A relay running both +versions translates: a moq-lite-06 peer's routes enter the route layer as the +relay's own claims, and lite-07 routes reach a moq-lite-06 peer as a hop list +that starts at their origin node. A relay drops a moq-lite-06 route whose hop +list names its own routing identity, since that is one of its claims coming +back. A track from a moq-transport publisher keeps its Object IDs, object properties, and END\_OF\_GROUP status objects across a moq-lite-07 link, since relays @@ -94,8 +106,9 @@ max_hops = 6 ``` The same setting is `--cluster-max-hops` or `MOQ_CLUSTER_MAX_HOPS`. It covers -moq-lite-03 through moq-lite-07-wip and moq-transport drafts 18 through 22 when -the MoQ Cluster extension is negotiated. A forwarded path counts this relay's +moq-lite-03 through moq-lite-06 and moq-transport drafts 18 through 22 when +the MoQ Cluster extension is negotiated. moq-lite-07-wip routes carry no hop +path, so it does not bound them. A forwarded path counts this relay's own hop even when its peer appends it on receipt. Older versions and peers without hop paths hide their depth, which this setting cannot bound. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 90c8bc15e34..bc265aba3b2 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -20,6 +20,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. +- **Relay by opting into proxying.** On moq-lite-07 a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 13e37ae6884..6bb0cd3d322 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,16 @@ How to update your code for each breaking change, newest first. ## Unreleased +### moq-lite-07 routes through the route layer ([#8023](https://github.com/moq-dev/moq/pull/8023)) + +moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node declares a random routing identity, every session on an origin feeds one router, and a request names the origin node the first hop picked. Older versions are unchanged. On moq-lite-07: + +- A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. +- A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. +- A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. +- ANNOUNCE_TAKEOVER is never sent. A job answering a claim at the claim's own path arrives as a restart, and one beneath it as a start, until [announce takeover](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) carries the claim's instance onto the job. +- A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. + ### Socket readers start at the beginning New `track` socket sessions subscribe from group 0, and `tcp` sessions join their From 6662e4d6609402aae06755c266f5545777fb8e31 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 02:59:32 -0800 Subject: [PATCH 15/61] test(net): a lost next hop restarts an epochless route, and lite-07 carries on One publisher reaches a relay through two neighbors and the preferred one goes away. Wires that mint an instance per announcement restart the route onto the standby's; lite-07 carries the publisher's instance and already serves the claim through both, so it continues with nothing announced. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/instance.rs | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/rs/moq-net/tests/integration/instance.rs b/rs/moq-net/tests/integration/instance.rs index f48942da2c6..e1063c3c107 100644 --- a/rs/moq-net/tests/integration/instance.rs +++ b/rs/moq-net/tests/integration/instance.rs @@ -100,3 +100,77 @@ async fn moq_transport_mints_per_announcement() { job_reannounced(version).await; } } + +/// One publisher reaches a relay through two neighbors, and the one the relay prefers +/// goes away. lite-07 carries the publisher's instance through both, so the relay keeps +/// serving the same stream through the other. Every other wire mints one per announcement, so changing +/// the next hop, with the same origin and no epoch, restarts the route rather than +/// updating it in place. +async fn next_hop_lost(version: &str) { + let version: Version = version.parse().unwrap(); + let publisher = produce_origin(1); + let preferred = produce_origin(2); + let standby = produce_origin(3); + let relay = produce_origin(4); + let _job = publisher + .route("job", moq_net::origin::RouteOptions::default()) + .unwrap(); + let _to_preferred = link(version, &publisher, &preferred).await; + let _to_standby = link(version, &publisher, &standby).await; + let mut announced = relay.consume().routes(); + let first_hop = link( + version, + &preferred, + &relay.clone().with_preference(origin::Preference::new(1)), + ) + .await; + let first = started(&mut announced).await; + let _second_hop = link(version, &standby, &relay).await; + if let Ok(event) = kio::clock::timeout(Duration::from_secs(1), announced.next()).await { + panic!("{version}: the standby route changed the preferred one: {event:?}"); + } + + first_hop.server.abort(moq_net::Error::Cancel); + first_hop.client.abort(moq_net::Error::Cancel); + let event = kio::clock::timeout(Duration::from_secs(10), announced.next()).await; + match version.to_string().as_str() { + // The relay already serves the claim through every feasible route, so the + // standby carries on with nothing to announce. + "moq-lite-07-wip" => { + if let Ok(event) = event { + panic!("{version}: expected the same stream to continue, got {event:?}"); + } + kio::clock::timeout(Duration::from_secs(10), relay.consume().request("job", None)) + .await + .expect("request hung") + .expect("the standby serves the claim"); + } + _ => match event.expect("announce hung").expect("announce ended") { + origin::RouteEvent::Restart(announce) => { + assert_ne!( + announce.route.instance(), + &first, + "{version}: spliced onto the old stream" + ) + } + other => panic!("{version}: expected a restart, got {other:?}"), + }, + } +} + +#[kio::test] +async fn next_hop_lost_lite07() { + next_hop_lost("moq-lite-07-wip").await; +} + +#[kio::test] +async fn next_hop_lost_restarts_older_lite() { + for version in ["moq-lite-06", "moq-lite-03"] { + next_hop_lost(version).await; + } +} + +#[kio::test] +async fn next_hop_lost_restarts_moq_transport() { + next_hop_lost("moq-transport-22").await; +} From 92cc680fd6bb47f428031ad771926c657165a0c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:02:59 -0800 Subject: [PATCH 16/61] test(net): a lite-07 route and its claims count against the announce cap Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/request_caps.rs | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/rs/moq-net/tests/integration/request_caps.rs b/rs/moq-net/tests/integration/request_caps.rs index 6e3928fc3fa..00817bddcd1 100644 --- a/rs/moq-net/tests/integration/request_caps.rs +++ b/rs/moq-net/tests/integration/request_caps.rs @@ -184,3 +184,31 @@ async fn an_announce_past_the_cap_closes_the_session() { ); } } + +/// On moq-lite-07 the peer's route takes a slot as well as each claim under it, so +/// whatever a peer makes the route layer hold stays under the cap: here its route and +/// first claim fill it, and the second claim closes the session. +#[kio::test] +async fn a_lite07_claim_past_the_cap_closes_the_session() { + let version: Version = "moq-lite-07-wip".parse().unwrap(); + let client = produce_origin(1); + let server = produce_origin(2); + let a = client.route("a", moq_net::origin::RouteOptions::hidden()).unwrap(); + let b = client.route("b", moq_net::origin::RouteOptions::hidden()).unwrap(); + a.announce().unwrap(); + b.announce().unwrap(); + + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(client.consume()); + options.server_subscribe = Some(server.clone()); + options.server_limits = Some(limits(2)); + let pair = connect_mock(options).await; + + let err = kio::clock::timeout(TIMEOUT, pair.client.closed()) + .await + .expect("the session stayed open"); + assert!( + matches!(err, moq_net::Error::Session(moq_net::SessionError::TooManyRequests)), + "closed with {err}" + ); +} From db9b5e331d6cdcc27a72ca7b76e611c7728ffbe9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:02:59 -0800 Subject: [PATCH 17/61] chore(quest): narrow routes/session.md to bounded per-stream state Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/session.md | 184 +++++---------------- 1 file changed, 44 insertions(+), 140 deletions(-) diff --git a/quest/m2/cluster-routing/routes/session.md b/quest/m2/cluster-routing/routes/session.md index 53002f8076c..cd06f1bd925 100644 --- a/quest/m2/cluster-routing/routes/session.md +++ b/quest/m2/cluster-routing/routes/session.md @@ -1,150 +1,54 @@ -# [XL] Route session integration +# [L] Route session integration: bounded state ## Goal -One `Router` per origin node, fed by every lite-07 session, drives that -node's origin and every announce stream it answers. Its shown claims and -selected routes become the origin's routes, its outbox becomes each -stream's ROUTE and ANNOUNCE messages, and the first hop pins each request -to the identity it selected. A lite-06 peer is translated at the relay. A -peer can make the router hold only bounded state. The configuration -changes the contract decided ship with it. - -lite-07's live wire switches to the route layer that -`drafts/draft-lcurley-moq-lite.md` specifies, in Rust: ANNOUNCE_START, ANNOUNCE_UPDATE, ANNOUNCE_RESTART, and -ANNOUNCE_TAKEOVER name a Route ID and carry the Announce Cost in place of -the Hop ID list (`HopsRef`, `Hop Base`/`Hop Keep`, and `cluster.max_hops` -go from lite-07); ANNOUNCE_OK loses its Hop ID and SETUP its Hop parameter; -TRACK, TRACKS, SUBSCRIBE, FETCH, and DATAGRAMS carry the Node ID; a Cost -parameter of 0 is refused; the subscriber sends ROUTE_REQUEST and echoes -ROUTE_END. The new messages' codecs already exist (`AnnounceBroadcast`'s -route variants and `AnnounceReply`); until this lands the subscriber -refuses route messages. A hop-less ANNOUNCE cannot run before the router -drives sessions, which is why the switch lands here. +The route layer drives every lite-07 session +([#8023](https://github.com/moq-dev/moq/pull/8023)): one `route::Node` per +origin, Route IDs on the wire, requests pinned to the first hop's origin node, +lite-06 translated at the relay. What remains: the state a node keeps per +stream it answers is bounded by the origins that stream reads, not the whole +node table, and the public API drops what the route layer replaced. ## Plan - Scope route state to interested origins, not the CDN's whole node table. - Send a route before any announcement referencing it. The epic's handoff contract - defines held routes, letting go, and collection. -- ✅ Decided 2026-10-15 (maintainer): retract and re-file when a session's - grant union crosses the scoped boundary in either direction (a `**` token - added, or its expiry). Every forwarded route the session delivered is - retracted, and its current advertisements are re-filed under the new key, - so no route, feasibility distance, or alias stays under the old one. - Expiry cannot be refused, so refusing the transition was not an option. -- Only the first hop selects the origin. It keeps `route_order`'s shape over - the origin nodes announcing the path: longest prefix, newest epoch (within - one instance only, per [Instance and epoch](https://github.com/moq-dev/moq/pull/7489), - decided 2026-10-30), the link's priority (`origin::Producer::with_preference`), seed plus route - metric, then a rendezvous hash of the path and the identity, and the newest - connection only for an otherwise identical route. The hash drops the hop - chain, which differs per node, so every node agrees on a pool member - (`pool_ties_agree_only_on_a_path_and_identity_hash`). The request then names - that identity, and every later hop follows its own route to it, so - forwarding needs a route but not the claim. -- Reselecting the origin at every hop loops although each origin's routes are - loop-free (RFC 8966 s2.7): a ring of same-epoch origins whose relays - prioritize different links, and nested wildcards where a relay has not - heard the specific claim, both cycle past requester exclusion; joining - another relay's subscription by path or instance deadlocks; and without an - instance a later hop splices another publisher under the requester's - announcement with no Restart (`local_priorities_loop_path_only_requests`, - `overlapping_wildcards_loop_where_specific_claims_are_unheard`, - `aggregation_cycles_starve_without_a_forwarding_constraint`, - `reselection_without_an_instance_splices_another_publisher`, - `sweep_forwarding_progress`). A relay therefore joins a forwarded request - only to a subscription following its route to the same identity. When that - identity becomes unreachable the forwarded request ends, and the first hop - re-resolves, resuming only on a matching epoch. A request carrying a cost - bound instead also never loops, but stalls where a transit hop has not - heard the claim and still splices epochless sources. -- Mixed versions: a lite-06 peer keeps today's path vector, translated at the - relay that speaks both, for the rollout window only. -- Interest changes: key the router by announce stream, not session. The old - stream stays open, its routes counting and its batches discarded, until - the new stream has answered, so no route drops in between. A stream is - just another neighbor key to the router. -- A batch carries changes against the last batch on its stream. Restricting - a view to an interest is still linear in that origin's claims per change; - make it incremental here. -- Decided 2026-11-05 (maintainer, /quest-plan, the epic's Plan items 10 to - 12): each stream's floor is a whole-interest marker. Project the router's - per-region view onto it, certifying nothing unless the view knows the - route's claims across the stream's interest at one revision; present per - route only the most specific claim covering the interest, moved rather - than ended when it is withdrawn; and certify absence only for what the - stream's `Hidden` filter shows. Rerun the handoff sweeps through that - projection: a scope a relay lost now goes offline downstream only on - ROUTE_END, the accepted cost, so update the scoped fixtures' expectations - to it, while no sweep may revive a known withdrawal or report a false END. - Also cover a fresh claim that arrives only on an uncertified stream while - an older certified floor on another stream proves it absent - ([review](https://github.com/moq-dev/moq/pull/7764#discussion_r4485081975)): - the merge rule hides it until the certifying peer refreshes or lets go, - and the test pins that bound. -- Bound what a peer makes the router hold (routes, claims, views per - session) by design, as the session caps announcements today. -- Public API: the route-change surface on `broadcast::Route` and its - bindings will likely change, `Hops` and saturating `Cost` go, zero-cost - links are refused (update `?cost=0` examples and report the breaking - configuration change), a per-connection `proxy` preference is added, and - `cluster.id`/`--cluster-id` is removed rather than kept as an unused label - (decided 2026-10-13). `Preference` continues to rank feasible transit - routes; report it. -- Open, for the implementer to settle and record: how much of today's route - trie, fronts, and `route_order` survives intact. - -Decided 2026-10-27 (maintainer): the route layer deletes the epochless instance -machinery in `rs/moq-net/src/model/origin.rs`: `Instance::Route(generation)` -and its renewals, `restarted_since`, the held-restart Release for epochless -routes, and Selection retirement, leaving Follow's same-epoch -Update/Restart split as the only takeover rule. Once each origin is its own -ANNOUNCE and the first hop pins the request to its identity, the first hop's -route is the instance, so the per-hop renewal has nothing left to protect. -An epochless change of serving route stays a Restart, including the -same-origin next-hop change with no epoch (test it), and the routing -contract's epochless fixtures keep passing. Minting an epoch for every route -instead was tried and rejected -([moq-dev/moq#6948](https://github.com/moq-dev/moq/pull/6948)): each relay -mints at its own time, so mint order outranks cost, preference, and the -local-broadcast tie and relays disagree on a pool path's winner, while -ranking a minted epoch last keeps the renewal it was meant to delete. - -Moved 2026-10-30 (maintainer, the [route epic](/quest/m1/api-audit/route/README.md)): -[Instance and epoch](https://github.com/moq-dev/moq/pull/7489) now deletes that -epochless machinery, so the route layer no longer does. A session mints -an `instance` (`~` plus random bits) for any route that -arrives without one, and newest-wins compares epochs only within one -instance, so mint order no longer outranks cost or preference. The rest of -the 2026-10-27 decision stands. The broadcast epoch fields here become that -publisher instance and its per-instance epoch, in place of the UUIDv7 -epoch. The route layer's own origin-node identity, with its seqno and -revision, stays separate ("keep administrative identity and broadcast epoch -separate", in the epic). - -Decided 2026-11-05 (maintainer, /quest-plan): a scoped peer's identities -are keyed by a route key the peer proves in its lite-07 SETUP: the session's -`PeerSetup::route_key` once its SETUP arrives, and `None` for a peer that proved -none. Relays set theirs with `--cluster-route-key`. Every identity a scoped, proven peer sends, its own included, is qualified -under its proven key, so relays sharing a key file fail over as one. A peer that proves no -key gets a per-session key. `Scope::qualify` still takes the key from the -caller, so this quest does not wait on the proof. Options shown: - -- ✅ A route key the peer proves once per session in SETUP, before the lite-07 cut - (2026-11-05, maintainer, /quest-plan; supersedes the per-announce proof option) -- A route label carried in the grant (the earlier recommendation), with the - proven key added after the cut -- A per-session key now -- A key the relay operator configures per scoped remote - -Research behind it: proving a neighbor's own key cannot remove qualification, -since a public key is public and a scoped peer could still forward a victim's -key with an inflated seqno; origin-signed routes still leave cost lies and -blackholing, at the price of per-update signatures and expiry refreshes. Keyed -by a proven key, every remaining attack stays inside the forger's grant, with -no grant field, no issuer-coordinated labels, and support for token-less -admission. + Today the router advertises every route to every watching stream, because + the epic's handoff contract (`routes/README.md`, "a sender that certifies + anything for a route answers every stream") has a sender answer even a + stream whose interest none of the route's regions meets. Scoping needs + that sentence amended (see Waiting). + What a peer makes the router hold is already capped: each route and claim + it sends takes one of the session's announce slots + (`a_lite07_claim_past_the_cap_closes_the_session`). What this bounds is + the state this node keeps per stream it answers. +- Present incrementally. `View::present` walks every claim of a route per + change and per interest (`Router::advertise`), so a large origin costs + linear work per update. Do it with the scoping change, which reshapes the + same per-stream presentation. +- Public API: `Hops`, `RouteInfo::hops`, and the saturating `Cost` stay while + lite-06 and the moq-transport cluster extension still carry hop chains. + Remove them, or hide them behind those wires, when the rollout window + closes (see Waiting). + +## Waiting + +- Maintainer: amend the handoff contract's "a sender that certifies anything + for a route answers every stream" (`routes/README.md`), or keep it. It is a + decided contract, so an agent does not change it. Options: **send a route + only while it meets the stream's interest (recommended)**: a route starts on + a stream before its first claim in or covering the interest, and when its + last such claim goes it sends a floor certifying the withdrawal, then ends + the route, so withdrawals still END and per-stream state is bounded by the + origins a peer reads, at a ROUTE_END round trip when an origin leaves an + interest; or **keep answering every stream** and bound only by per-peer + caps, simpler but carrying the whole node table on every stream. Done when + the README says which. +- Maintainer: decide when `Hops`, `RouteInfo::hops`, and the saturating + `Cost` leave the public API. Every older wire still fills them, so an agent + cannot drop them without picking the rollout's end. Options: **when lite-06 + retires (recommended)**, since removing them now only moves them behind a + private type; or **now**, keeping a private chain type for the older wires. + Done when the Plan item above names the point. ## Related From 2c9c3964c1f6b44c4d605a06804c88a3da484340 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 03:07:27 -0800 Subject: [PATCH 18/61] chore(net): allow route_hop unused without the async session Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c6bc7fe435b..a3d52a3b704 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2392,6 +2392,7 @@ impl Producer { /// session created the route node: a relay translating this node's claims for a /// lite-06 peer starts their chain with it. `None` while no session has, which /// leaves the node uncreated. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn route_hop(&self) -> Option { let node = self.shared.lock().route_node.clone()?; Some(crate::route::node::hop(node.declared())) From a627b584b67d4cbf1a5b8393bb2c32fa1e540c2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 08:48:40 -0800 Subject: [PATCH 19/61] feat(net): lite-07 carries ANNOUNCE_TAKEOVER on Route IDs A route-layer claim records the claim it took over and the instance that one served. The origin's own takeovers feed it, a relay keeps it from the ANNOUNCE_TAKEOVER it received, and an answering stream sends the claim as a takeover of its predecessor when the peer held that predecessor, under the same route, before the batch, and every claim it holds at that path serves the same instance. The receiver forwards the claim with its predecessor's route, and in place keeps the predecessor's sources. The legacy publisher and subscriber takeover paths were unreachable once lite-07 routed, so they go. Co-Authored-By: Claude Opus 5.5 --- quest/m2/announce-takeover.md | 6 +- rs/moq-net/MIGRATE.md | 1 - rs/moq-net/src/lite/publisher.rs | 55 +----- rs/moq-net/src/lite/publisher/routed.rs | 143 +++++++++++---- rs/moq-net/src/lite/subscriber.rs | 165 ++---------------- rs/moq-net/src/lite/subscriber/routed.rs | 118 ++++++++++++- rs/moq-net/src/model/origin.rs | 28 +-- rs/moq-net/src/route/node.rs | 38 ++-- .../tests/integration/announce_takeover.rs | 27 +-- 9 files changed, 295 insertions(+), 286 deletions(-) diff --git a/quest/m2/announce-takeover.md b/quest/m2/announce-takeover.md index 710e5ae1799..5323a2d5111 100644 --- a/quest/m2/announce-takeover.md +++ b/quest/m2/announce-takeover.md @@ -44,8 +44,10 @@ instance rule already covers the claim case and needs no wire message, so this quest removes the predecessor machinery and grows to [XL]. - **Removing ANNOUNCE_TAKEOVER.** Delete the 0x5 message and its codec in - `rs/moq-net/src/lite/announce.rs`, `compress.rs`, `publisher.rs`, - `subscriber.rs` and `replay.rs`; the predecessor tracking in + `rs/moq-net/src/lite/announce.rs`, `compress.rs`, `publisher/routed.rs`, + `subscriber/routed.rs` and `replay.rs`; the route layer's claim lineage + (`Claim::continues` in `rs/moq-net/src/route/node.rs`, fed by `OwnClaims` in + `origin.rs`); the predecessor tracking in `rs/moq-net/src/model/origin.rs` and `front.rs` (the `seen` stamps in `OriginConsumerState` and `PendingFront::via`); relay forwarding in `rs/moq-relay`; and `rs/moq-net/tests/integration/announce_takeover.rs`. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index b5c634ff3ab..61f04b2cb1d 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -11,7 +11,6 @@ moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node dec - A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. - A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. - A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. -- ANNOUNCE_TAKEOVER is never sent. A job answering a claim at the claim's own path arrives as a restart, and one beneath it as a start, until [announce takeover](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) carries the claim's instance onto the job. - A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. ### `track::Consumer::unmetered` reads outside any stats session diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 09cbdc8dbc0..7b5d610bff0 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1371,43 +1371,6 @@ impl AnnounceRun { Ok(()) } - /// Advertise `suffix` as continuing the peer's live `predecessor` (suffix and id): - /// ANNOUNCE_TAKEOVER, which retires the predecessor when it held `suffix`. - #[expect(clippy::too_many_arguments, reason = "private plumbing mirroring start and restart")] - fn takeover( - &mut self, - stream: &mut Stream, - predecessor: (crate::Path, u64), - suffix: crate::Path, - hops: Hops, - cost: crate::origin::Cost, - route: Served, - absolute: &crate::Path, - ) -> Result<(), Error> { - tracing::debug!(route = %absolute, predecessor = %predecessor.0, "takeover"); - let (id, wire) = self.encoder.takeover(predecessor.1, suffix.clone()); - self.live.insert( - suffix, - Advertised { - id: Some(id), - hops, - cost, - served: route.clone(), - }, - ); - let (instance, epoch) = route; - stream.writer.buffer(&lite::AnnounceBroadcast::Takeover { - predecessor: predecessor.1, - suffix: wire, - route: 0, - instance, - epoch, - cost, - })?; - self.sent(stream)?; - Ok(()) - } - /// Replace the peer's advertisement for `suffix` with another instance or epoch: /// ANNOUNCE_RESTART on lite-07, an end and a fresh start before it. fn restart( @@ -1662,26 +1625,10 @@ impl AnnounceRun { continue; } - // A takeover names its predecessor's id, so only one the peer holds. Elsewhere it + // Only lite-07 carries a takeover, and it answers from the route layer. Here one // is another instance, in place or not. - let taken = predecessor - .as_ref() - .filter(|_| self.version.has_announce_takeover()) - .filter(|predecessor| **predecessor == suffix || !self.live.contains_key(&suffix)) - .and_then(|predecessor| Some((predecessor.clone(), self.live.get(predecessor)?.id?))); let restart = restart || predecessor.as_ref() == Some(&suffix); match self.outgoing(&update.route, &absolute) { - Some((hops, cost)) if let Some((predecessor, id)) = taken => { - self.takeover( - stream, - (predecessor, id), - suffix, - hops, - cost, - served(&update.route), - &absolute, - )?; - } Some((hops, cost)) if restart && self.live.contains_key(&suffix) => { self.restart(stream, suffix, hops, cost, served(&update.route), &absolute)?; } diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 74ed7ad79de..865c15b9b23 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -7,7 +7,7 @@ //! peer cannot tell what was left out from what is gone. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, task::{Poll, ready}, }; @@ -278,8 +278,9 @@ impl Answer { paths.push(path); } sent.floor = floor; + let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path)?; + self.sync(stream, id, path, &mut started)?; } self.floor(stream, id) } @@ -316,47 +317,87 @@ impl Answer { } /// Bring the peer's claim at `path` for the route to `id` in line with the node's. - fn sync(&mut self, stream: &mut Stream, id: route::Id, path: Path) -> Result<(), Error> { + /// + /// `started` collects the claims this pass started: a claim takes over only one the + /// peer held before, so a snapshot replays no takeover. + fn sync( + &mut self, + stream: &mut Stream, + id: route::Id, + path: Path, + started: &mut BTreeSet, + ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); let wanted = sent.claims.get(&path).filter(|_| self.shows(&path)).cloned(); let wire = sent.wire; + let taken = (wanted.as_ref()) + .and_then(|claim| self.continued(id, &path, claim)) + .filter(|(taken, _)| !started.contains(taken)); let key = (id, path); match (self.live.get_mut(&key), wanted) { (None, Some(claim)) => { - tracing::debug!(route = %key.1, node = id, "announce"); let suffix = key .1 .strip_prefix(&self.prefix) .expect("shown claims are under the prefix"); - let (announce, suffix) = self.encoder.start(suffix); - let announce = announce.expect("lite-07 assigns announce ids"); - stream.writer.buffer(&Msg::Active { - suffix, - route: wire, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; + let announce = match taken { + Some((taken, predecessor)) => { + tracing::debug!(route = %key.1, node = id, predecessor = %taken, "takeover"); + self.takeover(stream, wire, predecessor, suffix, &claim)? + } + None => { + tracing::debug!(route = %key.1, node = id, "announce"); + let (announce, suffix) = self.encoder.start(suffix); + stream.writer.buffer(&Msg::Active { + suffix, + route: wire, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + announce.expect("lite-07 assigns announce ids") + } + }; + started.insert(key.1.clone()); self.live.insert(key, (announce, claim)); } (Some((announce, held)), Some(claim)) => { - if held.instance != claim.instance || held.epoch != claim.epoch { - tracing::debug!(route = %key.1, "restart"); - stream.writer.buffer(&Msg::Restart { - id: *announce, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - cost: origin::Cost::new(claim.cost), - })?; - } else if held.cost != claim.cost { - stream.writer.buffer(&Msg::Update { - id: *announce, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; - } - *held = claim; + let announce = *announce; + let restart = held.instance != claim.instance || held.epoch != claim.epoch; + let reprice = held.cost != claim.cost; + let replacement = match taken { + // In place: a new Announce ID replaces the one it continues. + Some((taken, predecessor)) if taken == key.1 => { + tracing::debug!(route = %key.1, node = id, "takeover"); + let suffix = key + .1 + .strip_prefix(&self.prefix) + .expect("held claims are under the prefix"); + self.takeover(stream, wire, predecessor, suffix, &claim)? + } + _ if restart => { + tracing::debug!(route = %key.1, "restart"); + stream.writer.buffer(&Msg::Restart { + id: announce, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + cost: origin::Cost::new(claim.cost), + })?; + announce + } + _ => { + if reprice { + stream.writer.buffer(&Msg::Update { + id: announce, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + } + announce + } + }; + self.live.insert(key, (replacement, claim)); } (Some(_), None) => { let (announce, _) = self.live.remove(&key).expect("matched above"); @@ -368,6 +409,45 @@ impl Answer { Ok(()) } + /// Send `claim` at `suffix` under the route `wire` as an ANNOUNCE_TAKEOVER of the + /// peer's live `predecessor`, returning its Announce ID. + fn takeover( + &mut self, + stream: &mut Stream, + wire: u64, + predecessor: u64, + suffix: Path, + claim: &node::Claim, + ) -> Result { + let (announce, suffix) = self.encoder.takeover(predecessor, suffix); + stream.writer.buffer(&Msg::Takeover { + predecessor, + suffix, + route: wire, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + cost: origin::Cost::new(claim.cost), + })?; + Ok(announce) + } + + /// The claim the peer holds that `claim` at `path` takes over, and its Announce ID. + /// + /// Only the claim it continues, under the same route at the instance it continued, + /// and only while every claim the peer holds at that path serves the same instance: + /// otherwise the peer cannot tell which one it resolved through. + fn continued(&self, id: route::Id, path: &Path, claim: &node::Claim) -> Option<(Path, u64)> { + let (taken, instance) = claim.continues.as_ref()?; + if !path.has_prefix(taken) { + return None; + } + let (predecessor, held) = self.live.get(&(id, taken.clone()))?; + let alike = self.routes.keys().all(|other| { + (self.live.get(&(*other, taken.clone()))).is_none_or(|(_, other)| other.instance == *instance) + }); + (held.instance == *instance && alike).then(|| (taken.clone(), *predecessor)) + } + /// The grant changed: end what it no longer covers, start what it now does, and /// floor every route again, which certifies only while the grant covers the prefix. fn refilter(&mut self, stream: &mut Stream) -> Result<(), Error> { @@ -375,8 +455,9 @@ impl Answer { ids.sort_unstable(); for id in ids { let paths: Vec = self.routes[&id].claims.keys().cloned().collect(); + let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path)?; + self.sync(stream, id, path, &mut started)?; } if self.routes[&id].floored { self.floor(stream, id)?; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index b932e1df108..39931d853b1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -599,35 +599,10 @@ impl Subscriber { &mut run.announced, )?; } - lite::AnnounceBroadcast::Takeover { - predecessor, - suffix, - instance, - epoch, - cost, - .. - } => { - // Resolved like a start against the live predecessor, which it retires when - // it takes the predecessor's own suffix. An unknown or retired predecessor is - // a protocol violation. - let (previous, suffix) = run.decoder.takeover(predecessor, suffix)?; - let previous = prefix.join(&previous)?; - let path = prefix.join(&suffix)?; - // Lite-07 carries no hop chain; its claims name their route instead. - self.takeover_announce( - previous, - path, - instance.map(|instance| (instance, epoch)), - crate::Hops::new(), - cost, - run.link_cost, - run.responder_origin, - &mut run.announced, - )?; - } lite::AnnounceBroadcast::Skipped | lite::AnnounceBroadcast::Synced => {} - // Only lite-07 carries the route layer, which `routed` handles above. - lite::AnnounceBroadcast::RouteStart { .. } + // Only lite-07 carries the route layer and takeovers, which `routed` handles above. + lite::AnnounceBroadcast::Takeover { .. } + | lite::AnnounceBroadcast::RouteStart { .. } | lite::AnnounceBroadcast::RouteUpdate { .. } | lite::AnnounceBroadcast::RouteHold { .. } | lite::AnnounceBroadcast::RouteEnd { .. } @@ -683,56 +658,7 @@ impl Subscriber { // outside our scope, so don't serve it. Reflections are already // filtered above. let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); - Ok(announced.offer(self, path, route, None)) - } - - /// Handle an ANNOUNCE_TAKEOVER: a new advertisement at `path` continuing what the peer's - /// live advertisement at `previous`, at or above it, served beneath it. - /// - /// At `previous` itself it replaces that advertisement like a restart; elsewhere it - /// starts one, and the predecessor stays. Either way the new route enters the origin - /// before anything leaves, taking over from the route the predecessor attached, if it - /// did, so what resolved through that one beneath `path` carries on. - #[expect( - clippy::too_many_arguments, - reason = "private plumbing; a single-use argument struct would only rename the parameters" - )] - fn takeover_announce( - &mut self, - previous: Path, - path: Path, - // See `start_announce`. - instance: Option<(crate::Instance, u64)>, - hops: crate::Hops, - cost: crate::origin::Cost, - link_cost: u64, - responder_origin: Option, - announced: &mut Announced, - ) -> Result { - // The continued route lies beneath its predecessor. - if !path.has_prefix(&previous) { - return Err(Error::ProtocolViolation); - } - if path == previous { - announced.restart(&path, instance); - } else { - // Like a start: one current advertisement per prefix. - if announced.contains(&path) { - return Err(Error::ProtocolViolation); - } - announced.reserve(path.clone(), instance)?; - } - - let Some(hops) = self.chain(&path, hops, responder_origin) else { - if path == previous { - announced.declined(path); - } - return Ok(false); - }; - - tracing::debug!(route = %self.log_path(&path), predecessor = %self.log_path(&previous), "takeover"); - let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); - Ok(announced.offer(self, path, route, Some(&previous))) + Ok(announced.offer(self, path, route)) } /// The route to announce for a prefix this peer advertised, charging our @@ -818,7 +744,7 @@ impl Subscriber { return Ok(true); } - Ok(announced.offer(self, path, metadata, None)) + Ok(announced.offer(self, path, metadata)) } /// Handle an ANNOUNCE_RESTART: a live advertisement serves another instance, or a @@ -849,7 +775,7 @@ impl Subscriber { tracing::debug!(route = %self.log_path(&path), hops = hops.len(), "restart"); let route = self.announced_route(&path, hops, cost, link_cost, responder_origin, announced); // Held to the limit like a start: a restart outside it is withheld, dropping the old route. - Ok(announced.offer(self, path, route, None)) + Ok(announced.offer(self, path, route)) } /// The full chain of a new or replacing advertisement, or `None` when it is @@ -7735,7 +7661,7 @@ mod tests { assert_eq!(hops, vec![crate::Hop::UNKNOWN]); } - fn restart_subscriber(session: SinkSession) -> (Subscriber, crate::origin::Consumer) { + pub(super) fn restart_subscriber(session: SinkSession) -> (Subscriber, crate::origin::Consumer) { let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let consumer = origin.consume(); let subscriber = Subscriber::new(SubscriberConfig { @@ -7753,53 +7679,6 @@ mod tests { (subscriber, consumer) } - /// A takeover in place keeps the sources the predecessor minted open: they serve the - /// instance it continues. A restart in place closes them. - #[kio::test] - async fn a_takeover_in_place_keeps_the_predecessors_sources() { - for takeover in [true, false] { - let (mut subscriber, _consumer) = restart_subscriber(SinkSession::new(Default::default())); - let mut announced = Announced::default(); - let path = Path::from_static("room/host"); - let hop = Some(crate::Hop::new(7).unwrap()); - subscriber - .start_announce( - path.clone(), - None, - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ) - .unwrap(); - let minted = announced.attached(&path).unwrap().live.consume(); - - match takeover { - true => subscriber.takeover_announce( - path.clone(), - path.clone(), - None, - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ), - false => subscriber.restart_announce( - path.clone(), - crate::Hops::new(), - crate::origin::Cost::UNKNOWN, - 0, - hop, - &mut announced, - ), - } - .unwrap(); - assert_eq!(minted.is_closed(), !takeover, "takeover: {takeover}"); - } - } - /// A restart re-prices the announced route in place: consumers observe another /// active update with the new metadata rather than a retract-and-announce. #[kio::test] @@ -8778,15 +8657,8 @@ impl Announced { /// Put the peer's route into the origin, so paths under it resolve through this /// session on demand, or hold it back when the session's limit does not cover it. - /// The route takes over from the one attached at `predecessor`, if any. Returns - /// whether it attached. - fn offer( - &mut self, - subscriber: &Subscriber, - path: Path, - route: crate::origin::RouteInfo, - predecessor: Option<&Path>, - ) -> bool { + /// Returns whether it attached. + fn offer(&mut self, subscriber: &Subscriber, path: Path, route: crate::origin::RouteInfo) -> bool { if !subscriber .auth .within_limit(crate::auth::Direction::Subscribe, path.as_str()) @@ -8795,23 +8667,10 @@ impl Announced { self.routes.insert(path, Held::Withheld(route)); return false; } - let in_place = predecessor == Some(&path); // An error means the prefix is outside our origin's scope, so don't serve it. - let predecessor = predecessor.and_then(|previous| match self.routes.get(previous)? { - Held::Attached(attached) => Some(&attached.dynamic), - _ => None, - }); - match subscriber.origin.forward(&path, route.clone(), predecessor) { + match subscriber.origin.forward(&path, route.clone(), None) { Ok(dynamic) => { - // In place, the successor continues the instance the predecessor's sources - // serve: they live on with it rather than closing with the advertisement. - let live = in_place - .then(|| self.attached(&path).map(|previous| std::mem::take(&mut previous.live))) - .flatten(); - self.attach(path.clone(), route, dynamic); - if let (Some(live), Some(attached)) = (live, self.attached(&path)) { - attached.live = live; - } + self.attach(path, route, dynamic); true } Err(_) => { @@ -8930,7 +8789,7 @@ impl Announced { if subscriber.going_away.is_set() { route.cost = crate::origin::Cost::DRAIN; } - self.offer(subscriber, path, route, None); + self.offer(subscriber, path, route); } _ => unreachable!("only attached and withheld routes change"), } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 6a084efa3d1..8a006e25058 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -205,14 +205,20 @@ impl Routed { epoch, cost, } => { - let (previous, suffix) = decoder.takeover(predecessor, suffix)?; - let (previous, path) = (self.path(prefix, &previous)?, self.path(prefix, &suffix)?); - // Kept compiling only: lite-07's ANNOUNCE_TAKEOVER is on its way out, so a - // claim taking over its predecessor's own path simply replaces it. - if previous == path { + let (_, suffix) = decoder.takeover(predecessor, suffix)?; + let path = self.path(prefix, &suffix)?; + // The predecessor is a live claim of the same route covering the new one. + let taken = self.claims.get(&predecessor).ok_or(Error::ProtocolViolation)?; + if taken.route != route || !path.has_prefix(&taken.path) { + return Err(Error::ProtocolViolation); + } + let continues = (taken.path.clone(), taken.claim.instance.clone()); + // At its own path it replaces the predecessor. + if continues.0 == path { self.end(predecessor)?; } - let claim = claim(instance, epoch, cost)?; + let mut claim = claim(instance, epoch, cost)?; + claim.continues = Some(continues); self.start(decoder.last(), route, path, claim)?; } Msg::Update { id, cost, .. } => { @@ -403,12 +409,29 @@ impl Routed { return; } } + // The route this one takes over, as this stream serves it at the instance it + // continues, so what resolved through that one beneath the claim carries on. + let taken = (served.claim.continues.as_ref()) + .and_then(|(taken, instance)| { + let entry = self.served.get_mut(&(id, taken.clone()))?; + (entry.route.route.instance.as_ref() == Some(instance)).then_some((taken, entry)) + }) + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. + true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), + false => (&entry.route.dynamic, None), + }); // A new instance or epoch replaces the old route, attached before it goes. tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); - match subscriber.origin.forward(&path, route.clone(), None) { + let (predecessor, live) = taken.unzip(); + match subscriber.origin.forward(&path, route.clone(), predecessor) { Ok(dynamic) => { let wake = crate::util::QueueWake::new(key.clone(), self.ready.clone()); - let entry = AnnouncedRoute::new(route, dynamic, wake); + let mut entry = AnnouncedRoute::new(route, dynamic, wake); + if let Some(live) = live.flatten() { + entry.live = live; + } entry.waker.wake_by_ref(); self.served.insert(key.clone(), ServedClaim { named, route: entry }); self.stream.serving(id, &key.1); @@ -457,6 +480,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos instance: instance.ok_or(Error::ProtocolViolation)?, epoch, cost: cost.value(), + continues: None, }) } @@ -483,3 +507,81 @@ fn scoped(link: &route::Link, regions: &Option>, key: [u8; 32]) -> rou ..link.clone() } } + +#[cfg(test)] +mod tests { + use crate::lite::{AnnounceBroadcast as Msg, AnnounceDecoder, PathRef, test_transport::SinkSession}; + use crate::{Instance, Path, origin::Cost, route::Advert}; + + /// A takeover in place keeps the sources the predecessor minted open: they serve the + /// instance it continues. A restart in place closes them. + #[kio::test] + async fn a_takeover_in_place_keeps_the_predecessors_sources() { + for takeover in [true, false] { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let path = Path::from_static("room/host"); + let advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + let start = Msg::Active { + suffix: PathRef::literal(path.clone()), + route: 0, + instance: Some(Instance::random()), + epoch: 0, + hops: Default::default(), + cost: Cost::new(0), + }; + for msg in [Msg::RouteStart { node: 7, advert }, start] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed + .receive( + &prefix, + Msg::Floor { + id: 0, + revision: Some(0), + }, + &mut decoder, + ) + .unwrap(); + routed.outputs(&subscriber); + let minted = routed.served[&(7, path.clone())].route.live.consume(); + + let instance = Some(Instance::random()); + let replace = match takeover { + true => Msg::Takeover { + predecessor: 0, + suffix: PathRef::literal(path.clone()), + route: 0, + instance, + epoch: 0, + cost: Cost::new(0), + }, + false => Msg::Restart { + id: 0, + instance, + epoch: 0, + cost: Cost::new(0), + }, + }; + routed.receive(&prefix, replace, &mut decoder).unwrap(); + routed + .receive( + &prefix, + Msg::Floor { + id: 0, + revision: Some(1), + }, + &mut decoder, + ) + .unwrap(); + routed.outputs(&subscriber); + assert_eq!(minted.is_closed(), !takeover, "takeover: {takeover}"); + } + } +} diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index c4f1b275266..ebb0ed615d7 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3614,21 +3614,21 @@ impl OwnClaims { } fn apply(&self, event: RouteEvent) { - match event { - RouteEvent::Start(announce) - | RouteEvent::Update(announce) - | RouteEvent::Restart(announce) - | RouteEvent::Takeover { announce, .. } => { - let route = announce.route; - let claim = route.instance.map(|instance| crate::route::node::Claim { - instance, - epoch: route.epoch, - cost: route.cost.value(), - }); - self.node.own(announce.prefix, claim); + let (announce, predecessor) = match event { + RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { + (announce, None) } - RouteEvent::End(announce) => self.node.own(announce.prefix, None), - } + RouteEvent::Takeover { announce, predecessor } => (announce, Some(predecessor)), + RouteEvent::End(announce) => return self.node.own(announce.prefix, None, None), + }; + let route = announce.route; + let claim = route.instance.map(|instance| crate::route::node::Claim { + instance, + epoch: route.epoch, + cost: route.cost.value(), + continues: None, + }); + self.node.own(announce.prefix, claim, predecessor.as_ref()); } } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e2799902c9f..3509f01cbb8 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -35,6 +35,11 @@ pub(crate) struct Claim { pub epoch: u64, /// The origin's Announce Cost. pub cost: u64, + /// The claim of the same identity this one took over, at or above its path, and the + /// instance that claim served: what was resolved through it beneath this path is + /// this claim's broadcast. Kept while the instance stays, and sent as + /// ANNOUNCE_TAKEOVER to a peer still holding that claim. + pub continues: Option<(Path, Instance)>, } /// A shown claim as the origin serves it through the selected route. @@ -170,16 +175,24 @@ impl Node { .expect("a node always declares an identity") } - /// This node's own claim at `path` changed: `None` withdraws it. - pub fn own(&self, path: Path, claim: Option) { + /// This node's own claim at `path` changed: `None` withdraws it. A claim taking over + /// the own claim at `predecessor` continues the instance that one serves now. + pub fn own(&self, path: Path, claim: Option, predecessor: Option<&Path>) { let mut state = self.lock(); let changed = match claim { - Some(claim) => { - let current = state - .router - .declared() - .and_then(|id| state.router.view(id)?.items.get(&path).cloned()); - current.as_ref() != Some(&claim) && state.router.announce(path, claim).is_ok() + Some(mut claim) => { + let view = state.router.declared().and_then(|id| state.router.view(id)); + let current = view.and_then(|view| view.items.get(&path)); + claim.continues = match predecessor { + Some(predecessor) => (view.and_then(|view| view.items.get(predecessor))) + .map(|taken| (predecessor.clone(), taken.instance.clone())), + // A re-price or a newer epoch of the same instance continues what it did. + None => current + .filter(|current| current.instance == claim.instance) + .and_then(|current| current.continues.clone()), + }; + let changed = current != Some(&claim); + changed && state.router.announce(path, claim).is_ok() } None => state.router.unannounce(path).is_ok(), }; @@ -470,6 +483,7 @@ mod tests { instance: instance.clone(), epoch: 0, cost, + continues: None, } } @@ -504,7 +518,7 @@ mod tests { let (origin, relay) = (Node::new(1, 1), Node::new(2, 2)); let answer = origin.answer(Link::default(), Path::empty()).unwrap(); let receive = relay.receive(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 5))); + origin.own(p("live/x"), Some(claim(&instance, 5)), None); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -520,7 +534,7 @@ mod tests { }] ); - origin.own(p("live/x"), None); + origin.own(p("live/x"), None, None); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -548,7 +562,7 @@ mod tests { let far_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let near = relay.receive(Link::default(), Path::empty()).unwrap(); let near_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0))); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); pump(&far_answer, &far); pump(&near_answer, &near); let metric = |outputs: Vec| match &served(outputs)[..] { @@ -581,7 +595,7 @@ mod tests { let first_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let second = relay.receive(Link::default(), Path::empty()).unwrap(); let second_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0))); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); pump(&first_answer, &first); pump(&second_answer, &second); outputs(&first); diff --git a/rs/moq-net/tests/integration/announce_takeover.rs b/rs/moq-net/tests/integration/announce_takeover.rs index d0a55fa3a49..90cdb661d58 100644 --- a/rs/moq-net/tests/integration/announce_takeover.rs +++ b/rs/moq-net/tests/integration/announce_takeover.rs @@ -112,9 +112,7 @@ fn lite07(version: Version) -> bool { } /// `W -> R -> E`: the edge reads through `W`'s claim on its first subscription, and each -/// hop announces the answer. Lite-07 routes the answer as another of `W`'s claims, a -/// restart at the claim's own path and a start beneath it, and never sends -/// ANNOUNCE_TAKEOVER, which quest/m2/announce-takeover.md removes. +/// hop announces the answer as a takeover of the claim on lite-07, a start before it. async fn chain(version: &str, claim: &str) { let version: Version = version.parse().unwrap(); let worker = Worker::new(1, claim, "w"); @@ -139,14 +137,18 @@ async fn chain(version: &str, claim: &str) { assert_eq!(watch(&edge, "pool/p").await, "w", "{version}"); assert_eq!(worker.outputs.borrow().len(), 1, "{version}"); - let answered = match (lite07(version), claim == "pool/p") { + let answered = if !lite07(version) { // An ordinary announcement, or one replacing the claim at its own path. - (false, true) => vec![ - ("end", "pool/p".to_string(), None), - ("start", "pool/p".to_string(), None), - ], - (true, true) => vec![("restart", "pool/p".to_string(), None)], - (_, false) => vec![("start", "pool/p".to_string(), None)], + if claim == "pool/p" { + vec![ + ("end", "pool/p".to_string(), None), + ("start", "pool/p".to_string(), None), + ] + } else { + vec![("start", "pool/p".to_string(), None)] + } + } else { + vec![("takeover", "pool/p".to_string(), Some(claim.to_string()))] }; assert_eq!(drain(&mut at_relay).await, answered, "{version}: at the relay"); assert_eq!(drain(&mut at_edge).await, answered, "{version}: at the edge"); @@ -252,7 +254,10 @@ async fn an_older_downstream_gets_a_start() { drain(&mut at_edge).await; assert_eq!(watch(&relay, "pool/p").await, "w"); - assert_eq!(drain(&mut at_relay).await, [("start", "pool/p".to_string(), None)]); + assert_eq!( + drain(&mut at_relay).await, + [("takeover", "pool/p".to_string(), Some("pool".to_string()))] + ); assert_eq!(drain(&mut at_edge).await, [("start", "pool/p".to_string(), None)]); assert_eq!(watch(&edge, "pool/p").await, "w"); assert_eq!(worker.outputs.borrow().len(), 1, "the edge asked the worker again"); From b9f3078ddd66b5c09398b9107cf3dcb0d9970865 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 08:55:29 -0800 Subject: [PATCH 20/61] test: skip native JS lite-07 lanes until js/net speaks the route layer js/net still speaks lite-07 from before the route layer, so the in-tree lanes running it on lite-07-wip fail against this checkout's relay. Native JS dials its defaults in the interop matrix, and every lane that would run it on lite-07 records a skip naming quest/m2/cluster-routing/routes/js.md, which restores them. The same goes for lite_tracks_interop and the lite-07 legs of the max-age and relay-fill JS suites. The routes epic records the maintainer's 2026-11-06 picks: ANNOUNCE_TAKEOVER ported onto Route IDs, and a route sent on a stream only while it meets the stream's interest. session.md keeps the scoping work it implements. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/README.md | 15 ++++-- quest/m2/cluster-routing/routes/js.md | 12 +++++ quest/m2/cluster-routing/routes/session.md | 23 +++------ rs/moq-net/src/test_interop.rs | 2 +- rs/moq-tokio/tests/integration/max_age.rs | 4 +- rs/moq-tokio/tests/integration/relay_fill.rs | 4 +- test/interop/README.md | 6 +++ test/interop/interop.sh | 52 ++++++++++++++++++-- test/interop/socket.sh | 5 ++ test/justfile | 2 +- 10 files changed, 95 insertions(+), 30 deletions(-) diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index f11c48a24d6..76724476ea5 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -20,13 +20,14 @@ claim whose withdrawal is already known. 5. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Floor regions have no order, and a repeated region is a violation.** A sorted order would need Rust's `Path` ordering reproduced exactly in JS. Decoding into a map rejects duplicates at the same cost. Superseded the same day by item 10, which removes regions. 6. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The Hop setup parameter and ANNOUNCE_OK's Hop ID are dropped from lite-07.** Their only roles were path-vector exclusion and the implicit trailing hop. ANNOUNCE_OK reserves no body for a route-key proof: [moq-dev/moq#7937](https://github.com/moq-dev/moq/pull/7937), accepted the same day, proves the route key once per session in SETUP. The live hop-based codec still switches in session integration, as choice 1 requires. 7. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **An echo naming a Route ID that was never ended, or was already echoed, is a violation at the publisher.** The publisher can check this from its live and unacknowledged ids alone, with no growing state. -8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [quest/m2/announce-takeover.md](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) deletes the message, so this keeps the change minimal. +8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [quest/m2/announce-takeover.md](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) deletes the message, so this keeps the change minimal. Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether session integration ports the message or lets lite-07 answer a claim as a restart or start until announce-takeover lands: ✅ **port it**, so a job answering a claim stays a takeover across relays. Rejected: accepting the restart and start. 9. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Unknown announce types stay skipped by length on lite-06+**, as before. An unknown reply type is refused. 10. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_FLOOR is a marker, not a region list:** `{Route ID, Revision}`. It closes a batch and certifies the stream's claims for that route across the whole requested prefix as the origin's at that Revision. A publisher that cannot vouch for the whole requested prefix at one Revision still closes the batch, with a floor that certifies nothing, encoded per item 13. Regions, the replacing list, and the 65,535-byte overflow go away. Accepted cost: a relay with partial knowledge of an origin certifies nothing for that route, so a receiver lets its stale claims go only on ROUTE_END. Rejected: one region per message; a multipart atomic list. 11. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Clamped claims collapse to the best one.** When several of a route's claims cover a request's prefix (`live` at cost 10 and `live/a` at cost 1, request `live/a/x`), the publisher presents only the most specific, and when it is withdrawn moves the presented claim to the next covering one with ANNOUNCE_UPDATE or ANNOUNCE_RESTART instead of ending it. No wire change; one live claim per prefix per route holds, and a receiver merges claims at the prefix each stream presents them. Rejected: carrying each claim's ancestor depth. 12. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **A floor certifies only what its own request shows.** A stream whose request did not set `Hidden` certifies nothing about hidden claims. No new wire field. Rejected: isolating differently filtered views. 13. ✅ Decided 2026-11-05 (maintainer): **ANNOUNCE_FLOOR encodes `Revision + 1`, with 0 meaning the floor certifies nothing,** the same offset as `Max Age`. Rust decodes it to `Option`. It replaces a Revision of 2^64-1 as the marker: a huge sentinel is a bad pattern and costs 9 bytes as a lite-07 varint. Rejected: a separate message; a boolean field. 14. ✅ Decided 2026-11-05 (maintainer: "use 0 as a sentinel, or make a new message type"; the agent picked the message): **A held route is ROUTE_HOLD (0xB), `{Route ID}`,** replacing a ROUTE_UPDATE with a Metric of 2^64-1. A metric of 0 is real (the origin's own route), so 0 cannot mark held, and offsetting every metric by one would still leave a held update's seqno and down-only bit meaningless and refused. The message carries only what a hold means, item 4's rule goes away, and a metric is now only bounded by overflow, so the wire refuses no metric. The 2^64-1 Seqno and Revision bounds stay: they are bounds, not markers, leaving room for a ROUTE_REQUEST's successor and for the floor's offset. +15. ✅ Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether to amend the handoff contract's "a sender that certifies anything for a route answers every stream" or keep it: ✅ **send a route on a stream only while it meets the stream's interest**, with a floor certifying the last withdrawal before the route ends. Withdrawals still END, and what a node keeps per stream it answers is bounded by the origins that stream reads, at a ROUTE_END round trip when an origin leaves an interest. Rejected: keep answering every stream and bound only by per-peer caps, which is simpler but carries the whole node table on every stream. [Route session integration](/quest/m2/cluster-routing/routes/session.md) implements it. The route layer's core landed in @@ -330,10 +331,14 @@ loss never reports an END: delivered (`a_lost_scope_keeps_newer_knowledge_where_still_certified`). Otherwise a wildcard above both was last learned through the lost floor and is unknown at the kept floors' revision, so the view is rebuilt from the - neighbours' last views, a collection of that region. A sender that - certifies anything for a route answers every stream, even one whose - interest none of its regions meets, so its peer learns it certifies nothing - there rather than waiting (`a_disjoint_interest_is_answered_with_nothing`). + neighbours' last views, a collection of that region. A sender sends a + route on a stream only while the route meets the stream's interest (Plan + item 15): it starts the route before its first claim in or covering the + interest, and when the last such claim goes it sends a floor certifying the + withdrawal, then ends the route. A peer therefore holds no route, and + waits on no certifier, for an origin its interest does not meet. The router + still answers every stream until session integration scopes it + (`a_disjoint_interest_is_answered_with_nothing`). A claim is available only through a certifier that is not held; another scope's usable route cannot serve it. A certifier that has not answered yet counts as serving, as for a new next hop whose announcements trail its diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index 0426fdc7758..eb052a55830 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -18,6 +18,18 @@ Decided 2026-10-27 (maintainer), from the cheapest one; fix it with the route-selection rework here. - Feed the availability API from [Announcement availability](/quest/m2/cluster-routing/routes/availability.md). - Rust/JS interop covers the route messages, the floor marker, and the echo. +- Restore the native JS lite-07 lanes. Decided 2026-11-06 (maintainer, + /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), + asked how to handle the in-tree lanes running JS on lite-07-wip once the + Rust side moved to the route wire: ✅ **skip them until this quest lands**, + so the nightly report stays meaningful. Rejected: leave them red; hold + #8023 for this quest. Every skip names this quest (`grep -rn routes/js.md + test rs`): `JS_LITE07_SKIP` in `test/interop/interop.sh` and `socket.sh` + (native JS grant printing, claims, track replacement, sockets, the lite-07 + compress and drop legs, the compressed catalog's lite-07 reader, and + `--versions moq-lite-07-wip` with JS clients), `lite_tracks_interop` in + `test/justfile`, and the lite-07 legs of `max_age_relay_javascript` and + `relay_fill_javascript`. ## Required diff --git a/quest/m2/cluster-routing/routes/session.md b/quest/m2/cluster-routing/routes/session.md index cd06f1bd925..9afc6653e1c 100644 --- a/quest/m2/cluster-routing/routes/session.md +++ b/quest/m2/cluster-routing/routes/session.md @@ -12,11 +12,13 @@ node table, and the public API drops what the route layer replaced. ## Plan - Scope route state to interested origins, not the CDN's whole node table. - Today the router advertises every route to every watching stream, because - the epic's handoff contract (`routes/README.md`, "a sender that certifies - anything for a route answers every stream") has a sender answer even a - stream whose interest none of the route's regions meets. Scoping needs - that sentence amended (see Waiting). + Today the router advertises every route to every watching stream. The + epic's handoff contract now has a sender send a route on a stream only + while it meets the stream's interest, with a floor certifying the last + withdrawal before the route ends (`routes/README.md` Plan item 15, decided + 2026-11-06): start the route before its first claim in or covering the + interest, and end it after the floor that certifies its last such claim's + withdrawal. Update `a_disjoint_interest_is_answered_with_nothing` to match. What a peer makes the router hold is already capped: each route and claim it sends takes one of the session's announce slots (`a_lite07_claim_past_the_cap_closes_the_session`). What this bounds is @@ -32,17 +34,6 @@ node table, and the public API drops what the route layer replaced. ## Waiting -- Maintainer: amend the handoff contract's "a sender that certifies anything - for a route answers every stream" (`routes/README.md`), or keep it. It is a - decided contract, so an agent does not change it. Options: **send a route - only while it meets the stream's interest (recommended)**: a route starts on - a stream before its first claim in or covering the interest, and when its - last such claim goes it sends a floor certifying the withdrawal, then ends - the route, so withdrawals still END and per-stream state is bounded by the - origins a peer reads, at a ROUTE_END round trip when an origin leaves an - interest; or **keep answering every stream** and bound only by per-peer - caps, simpler but carrying the whole node table on every stream. Done when - the README says which. - Maintainer: decide when `Hops`, `RouteInfo::hops`, and the saturating `Cost` leave the public API. Every older wire still fills them, so an agent cannot drop them without picking the rollout's end. Options: **when lite-06 diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index cee73c582a5..80446166634 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -554,7 +554,7 @@ fn lite_auth_unsupported_interop() { /// with a listing Rust decodes, and JS decodes Rust's listing events and re-encodes them byte for /// byte. #[test] -#[ignore = "requires Bun; run by just test interop"] +#[ignore = "JS encodes TRACKS without a Node ID until quest/m2/cluster-routing/routes/js.md, which restores it to just test interop"] fn lite_tracks_interop() { use crate::coding::Decode; diff --git a/rs/moq-tokio/tests/integration/max_age.rs b/rs/moq-tokio/tests/integration/max_age.rs index fda969668c1..c4866d01c86 100644 --- a/rs/moq-tokio/tests/integration/max_age.rs +++ b/rs/moq-tokio/tests/integration/max_age.rs @@ -141,7 +141,9 @@ async fn max_age_relay_rust() { #[tokio::test] #[ignore = "requires Bun; run by just test max-age in the interop workflow"] async fn max_age_relay_javascript() { - for version in ["moq-lite-07-wip", "moq-transport-18", "moq-transport-22"] { + // moq-lite-07-wip returns once js/net speaks its route layer + // (quest/m2/cluster-routing/routes/js.md). + for version in ["moq-transport-18", "moq-transport-22"] { for publish in [false, true] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap(), Some(publish))) .await diff --git a/rs/moq-tokio/tests/integration/relay_fill.rs b/rs/moq-tokio/tests/integration/relay_fill.rs index 693e4cbb61b..01c35216814 100644 --- a/rs/moq-tokio/tests/integration/relay_fill.rs +++ b/rs/moq-tokio/tests/integration/relay_fill.rs @@ -275,7 +275,9 @@ async fn relay(version: Version) -> anyhow::Result<()> { #[tokio::test] #[ignore = "requires Bun; run by just test relay-fill in the interop workflow"] async fn relay_fill_javascript() { - for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { + // moq-lite-07-wip returns once js/net speaks its route layer + // (quest/m2/cluster-routing/routes/js.md). + for version in ["moq-lite-05", "moq-lite-06"] { kio::clock::timeout(TIMEOUT, relay(version.parse().unwrap())) .await .unwrap() diff --git a/test/interop/README.md b/test/interop/README.md index 1939304a3b6..40f68e764c6 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -20,6 +20,12 @@ and Bun over WebSocket and WebTransport. Fresh readers request or follow the claim's own path and child paths. A reader-first case checks that an absent request ends while its follower waits for the publisher to arrive. +Until [`js/net` speaks lite-07's route layer](../../quest/m2/cluster-routing/routes/js.md), +native JS dials its defaults in the matrix, leaving its grant unchecked, and every +lane that would run it on lite-07 records a skip naming that quest: claims, track +replacement, sockets, and the lite-07 compress and drop legs. The compressed catalog +runs without its lite-07 reader, and `--versions moq-lite-07-wip` refuses JS clients. + Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and 660 Hz right, so a channel swap fails too. The Rust CLI imports it from ffmpeg in diff --git a/test/interop/interop.sh b/test/interop/interop.sh index a2e22c223c5..60039316ce6 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -276,6 +276,26 @@ needs_js() { needs js || needs js-native || needs js-native-node || needs js-native-bun } +# js/net speaks the lite-07 wire from before the route layer until +# quest/m2/cluster-routing/routes/js.md ports it and removes this: native JS dials its +# defaults in the matrix, and a lane running it on moq-lite-07-wip is skipped. +JS_LITE07_SKIP="JS speaks pre-route lite-07 until quest/m2/cluster-routing/routes/js.md" + +# js_lite07 : true if the lane runs native JS on moq-lite-07-wip. +js_lite07() { + local version="$1" client + [[ "$version" == moq-lite-07-wip ]] || return 1 + for client in "${@:2}"; do [[ "$client" != js-native-* ]] || return 0; done + return 1 +} + +for version in "${VERSION_LIST[@]}"; do + if [[ "$version" == moq-lite-07-wip ]] && needs_js; then + echo "error: $JS_LITE07_SKIP; pin moq-lite-07-wip without JS clients" >&2 + exit 2 + fi +done + # Released-wire compatibility (compat.sh) pins one version and may run released # binaries that predate AUTH, so it runs anonymous: it compares the wire, and the # default matrix covers auth. @@ -285,11 +305,12 @@ COMPAT=0 # The clients that print the grant they received as an `auth granted` line. The # Python, Go, and C++ clients read it through moq-ffi; moq-c's frozen ABI and # GStreamer expose none, and the browser's shared connection keeps its session private. -# Grants are checked on $AUTH_VERSION, so only in the `auto` pass. +# Native JS prints one too, but dials its defaults until it speaks lite-07's route +# layer (see JS_LITE07_SKIP). Grants are checked on $AUTH_VERSION, so only in the `auto` pass. prints_grant() { [[ "$COMPAT" -eq 0 && "$VERSION" == auto ]] || return 1 case "$1" in - rust | python | go | cpp | js-native-node | js-native-bun) return 0 ;; + rust | python | go | cpp) return 0 ;; *) return 1 ;; esac } @@ -348,8 +369,9 @@ DROP_VERSIONS=(moq-lite-05 moq-lite-06 moq-lite-07-wip) # The lite drafts the compressed-track lanes cross: lite-07 carries the flag, lite-06 gets inflated bytes. COMPRESS_VERSIONS=(moq-lite-07-wip moq-lite-06) # The compressed catalog's concurrent readers: lite-07 carries the flag, and lite-06 and a -# moq-transport reader that does not offer FLATE share the relay's one inflated copy. -CATALOG_VERSIONS=(moq-lite-07-wip moq-lite-06 moq-transport-22) +# moq-transport reader that does not offer FLATE share the relay's one inflated copy. The +# lite-07 reader returns with quest/m2/cluster-routing/routes/js.md (see JS_LITE07_SKIP). +CATALOG_VERSIONS=(moq-lite-06 moq-transport-22) # Every publisher in this checkout sends a tone per audio channel, and every subscriber decodes # AUDIO_MS of it for tone.py (the browser: its page) to check against the publisher's list. This @@ -1904,6 +1926,11 @@ run_drop_pair() { local name="drop-${version}-${pub}-${sub}" local fifo="$LOGS/tail-ack" broadcast="$name-$$-$RANDOM" local cell=(case=drop publisher="$pub" subscriber="$sub" version="$version" log="$LOGS/$name-sub.log") + if js_lite07 "$version" "$pub" "$sub"; then + echo " SKIP drop $version $pub -> $sub" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL drop $version $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -1975,6 +2002,11 @@ run_tail_pair() { fi local cell=(case="$kind" publisher="$pub" subscriber="$sub" log="$LOGS/$name-sub.log") [[ -z "${4:-}" ]] || cell+=(version="$version") + if js_lite07 "$version" "$pub" "$sub"; then + echo " SKIP $label $pub -> $sub" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$pub" || is_broken "$sub"; then echo " FAIL $label $pub -> $sub (client unavailable)" record fail "${cell[@]}" detail="client unavailable" @@ -2320,6 +2352,11 @@ run_claims_case() { local prefix="claims-$1-$2-$$-$RANDOM" command=(bun) local cell=(case="claims $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") [[ "$runtime" != js-native-node ]] || command=(node --import tsx) + if js_lite07 moq-lite-07-wip "$runtime"; then + echo " SKIP claims $transport $runtime" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2377,6 +2414,11 @@ run_replacement_case() { local runtime="$1" transport="$2" name="replace-$1-$2" started=$SECONDS local prefix="replace-$1-$2-$$-$RANDOM" local cell=(case="track replacement $transport" publisher="$runtime" subscriber="$runtime" version=moq-lite-07-wip log="$LOGS/$name.log") + if js_lite07 moq-lite-07-wip "$runtime"; then + echo " SKIP track replacement $transport $runtime" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$runtime"; then record fail "${cell[@]}" detail="client unavailable" overall=1 @@ -2461,7 +2503,7 @@ run_catalog_fanout() { got=$(relay_log_since "$from" | sed -n 's/.*conn{id=[0-9]* .*negotiated version=\([^ ]*\).*/\1/p' | sort | tr '\n' ' ') if [[ -z "$why" ]] && ! grep -qx 'catalog acknowledged' "$LOGS/$name-pub.log"; then why="publisher did not acknowledge" - elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-lite-07-wip moq-transport-22 " ]]; then + elif [[ -z "$why" && "$got" != "moq-lite-06 moq-lite-07-wip moq-transport-22 " ]]; then why="relay connections negotiated ${got:-nothing}" elif [[ -z "$why" && "$((after - before))" != "$bytes" ]]; then why="relay inflated $((after - before)) bytes, not the catalog's ${bytes:-?} once" diff --git a/test/interop/socket.sh b/test/interop/socket.sh index 42fd013e36c..f28f3ab351b 100644 --- a/test/interop/socket.sh +++ b/test/interop/socket.sh @@ -25,6 +25,11 @@ run_socket_pair() { local name="socket-$protocol-$dialer-$acceptor-$version" prefix="socket-$protocol-$dialer-$acceptor-$version-$$-$RANDOM" local events="$LOGS/socket-events" ack="$LOGS/socket-ack" local cell=(case="socket $protocol" publisher="$dialer" subscriber="$acceptor" version="$version" log="$LOGS/$name-dial.log") + if js_lite07 "$version" "$dialer" "$acceptor"; then + echo " SKIP socket $protocol $dialer -> $acceptor $version" + record skip "${cell[@]}" detail="$JS_LITE07_SKIP" + return + fi if is_broken "$dialer" || is_broken "$acceptor"; then record fail "${cell[@]}" detail="client unavailable" overall=1 diff --git a/test/justfile b/test/justfile index bd9db4d79e4..b2d851bcc7e 100644 --- a/test/justfile +++ b/test/justfile @@ -64,6 +64,7 @@ harness: # Finite track tails run too; `--tail` selects just the CLI FETCH close and Rust/Node/Bun tail checks. # `--versions ietf` reruns them with every client pinned to each even moq-transport draft. # Both runs first check the Rust/JS wire and session interop suites, in both directions. +# `lite_tracks_interop` rejoins them with quest/m2/cluster-routing/routes/js.md. [positional-arguments] interop *args: cd .. && bun install --frozen-lockfile @@ -75,7 +76,6 @@ interop *args: ietf_range_updates_interop \ padding_interop \ sign_interop \ - tracks_interop \ varint_interop \ --run-ignored all ./interop/interop.sh "$@" From e896ae79bca05e8f3503da845edc9b802d69338d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 09:52:44 -0800 Subject: [PATCH 21/61] fix(net): lite-07 receive admits each claim by the limit, bounds pending, serves per claim - A lite-07 stream hands the node only the peer's claims the session's limit admits, not every claim within its regions. A limit narrower than its regions certifies nothing, so a floor never certifies a claim it left out absent. A permit change that admits a different set files the stream again, as a scope change does. - Claim changes between floors are kept as their net change to the committed claims, so what a stream holds is bounded by its claims, not by the messages it read. - The node serves a claim only through the streams that can carry it: within what this node asked the neighbor for and within its scope. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 19 +++ rs/moq-net/src/lite/subscriber/routed.rs | 192 +++++++++++++++++++++-- rs/moq-net/src/route/mod.rs | 18 ++- rs/moq-net/src/route/node.rs | 36 ++++- 4 files changed, 247 insertions(+), 18 deletions(-) diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 8419aa4dd64..f2fbca84b73 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -527,6 +527,15 @@ pub(crate) struct Permit { } impl Permit { + /// A permit limiting the peer to `limit`, with no grant of its own. + #[cfg(test)] + pub(crate) fn limited(limit: Patterns) -> Self { + Self { + granted: None, + limit: Some(limit), + } + } + /// Whether both allow `path`. pub(crate) fn matches(&self, path: &str) -> bool { self.granted(path) && self.within_limit(path) @@ -554,6 +563,16 @@ impl Permit { covers(&self.granted) && covers(&self.limit) } + /// Whether the limit alone allows every path at or beneath `prefix`, judged + /// pattern by pattern, so a `false` may be spurious but a `true` never is. + #[cfg(feature = "async")] + pub(crate) fn within_limit_subtree(&self, prefix: &str) -> bool { + let Ok(subtree) = moq_pattern::Pattern::subtree(prefix) else { + return false; + }; + self.limit.as_ref().is_none_or(|limit| limit.contains(&subtree)) + } + /// The prefixes this confines the peer to, or `None` when it confines nothing: the /// heads of the scope's patterns, which grants this side issued. What the peer /// granted this side limits only what this side does, so a client never scopes the diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 8a006e25058..b804c10cd84 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -24,13 +24,25 @@ struct WireRoute { reach: route::Reach, /// The Announce IDs of its live claims. claims: BTreeSet, - /// Claim changes since its last ANNOUNCE_FLOOR. - pending: Vec<(Path, Option)>, + /// How its claims differ from `committed`, by path, so what the stream keeps + /// between floors is bounded by its claims rather than the messages it read. + pending: BTreeMap>, /// Its claims and floor as of that ANNOUNCE_FLOOR, replayed when the stream /// files again. `None` until the first. committed: Option<(BTreeMap, Option)>, } +impl WireRoute { + /// The claim at `path` is now `claim`, pending the next floor. + fn change(&mut self, path: Path, claim: Option) { + let committed = self.committed.as_ref().and_then(|(claims, _)| claims.get(&path)); + match committed == claim.as_ref() { + true => self.pending.remove(&path), + false => self.pending.insert(path, claim), + }; + } +} + /// One live claim the peer sent, by its Announce ID. struct WireClaim { route: u64, @@ -54,6 +66,9 @@ pub(super) struct Routed { link: route::Link, /// The scope's regions it filed under, or `None` unscoped. regions: Option>, + /// The permit it filed under. The node holds only the peer's claims its limit + /// admits; the regions alone can be wider. + permit: crate::auth::Permit, /// The peer's SETUP, whose proven route key files a scoped peer's identities. peer_setup: crate::lite::PeerSetup, /// Files them instead while the peer proved none: this session's own. @@ -113,6 +128,7 @@ impl Routed { stream, link, regions, + permit: permit.clone(), peer_setup, session_key, prefix, @@ -153,7 +169,7 @@ impl Routed { node, reach, claims: BTreeSet::new(), - pending: Vec::new(), + pending: BTreeMap::new(), committed: None, }, ); @@ -173,6 +189,7 @@ impl Routed { self.replies.push(AnnounceReply::RouteEnd { id }); } Msg::Floor { id, revision } => { + let certifies = self.certifies(); let route = self.routes.get_mut(&id).ok_or(Error::ProtocolViolation)?; let changes = std::mem::take(&mut route.pending); let (committed, floor) = route.committed.get_or_insert_default(); @@ -183,6 +200,11 @@ impl Routed { }; } *floor = revision; + // The node never held a claim the limit leaves out, so it neither starts nor ends. + let changes = (changes.into_iter()) + .filter(|(path, _)| admits(&self.root, &self.permit, path)) + .collect(); + let revision = revision.filter(|_| certifies); self.stream.batch(route.node, changes, revision); } Msg::Active { @@ -275,7 +297,7 @@ impl Routed { } self.held.push(self.slots.acquire()?); wire.claims.insert(id); - wire.pending.push((path.clone(), Some(claim.clone()))); + wire.change(path.clone(), Some(claim.clone())); self.claims.insert(id, WireClaim { route, path, claim }); Ok(()) } @@ -286,14 +308,14 @@ impl Routed { self.held.pop(); let route = self.routes.get_mut(&claim.route).ok_or(Error::ProtocolViolation)?; route.claims.remove(&id); - route.pending.push((claim.path, None)); + route.change(claim.path, None); Ok(()) } fn pending(&mut self, id: u64, change: (Path, Option)) -> Result<(), Error> { let route = self.claims.get(&id).ok_or(Error::ProtocolViolation)?.route; let route = self.routes.get_mut(&route).ok_or(Error::ProtocolViolation)?; - route.pending.push(change); + route.change(change.0, change.1); Ok(()) } @@ -310,12 +332,18 @@ impl Routed { self.outputs(subscriber); } - /// The session's permit changed: a scope the stream no longer matches files it - /// again, the peer's routes and claims replayed under a new key before the old - /// one lets go. + /// The session's permit changed: a scope the stream no longer matches, or a + /// limit admitting a different set of the peer's claims, files it again, the + /// peer's routes and the claims the limit admits replayed before the old stream + /// lets go. The peer still holds each claim either way. pub(super) fn limit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { let regions = regions(&self.root, permit); - if regions == self.regions { + let admitted = (self.routes.values()) + .filter_map(|route| route.committed.as_ref()) + .flat_map(|(claims, _)| claims.keys()) + .any(|path| admits(&self.root, &self.permit, path) != admits(&self.root, permit, path)); + self.permit = permit.clone(); + if regions == self.regions && !admitted { return Ok(()); } let key = self.peer_setup.route_key().unwrap_or(self.session_key); @@ -332,11 +360,14 @@ impl Routed { let route = &self.routes[id]; stream.route(route.node, Some(route.reach)); } + let certifies = self.certifies(); for id in &ids { let route = &self.routes[id]; if let Some((claims, floor)) = &route.committed { - let changes = claims.iter().map(|(path, claim)| (path.clone(), Some(claim.clone()))); - stream.batch(route.node, changes.collect(), *floor); + let changes = (claims.iter()) + .filter(|(path, _)| admits(&self.root, &self.permit, path)) + .map(|(path, claim)| (path.clone(), Some(claim.clone()))); + stream.batch(route.node, changes.collect(), floor.filter(|_| certifies)); } } if self.drained { @@ -352,6 +383,26 @@ impl Routed { Ok(()) } + /// Whether the peer's floors may certify what reaches the node: the limit admits + /// all of the requested prefix within the scope's regions. A narrower limit leaves + /// claims out that a floor would certify absent, so the claims it admits stay only + /// while the peer carries them. + fn certifies(&self) -> bool { + let Some(regions) = &self.regions else { + return true; + }; + (regions.iter()) + .filter_map(|region| match () { + _ if self.prefix.has_prefix(region) => Some(&self.prefix), + _ if region.has_prefix(&self.prefix) => Some(region), + _ => None, + }) + .all(|covered| { + (covered.strip_prefix(&self.root)) + .is_some_and(|relative| self.permit.within_limit_subtree(relative.as_str())) + }) + } + /// Act on everything the node queued for the stream: serve the claims it hands /// over, and queue the replies it asks for. pub(super) fn outputs(&mut self, subscriber: &Subscriber) { @@ -492,6 +543,12 @@ fn hops(id: route::Id) -> crate::Hops { hops } +/// Whether `permit`'s limit admits the peer's claim at the absolute `path`. +fn admits(root: &Path, permit: &crate::auth::Permit, path: &Path) -> bool { + path.strip_prefix(root) + .is_some_and(|relative| permit.within_limit(relative.as_str())) +} + /// The absolute regions a permit confines the peer's claims to. fn regions(root: &Path, permit: &crate::auth::Permit) -> Option> { let regions = permit.regions()?; @@ -511,7 +568,116 @@ fn scoped(link: &route::Link, regions: &Option>, key: [u8; 32]) -> rou #[cfg(test)] mod tests { use crate::lite::{AnnounceBroadcast as Msg, AnnounceDecoder, PathRef, test_transport::SinkSession}; - use crate::{Instance, Path, origin::Cost, route::Advert}; + use crate::{Instance, Path, auth::Permit, origin::Cost, route::Advert}; + + const ADVERT: Advert = Advert { + seqno: 0, + metric: 0, + down_only: false, + }; + + fn start(path: &'static str) -> Msg { + Msg::Active { + suffix: PathRef::literal(Path::from_static(path)), + route: 0, + instance: Some(Instance::random()), + epoch: 0, + hops: Default::default(), + cost: Cost::new(0), + } + } + + fn limited(pattern: &str) -> Permit { + Permit::limited([pattern.parse::().unwrap()].into_iter().collect()) + } + + /// The claims the stream serves into the origin, by path. + fn served(routed: &super::Routed) -> Vec { + let mut served: Vec = routed.served.keys().map(|(_, path)| path.to_string()).collect(); + served.sort(); + served + } + + /// A limit narrower than its regions still leaves out the claims it does not + /// admit, and a change that keeps the regions brings them in and out again. + #[kio::test] + async fn the_limit_admits_each_claim() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let public = limited("live/*/public/**"); + let everything = limited("live/**"); + assert_eq!(public.regions(), everything.regions(), "the limits share their regions"); + + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &public).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed.outputs(&subscriber); + assert_eq!(served(&routed), ["live/a/public/x"]); + + routed.limit(&subscriber, &everything).unwrap(); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + + routed.limit(&subscriber, &public).unwrap(); + assert_eq!(served(&routed), ["live/a/public/x"]); + } + + /// What the stream keeps between floors is the net change to its committed claims, + /// however many messages the peer sends without one. + #[kio::test] + async fn pending_changes_net_out() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("room/host"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + let pending = |routed: &super::Routed| routed.routes[&0].pending.len(); + assert_eq!(pending(&routed), 0); + + for cost in 1..1000 { + let update = Msg::Update { + id: 0, + hops: Default::default(), + cost: Cost::new(cost), + }; + routed.receive(&prefix, update, &mut decoder).unwrap(); + } + assert_eq!(pending(&routed), 1, "updates to one claim"); + + let back = Msg::Update { + id: 0, + hops: Default::default(), + cost: Cost::new(0), + }; + routed.receive(&prefix, back, &mut decoder).unwrap(); + assert_eq!(pending(&routed), 0, "an update back to the committed claim"); + + for _ in 0..100 { + routed.receive(&prefix, start("room/guest"), &mut decoder).unwrap(); + let id = decoder.last(); + routed.receive(&prefix, Msg::EndedId { id }, &mut decoder).unwrap(); + } + assert_eq!(pending(&routed), 0, "claims started and ended between floors"); + } /// A takeover in place keeps the sources the predecessor minted open: they serve the /// instance it continues. A restart in place closes them. diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index c9b3bb12fcc..89efac0beb1 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -271,6 +271,16 @@ struct Peer { asked: Option, } +impl Peer { + /// Whether a request for a claim at `claim` may go through this neighbor: the + /// claim lies within what this node asked it for and within its scope. A route + /// reaches an origin node, not each of its claims. + fn carries(&self, claim: &Path) -> bool { + self.asked.as_ref().is_none_or(|asked| within(claim, asked)) + && (self.link.scope.as_ref()).is_none_or(|scope| scope.regions.iter().any(|region| within(claim, region))) + } +} + /// An identity this node serves itself. struct Origin { seqno: Seqno, @@ -846,14 +856,16 @@ impl Router { best.map(|(_, selected)| selected) } - /// Every usable feasible route to `id` through a neighbor, with this node's - /// metric through each. Any of them forwards without a loop. - pub fn routes(&self, id: Id) -> Vec<(N, Metric)> { + /// Every usable feasible route to `id` through a neighbor that can carry + /// `claim`, with this node's metric through each. Any of them forwards without a + /// loop. + pub fn routes(&self, id: Id, claim: &Path) -> Vec<(N, Metric)> { if self.origins.contains_key(&id) { return Vec::new(); } (self.rx.get(&id).into_iter().flatten()) .filter(|(_, entry)| !entry.held && self.feasible(id, entry.advert)) + .filter(|(next, _)| self.neighbors[*next].carries(claim)) .filter_map(|(&next, entry)| Some((next, add(entry.advert.metric, self.neighbors[&next].link.cost)?))) .collect() } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 3509f01cbb8..e45f4d8d411 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -374,9 +374,9 @@ impl State { } /// Serve each shown claim of `id` through every stream with a usable feasible - /// route to it, keeping it where it is, unavailable, while there is none. + /// route to it that can carry the claim, keeping it where it is, unavailable, while + /// there is none. fn home(&mut self, id: Id) { - let routes = self.router.routes(id); let claims: Vec<(Path, Claim, bool)> = (self.router.shown().range((id, Path::empty())..)) .take_while(|((shown, _), _)| *shown == id) .map(|((_, claim), shown)| (claim.clone(), shown.item.clone(), shown.available)) @@ -387,6 +387,7 @@ impl State { .collect(); for (claim, item, available) in claims { gone.remove(&claim); + let routes = self.router.routes(id, &claim); let home = self.homes.entry((id, claim.clone())).or_default(); let wanted: Vec<(Key, Metric, bool)> = match routes.is_empty() { true => (home.keys.iter()) @@ -586,6 +587,37 @@ mod tests { assert!(served(outputs(&near)).is_empty(), "the remaining route serves on"); } + /// Two streams reach one origin node for different prefixes: each claim is served + /// only through the streams that asked for it, however cheap the other. + #[test] + fn a_claim_is_served_only_through_streams_that_carry_it() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let a = relay.receive(Link::default(), p("live/a")).unwrap(); + let a_answer = origin.answer(Link::default(), p("live/a")).unwrap(); + let costly = Link { + cost: 3, + ..Link::default() + }; + let b = relay.receive(costly, p("live/b")).unwrap(); + let b_answer = origin.answer(Link::default(), p("live/b")).unwrap(); + origin.own(p("live/a/x"), Some(claim(&instance, 0)), None); + origin.own(p("live/b/y"), Some(claim(&instance, 0)), None); + pump(&a_answer, &a); + pump(&b_answer, &b); + let claims = |outputs: Vec| -> Vec { + (served(outputs).into_iter()) + .filter_map(|output| match output { + Output::Serve { claim, .. } => Some(claim), + _ => None, + }) + .collect() + }; + assert_eq!(claims(outputs(&a)), [p("live/a/x")]); + assert_eq!(claims(outputs(&b)), [p("live/b/y")]); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); From 0e32a180cc4dab24d46a4bf1dbe2b6facfb708dd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:02:41 -0800 Subject: [PATCH 22/61] fix(net): coalesce a lite-07 stream's unread route outputs per identity A stream whose reader stalls kept every route change and batch the node queued. Its outbox now holds one entry per identity: an end first, then the last start and hold, and one batch with each claim's last change and the last floor, starts before ends as the router orders them. A claim the reader was never sent leaves nothing when it ends, so what a stalled stream holds is bounded by its claims. A route started and ended unread reaches the reader as an end alone, which it acks at once since the peer never held the route. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 16 +- rs/moq-net/src/route/node.rs | 230 ++++++++++++++++++++++-- 2 files changed, 226 insertions(+), 20 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 865c15b9b23..e3331f9da30 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -116,7 +116,9 @@ impl Answer { let answer = node .answer(self.link.clone(), self.prefix.clone()) .map_err(|_| Error::ProtocolViolation)?; - while let Some(output) = answer.try_output() { + // Held before its first outputs, so an end it acks at once reaches the node. + self.stream = Some(answer); + while let Some(output) = self.stream.as_ref().and_then(node::Stream::try_output) { self.output(stream, output)?; } // Every route the peer holds has a floor before SYNCED; one the node has @@ -132,7 +134,6 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; - self.stream = Some(answer); self.phase = Phase::Running; } @@ -253,8 +254,15 @@ impl Answer { } self.echoes.insert(sent.wire, id); } - // Held only follows a route the peer holds, and an end of one it does not is moot. - (Some(route::Reach::Held) | None, None) => {} + // The node coalesced the route's start away: the peer never held it, so nothing + // of it lingers, and the end is acked at once. + (None, None) => { + if let Some(answer) = &self.stream { + answer.echo(id); + } + } + // Held only follows a route the peer holds. + (Some(route::Reach::Held), None) => {} } Ok(()) } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e45f4d8d411..9367e4c956c 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -15,8 +15,9 @@ //! only once another serves it, so the origin always holds a route for it. use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap, VecDeque}, sync::{Arc, Mutex, MutexGuard}, + task::{Poll, ready}, }; use crate::{Instance, Path}; @@ -53,7 +54,8 @@ pub(crate) struct Served { pub available: bool, } -/// What a stream's session acts on, in order. +/// What a stream's session acts on, in order. One identity's route changes and +/// batches coalesce while the session leaves them unread. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Output { /// Send ROUTE_START, ROUTE_UPDATE, ROUTE_HOLD, or for `None` ROUTE_END. @@ -93,10 +95,134 @@ impl std::fmt::Debug for Node { } } +/// Each stream's outputs, by its key. +type Queues = HashMap>; + +/// What one stream has yet to read. Route changes and claim batches coalesce per +/// identity, and a claim the reader was never sent leaves nothing when it ends, so a +/// stream whose reader stalls holds one entry per identity, bounded by its claims, +/// however much the node changes meanwhile. +#[derive(Default)] +struct Outbox { + /// Identities with route changes or claims pending, in the order they first queued. + order: VecDeque, + pending: HashMap, + /// Outputs to read in order: the other kinds, and a pending entry once taken. + ready: VecDeque, + /// The claims the reader was sent for each identity. + delivered: HashMap>, +} + +/// One identity's route changes and claim batches since the reader last took it. +#[derive(Default)] +struct Pending { + /// A ROUTE_END goes first, retiring every claim sent before. + ended: bool, + /// The last finite reach since, which starts the route if the reader holds none. + finite: Option, + /// The route was held after that. + held: bool, + /// The batches since, merged: each claim's last change, and the last floor. + batch: Option<(BTreeMap>, Option)>, +} + +impl Outbox { + fn is_empty(&self) -> bool { + self.ready.is_empty() && self.order.is_empty() + } + + /// The pending entry for `id`, queued behind the others when new. + fn entry<'a>(order: &mut VecDeque, pending: &'a mut HashMap, id: Id) -> &'a mut Pending { + pending.entry(id).or_insert_with(|| { + order.push_back(id); + Pending::default() + }) + } + + fn push(&mut self, output: Output) { + match output { + Output::Route { id, reach } => { + let entry = Self::entry(&mut self.order, &mut self.pending, id); + match reach { + None => { + *entry = Pending { + ended: true, + ..Pending::default() + } + } + Some(Reach::Held) => entry.held = true, + Some(finite) => { + entry.finite = Some(finite); + entry.held = false; + } + } + } + Output::Batch { id, changes, floor } => { + let entry = Self::entry(&mut self.order, &mut self.pending, id); + let sent = self.delivered.get(&id); + let ended = entry.ended; + let (merged, last) = entry.batch.get_or_insert_default(); + for (path, claim) in changes { + // The reader holds nothing here once the entry reaches it. + let unsent = ended || sent.is_none_or(|sent| !sent.contains(&path)); + match claim { + None if unsent => merged.remove(&path), + claim => merged.insert(path, claim), + }; + } + *last = floor; + } + other => self.ready.push_back(other), + } + } + + fn pop(&mut self) -> Option { + if self.ready.is_empty() { + let id = self.order.pop_front()?; + let pending = self.pending.remove(&id).expect("every queued identity is pending"); + self.take(id, pending); + } + self.ready.pop_front() + } + + /// Queue one identity's pending entry to read, as the router would have sent it. + fn take(&mut self, id: Id, pending: Pending) { + if pending.ended { + self.delivered.remove(&id); + self.ready.push_back(Output::Route { id, reach: None }); + } + if let Some(reach) = pending.finite { + self.ready.push_back(Output::Route { id, reach: Some(reach) }); + } + if pending.held { + self.ready.push_back(Output::Route { + id, + reach: Some(Reach::Held), + }); + } + if let Some((merged, floor)) = pending.batch { + let sent = self.delivered.entry(id).or_default(); + // As the router orders a batch: what starts or changes, then what ends. + let (ends, starts): (Vec<_>, Vec<_>) = merged.into_iter().partition(|(_, claim)| claim.is_none()); + for (path, _) in &starts { + sent.insert(path.clone()); + } + for (path, _) in &ends { + sent.remove(path); + } + if sent.is_empty() { + self.delivered.remove(&id); + } + let changes = starts.into_iter().chain(ends).collect(); + self.ready.push_back(Output::Batch { id, changes, floor }); + } + } +} + struct State { router: Router, next_key: Key, - queues: HashMap>, + queues: Queues, /// Each shown claim of another identity, and the streams serving it. homes: BTreeMap<(Id, Path), Home>, } @@ -117,7 +243,7 @@ struct Slot { impl Home { /// Unserve every leaving stream once a wanted one serves the claim. - fn release(&mut self, queues: &HashMap>, id: Id, claim: &Path) { + fn release(&mut self, queues: &Queues, id: Id, claim: &Path) { if !self.keys.values().any(|slot| !slot.leaving && slot.confirmed) { return; } @@ -220,7 +346,7 @@ impl Node { let key = state.next_key; state.next_key += 1; state.router.connect(key, link)?; - let queue = kio::Queue::new(); + let queue = kio::Shared::default(); state.queues.insert(key, queue.clone()); if let Err(err) = open(&mut state.router, key) { state.router.disconnect(key); @@ -237,6 +363,11 @@ impl Node { }) } + #[cfg(test)] + pub(crate) fn retained(&self) -> super::Retained { + self.lock().router.retained() + } + #[cfg(test)] pub(crate) fn shown(&self) -> BTreeMap<(Id, Path), (Claim, bool)> { (self.lock().router.shown().iter()) @@ -249,7 +380,7 @@ impl Node { pub(crate) struct Stream { node: Node, key: Key, - queue: kio::Queue, + queue: kio::Shared, } impl std::fmt::Debug for Stream { @@ -260,17 +391,21 @@ impl std::fmt::Debug for Stream { impl Stream { /// The next output for this stream's session. - pub fn poll_output(&self, waiter: &kio::Waiter) -> std::task::Poll { - match self.queue.poll_pop(waiter) { - std::task::Poll::Ready(Ok(output)) => std::task::Poll::Ready(output), - // The queue closes only once the stream dropped. - _ => std::task::Poll::Pending, - } + pub fn poll_output(&self, waiter: &kio::Waiter) -> Poll { + let ready = |outbox: &kio::Ref<'_, Outbox>| match outbox.is_empty() { + true => Poll::Pending, + false => Poll::Ready(()), + }; + let mut outbox = ready!(self.queue.poll(waiter, ready)); + Poll::Ready(outbox.pop().expect("an outbox holding something pops it")) } /// The next output already queued, without waiting. pub fn try_output(&self) -> Option { - self.queue.try_pop().ok().flatten() + if self.queue.read().is_empty() { + return None; + } + self.queue.lock().pop() } fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { @@ -331,7 +466,6 @@ impl Stream { impl Drop for Stream { fn drop(&mut self) { - self.queue.close(); let mut state = self.node.lock(); state.queues.remove(&self.key); state.router.disconnect(self.key); @@ -344,9 +478,9 @@ impl Drop for Stream { } } -fn push(queues: &HashMap>, key: Key, output: Output) { +fn push(queues: &Queues, key: Key, output: Output) { if let Some(queue) = queues.get(&key) { - let _ = queue.try_push(output); + queue.lock().push(output); } } @@ -618,6 +752,70 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A reader that stops reading holds one entry for the route, however often its + /// claims change: the last change to each, and none for a claim it was never sent. + #[test] + fn a_stalled_reader_holds_the_net_change() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + outputs(&answer); + for cost in 0..1000 { + origin.own(p("live/x"), Some(claim(&instance, cost)), None); + origin.own(p("live/churn"), Some(claim(&instance, cost)), None); + origin.own(p("live/churn"), None, None); + } + let read = outputs(&answer); + let batch = |read: &[Output]| match read { + [.., Output::Batch { changes, .. }] => changes.clone(), + _ => panic!("no batch: {read:?}"), + }; + assert!(read.len() <= 2, "{read:?}"); + assert_eq!(batch(&read), [(p("live/x"), Some(claim(&instance, 999)))]); + + // Once sent, a claim's end is kept, behind what started. + for cost in 0..1000 { + origin.own(p("live/y"), Some(claim(&instance, cost)), None); + } + origin.own(p("live/x"), None, None); + let read = outputs(&answer); + assert_eq!(read.len(), 1, "{read:?}"); + assert_eq!( + batch(&read), + [(p("live/y"), Some(claim(&instance, 999))), (p("live/x"), None)] + ); + } + + /// A route that starts and ends while the reader stalls reaches it as an end alone, + /// which the reader acks at once since it never held the route. + #[test] + fn a_route_started_and_ended_unread_is_an_end() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + let answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&answer, &receive); + let edge = relay + .answer( + Link { + proxy: true, + ..Link::default() + }, + Path::empty(), + ) + .unwrap(); + drop(receive); + let read: Vec = (outputs(&edge).into_iter()) + .filter(|output| matches!(output, Output::Route { id: 1, .. } | Output::Batch { id: 1, .. })) + .collect(); + assert_eq!(read, [Output::Route { id: 1, reach: None }]); + assert_eq!(relay.retained().echoes, 1, "the end awaits its echo"); + edge.echo(1); + assert_eq!(relay.retained().echoes, 0); + } + #[test] fn a_route_lost_with_its_session_leaves_the_others_serving() { let instance = Instance::random(); From 2329bb19170885172f81d889528a0ac3c08c3998 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:03:03 -0800 Subject: [PATCH 23/61] chore: leave MIGRATE.md to the release cut Main now has the release cut write migration notes; this PR carries them in its Impact section instead. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/MIGRATE.md | 10 ---------- rs/moq-net/MIGRATE.md | 9 --------- rs/moq-relay/MIGRATE.md | 14 -------------- 3 files changed, 33 deletions(-) diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 3070c7399d2..e29b62468e9 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,16 +4,6 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased -### `--cluster-id` is removed - -Remove `--cluster-id` and `MOQ_CLUSTER_ID`: `moq` refuses both as unknown settings. -The Hop ID is random on every run and nothing pins it. - -### Zero-cost cluster links are refused - -A `--cluster-connect` or `--cluster-api-url` peer priced at 0 (`?cost=0`, or an object `cost` or `egress` of 0) is refused, as is a `--cluster-node` carrying `?cost=0` under `--cluster-lan`. -A link costs at least 1, the price of an unpriced link: use `?cost=1` for a same-datacenter sibling. - ### The iroh transport is removed ([#7345](https://github.com/moq-dev/moq/pull/7345)) The `iroh` Cargo feature and the `--iroh-*` flags are gone, and a `MOQ_IROH_*` variable stops startup like any other unknown `MOQ_*` variable. diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 3d7ff6a09ef..c9a8ccd87ec 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,15 +4,6 @@ How to update your code for each breaking change, newest first. ## Unreleased -### moq-lite-07 routes through the route layer ([#8023](https://github.com/moq-dev/moq/pull/8023)) - -moq-lite-07-wip (opt-in) replaces the hop list with routes: each origin node declares a random routing identity, every session on an origin feeds one router, and a request names the origin node the first hop picked. Older versions are unchanged. On moq-lite-07: - -- A relay built on the library must opt into proxying, or its peers are sent only the routes to its origin's own claims: publish `origin.clone().with_proxy(true)`, or mark one consumer with `consumer.with_proxy(true)`. `moq-relay` does this for every session; an application publishing only its own broadcasts leaves it off. -- A link cost of 0 is refused at SETUP: `Client::with_cost(0)` fails the session. Use 1, the price of an unpriced link. -- A route carries no hop chain. `RouteInfo::hops()` on a route learned over moq-lite-07 names only its origin node, and `origin::Config::max_hops` does not bound it: routes are loop-free by feasibility instead. -- A pool's paths spread by a hash of the path and each origin node's routing identity, which every relay computes alike, rather than by publisher instance, so every relay picks the same origin for a path. - ### Snapshots split connected sessions into idle and active `stats::Snapshot::activity()` returns each tier's `stats::Activity`: connected sessions with diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 3823718c8af..6f06755853d 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,20 +5,6 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased -### `cluster.id` is removed - -Remove `--cluster-id`, `MOQ_CLUSTER_ID`, and `[cluster] id`; in Rust, drop `cluster::Config::id`. -A relay that still sets one stops at startup: the flag, variable, and key are refused as unknown settings. -Each relay picks a random Hop ID on every start and nothing pins it, so a restarted relay is a new node to the mesh. -`Cluster::id()` still reports the one in use. - -### Zero-cost cluster links are refused - -A link costs at least 1, the price of an unpriced link, since routing needs every link to cost something. -A peer priced at 0 (`?cost=0` on its URL, or an object `cost` or `egress` of 0) stops startup in `cluster.connect` and rejects a `cluster.api.url` update, which keeps the last good list. -With LAN discovery on, a `cluster.node` whose `?cost=` is 0 or malformed stops startup, and a LAN peer advertising one is skipped. -Price a same-datacenter sibling at 1, and raise the other links' costs if it must stay relatively cheaper. - ### Shutdown reports deadline cuts separately from end reports `shutdown::Observer::drained()` now returns `shutdown::Drained` instead of `()`. From 6746f6e1d8067ab272fbe354d6fba20ed7b686d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:08:03 -0800 Subject: [PATCH 24/61] refactor(net): name the outbox's merged changes for clippy Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 9367e4c956c..9e5d00083e3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -123,9 +123,12 @@ struct Pending { /// The route was held after that. held: bool, /// The batches since, merged: each claim's last change, and the last floor. - batch: Option<(BTreeMap>, Option)>, + batch: Option<(Changes, Option)>, } +/// Each claim's last change, by path. +type Changes = BTreeMap>; + impl Outbox { fn is_empty(&self) -> bool { self.ready.is_empty() && self.order.is_empty() From 2d6790dbb2b8b172cade4e288f2839162c97eba2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:21:02 -0800 Subject: [PATCH 25/61] fix(net): ack routes ended unread at once, so a stalled outbox holds only held routes The outbox now knows which routes its reader holds. A ROUTE_END of a route the reader never read, a start or a restart since its last read, cancels what is pending and is acked to the router at once, as the reader would: no peer holds it. So churning fresh identities leaves nothing behind, and an end, restart, and end of a route the reader holds reaches it as one end with one echo owed. The answer no longer acks an unread end itself; the node does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 16 +- rs/moq-net/src/route/node.rs | 206 ++++++++++++++++-------- 2 files changed, 141 insertions(+), 81 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index e3331f9da30..865c15b9b23 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -116,9 +116,7 @@ impl Answer { let answer = node .answer(self.link.clone(), self.prefix.clone()) .map_err(|_| Error::ProtocolViolation)?; - // Held before its first outputs, so an end it acks at once reaches the node. - self.stream = Some(answer); - while let Some(output) = self.stream.as_ref().and_then(node::Stream::try_output) { + while let Some(output) = answer.try_output() { self.output(stream, output)?; } // Every route the peer holds has a floor before SYNCED; one the node has @@ -134,6 +132,7 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; + self.stream = Some(answer); self.phase = Phase::Running; } @@ -254,15 +253,8 @@ impl Answer { } self.echoes.insert(sent.wire, id); } - // The node coalesced the route's start away: the peer never held it, so nothing - // of it lingers, and the end is acked at once. - (None, None) => { - if let Some(answer) = &self.stream { - answer.echo(id); - } - } - // Held only follows a route the peer holds. - (Some(route::Reach::Held), None) => {} + // Held only follows a route the peer holds, and an end of one it does not is moot. + (Some(route::Reach::Held) | None, None) => {} } Ok(()) } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 9e5d00083e3..fd0e3e130e0 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -99,24 +99,26 @@ impl std::fmt::Debug for Node { type Queues = HashMap>; /// What one stream has yet to read. Route changes and claim batches coalesce per -/// identity, and a claim the reader was never sent leaves nothing when it ends, so a -/// stream whose reader stalls holds one entry per identity, bounded by its claims, -/// however much the node changes meanwhile. +/// identity, and what the reader was never sent leaves nothing when it ends: a route +/// started and ended unread is acked to the router at once, as the reader would. A +/// stream whose reader stalls holds one entry per route it holds or will start, +/// bounded by its claims, however much the node changes meanwhile. #[derive(Default)] struct Outbox { - /// Identities with route changes or claims pending, in the order they first queued. - order: VecDeque, - pending: HashMap, + /// Identities with route changes or claims pending, by when they first queued. + order: BTreeMap, + next: u64, + pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// The claims the reader was sent for each identity. + /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, } /// One identity's route changes and claim batches since the reader last took it. #[derive(Default)] struct Pending { - /// A ROUTE_END goes first, retiring every claim sent before. + /// A ROUTE_END goes first, ending the route the reader holds and its claims. ended: bool, /// The last finite reach since, which starts the route if the reader holds none. finite: Option, @@ -135,35 +137,57 @@ impl Outbox { } /// The pending entry for `id`, queued behind the others when new. - fn entry<'a>(order: &mut VecDeque, pending: &'a mut HashMap, id: Id) -> &'a mut Pending { - pending.entry(id).or_insert_with(|| { - order.push_back(id); - Pending::default() - }) + fn entry(&mut self, id: Id) -> &mut Pending { + let (order, next) = (&mut self.order, &mut self.next); + let (_, entry) = self.pending.entry(id).or_insert_with(|| { + let seq = *next; + *next += 1; + order.insert(seq, id); + (seq, Pending::default()) + }); + entry } - fn push(&mut self, output: Output) { + /// Queue `output`. Returns the identity of a ROUTE_END the reader will never read, + /// since it never read the route it ends, for the caller to ack in its place. + fn push(&mut self, output: Output) -> Option { match output { - Output::Route { id, reach } => { - let entry = Self::entry(&mut self.order, &mut self.pending, id); - match reach { - None => { - *entry = Pending { - ended: true, - ..Pending::default() - } - } - Some(Reach::Held) => entry.held = true, - Some(finite) => { - entry.finite = Some(finite); - entry.held = false; - } + Output::Route { id, reach: None } => { + let holds = self.delivered.contains_key(&id); + let entry = self.entry(id); + // The reader holds the route once it reads what is pending: an end of + // that keeps any end before, an end of a start it never read is acked. + let ended = entry.ended; + let acked = !holds || ended; + *entry = Pending { + ended: holds, + ..Pending::default() + }; + if !holds && let Some((seq, _)) = self.pending.remove(&id) { + self.order.remove(&seq); } + acked.then_some(id) + } + Output::Route { + id, + reach: Some(Reach::Held), + } => { + self.entry(id).held = true; + None + } + Output::Route { + id, + reach: Some(finite), + } => { + let entry = self.entry(id); + entry.finite = Some(finite); + entry.held = false; + None } Output::Batch { id, changes, floor } => { - let entry = Self::entry(&mut self.order, &mut self.pending, id); + let ended = self.entry(id).ended; + let (_, entry) = self.pending.get_mut(&id).expect("just queued"); let sent = self.delivered.get(&id); - let ended = entry.ended; let (merged, last) = entry.batch.get_or_insert_default(); for (path, claim) in changes { // The reader holds nothing here once the entry reaches it. @@ -174,15 +198,19 @@ impl Outbox { }; } *last = floor; + None + } + other => { + self.ready.push_back(other); + None } - other => self.ready.push_back(other), } } fn pop(&mut self) -> Option { if self.ready.is_empty() { - let id = self.order.pop_front()?; - let pending = self.pending.remove(&id).expect("every queued identity is pending"); + let (_, id) = self.order.pop_first()?; + let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); self.take(id, pending); } self.ready.pop_front() @@ -195,6 +223,7 @@ impl Outbox { self.ready.push_back(Output::Route { id, reach: None }); } if let Some(reach) = pending.finite { + self.delivered.entry(id).or_default(); self.ready.push_back(Output::Route { id, reach: Some(reach) }); } if pending.held { @@ -204,17 +233,15 @@ impl Outbox { }); } if let Some((merged, floor)) = pending.batch { - let sent = self.delivered.entry(id).or_default(); // As the router orders a batch: what starts or changes, then what ends. let (ends, starts): (Vec<_>, Vec<_>) = merged.into_iter().partition(|(_, claim)| claim.is_none()); - for (path, _) in &starts { - sent.insert(path.clone()); - } - for (path, _) in &ends { - sent.remove(path); - } - if sent.is_empty() { - self.delivered.remove(&id); + if let Some(sent) = self.delivered.get_mut(&id) { + for (path, _) in &starts { + sent.insert(path.clone()); + } + for (path, _) in &ends { + sent.remove(path); + } } let changes = starts.into_iter().chain(ends).collect(); self.ready.push_back(Output::Batch { id, changes, floor }); @@ -481,23 +508,35 @@ impl Drop for Stream { } } -fn push(queues: &Queues, key: Key, output: Output) { - if let Some(queue) = queues.get(&key) { - queue.lock().push(output); - } +/// Queue `output` for the stream `key`. Returns the identity of a ROUTE_END its +/// reader will never read, whose echo the router is owed. +fn push(queues: &Queues, key: Key, output: Output) -> Option { + queues.get(&key).and_then(|queue| queue.lock().push(output)) } impl State { /// Route the router's messages and shown claims to the streams. fn flush(&mut self) { - for out in self.router.drain() { - let (key, output) = match out { - Out::Route { to, id, reach } => (to, Output::Route { id, reach }), - Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), - Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), - Out::Echo { to, id } => (to, Output::Echo { id }), - }; - push(&self.queues, key, output); + loop { + let mut acked = Vec::new(); + for out in self.router.drain() { + let (key, output) = match out { + Out::Route { to, id, reach } => (to, Output::Route { id, reach }), + Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), + Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Echo { to, id } => (to, Output::Echo { id }), + }; + if let Some(id) = push(&self.queues, key, output) { + acked.push((key, id)); + } + } + if acked.is_empty() { + break; + } + // A ROUTE_END coalesced away before its reader read the route: no peer holds it. + for (key, id) in acked { + self.router.echo(key, id); + } } // The shown claims below are read whole; the events only say which changed. self.router.events(); @@ -789,17 +828,13 @@ mod tests { ); } - /// A route that starts and ends while the reader stalls reaches it as an end alone, - /// which the reader acks at once since it never held the route. + /// Routes that start and end while the reader stalls leave nothing for it, and the + /// router is acked at once as the reader would: churning fresh identities holds + /// nothing. A route the reader holds ends once, however often it restarted unread. #[test] - fn a_route_started_and_ended_unread_is_an_end() { + fn routes_ended_unread_leave_nothing() { let instance = Instance::random(); - let origin = Node::new(1, 1); let relay = Node::new(2, 2); - let receive = relay.receive(Link::default(), Path::empty()).unwrap(); - let answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0)), None); - pump(&answer, &receive); let edge = relay .answer( Link { @@ -809,13 +844,46 @@ mod tests { Path::empty(), ) .unwrap(); - drop(receive); - let read: Vec = (outputs(&edge).into_iter()) - .filter(|output| matches!(output, Output::Route { id: 1, .. } | Output::Batch { id: 1, .. })) - .collect(); - assert_eq!(read, [Output::Route { id: 1, reach: None }]); - assert_eq!(relay.retained().echoes, 1, "the end awaits its echo"); - edge.echo(1); + outputs(&edge); + let advert = |seqno| { + Some(Reach::Finite(Advert { + seqno, + metric: 0, + down_only: false, + })) + }; + for origin in 10..110 { + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(origin, advert(0)); + receive.batch(origin, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + drop(receive); + } + let of = |read: Vec, of: fn(Id) -> bool| -> Vec { + (read.into_iter()) + .filter(|output| matches!(output, Output::Route { id, .. } | Output::Batch { id, .. } if of(*id))) + .collect() + }; + let read = of(outputs(&edge), |id| id >= 10); + assert_eq!(read, [], "the churned routes"); + assert_eq!(relay.retained().echoes, 0); + + // The reader holds a route, which ends, restarts, and ends again unread. + let receive = relay.receive(Link::default(), Path::empty()).unwrap(); + receive.route(7, advert(0)); + receive.batch(7, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(1)); + assert_eq!(of(outputs(&edge), |id| id == 7).len(), 2, "the route and its batch"); + receive.route(7, None); + receive.route(7, advert(1)); + receive.batch(7, vec![(p("live/x"), Some(claim(&instance, 0)))], Some(2)); + receive.route(7, None); + let read = of(outputs(&edge), |id| id == 7); + assert_eq!(read, [Output::Route { id: 7, reach: None }]); + assert_eq!( + relay.retained().echoes, + 1, + "only the end the reader reads awaits its echo" + ); + edge.echo(7); assert_eq!(relay.retained().echoes, 0); } From 554fae60f1e907c3c8d2e4b5d430e16e7591e1f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 10:41:13 -0800 Subject: [PATCH 26/61] fix(net): present a handle's mounts on lite-07 A lite-07 answer read only the requested prefix from the route node, so a mounted handle's target never reached a lite-07 reader, as it does in process and on lite-06. The answer now reads each mount beneath its prefix through a node stream of its own, at the target, and names what it presents under the mount point on the same routes. What the origin holds beneath a mount point stays hidden. A route stands while any of its streams carries it; one that stops early ends what it presented and is acked at once. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 190 ++++++++++++++++++------ rs/moq-net/src/model/mod.rs | 2 + rs/moq-net/src/model/origin.rs | 58 +++++++- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/integration/mount.rs | 172 +++++++++++++++++++++ 5 files changed, 376 insertions(+), 47 deletions(-) create mode 100644 rs/moq-net/tests/integration/mount.rs diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 865c15b9b23..cde695a2ae3 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -5,6 +5,9 @@ //! hidden claims unless the peer asked for them, and anything this handle's scope or //! the session's grant leaves out, in which case a floor certifies nothing, since the //! peer cannot tell what was left out from what is gone. +//! +//! A handle's mounts beneath the prefix are read through a node stream each, at the +//! target, and what they present is named under the mount point on the same routes. use std::{ collections::{BTreeMap, BTreeSet, HashMap}, @@ -24,7 +27,10 @@ use crate::{ struct Sent { /// Its Route ID on the stream. wire: u64, - /// Every claim the node sent for it, before filtering. + /// Each source's last reach for it, by index: the route stands while one has one, + /// and the first one's is the peer's. + reach: BTreeMap, + /// Every claim the node sent for it, named as the handle names it, before filtering. claims: BTreeMap, /// The node's floor for those claims. floor: Option, @@ -32,6 +38,20 @@ struct Sent { floored: bool, } +impl Sent { + fn reach(&self) -> Option { + self.reach.values().next().copied() + } +} + +/// A node stream the answer reads. +struct Source { + stream: node::Stream, + /// The mount it reads through, which names what it presents. `None` for the + /// requested prefix's own. + mount: Option, +} + enum Phase { /// Waiting for the session's grant before the first message. Init, @@ -50,12 +70,17 @@ pub(super) struct Answer { prefix: Path, hidden: bool, link: route::Link, - stream: Option, + /// The prefix's own node stream, then one per mount beneath it. Empty until the + /// grant is known, and once withdrawing. + sources: Vec, + /// The source read first next, so a busy one does not hold the others. + turn: usize, encoder: lite::AnnounceEncoder, routes: HashMap, next_route: u64, - /// Route IDs ended and not yet echoed, and the origin node each reached. - echoes: HashMap, + /// Route IDs ended and not yet echoed, the origin node each reached, and the + /// source that ended it. + echoes: HashMap, /// The claims the peer holds: their Announce ID and attributes. live: HashMap<(route::Id, Path), (u64, node::Claim)>, gate: crate::auth::Gate, @@ -82,7 +107,8 @@ impl Answer { origin, hidden, link, - stream: None, + sources: Vec::new(), + turn: 0, encoder: lite::AnnounceEncoder::new(Version::Lite07), routes: HashMap::new(), next_route: 0, @@ -113,11 +139,19 @@ impl Answer { active: 0, })?; let node = self.origin.route_node(); - let answer = node - .answer(self.link.clone(), self.prefix.clone()) - .map_err(|_| Error::ProtocolViolation)?; - while let Some(output) = answer.try_output() { - self.output(stream, output)?; + let mounts = self.origin.mounted(&self.prefix); + let interests = std::iter::once((self.prefix.clone(), None)) + .chain(mounts.into_iter().map(|mount| (mount.interest.clone(), Some(mount)))); + for (interest, mount) in interests { + let stream = node + .answer(self.link.clone(), interest) + .map_err(|_| Error::ProtocolViolation)?; + self.sources.push(Source { stream, mount }); + } + for index in 0..self.sources.len() { + while let Some(output) = self.sources[index].stream.try_output() { + self.output(stream, index, output)?; + } } // Every route the peer holds has a floor before SYNCED; one the node has // not certified yet certifies nothing. @@ -132,7 +166,6 @@ impl Answer { } } stream.writer.buffer(&Msg::Synced)?; - self.stream = Some(answer); self.phase = Phase::Running; } @@ -152,7 +185,7 @@ impl Answer { if withdrawing { // Let go of the node first, so nothing more is queued, then end every route. - self.stream = None; + self.sources.clear(); for (_, sent) in std::mem::take(&mut self.routes) { stream.writer.buffer(&Msg::RouteEnd { id: sent.wire })?; } @@ -182,9 +215,16 @@ impl Answer { Poll::Pending => {} } - let answer = self.stream.as_ref().expect("running streams hold the node"); - if let Poll::Ready(output) = answer.poll_output(waiter) { - self.output(stream, output)?; + let count = self.sources.len(); + let ready = (0..count).map(|offset| (self.turn + offset) % count).find_map(|index| { + match self.sources[index].stream.poll_output(waiter) { + Poll::Ready(output) => Some((index, output)), + Poll::Pending => None, + } + }); + if let Some((index, output)) = ready { + self.turn = index + 1; + self.output(stream, index, output)?; continue; } return Poll::Pending; @@ -193,22 +233,36 @@ impl Answer { /// A ROUTE_REQUEST, or the echo of a ROUTE_END. fn reply(&mut self, reply: lite::AnnounceReply) -> Result<(), Error> { - let answer = self.stream.as_ref().expect("running streams hold the node"); match reply { - lite::AnnounceReply::RouteRequest { node, seqno } => answer.request(node, seqno), + lite::AnnounceReply::RouteRequest { node, seqno } => { + // Through the first source carrying the route, or the prefix's own. + let index = (self.routes.get(&node)) + .and_then(|sent| sent.reach.keys().next().copied()) + .unwrap_or(0); + if let Some(source) = self.sources.get(index) { + source.stream.request(node, seqno); + } + } lite::AnnounceReply::RouteEnd { id } => { // The echo names the Route ID; the node knows the route by its origin node. - let node = self.echoes.remove(&id).ok_or(Error::ProtocolViolation)?; - answer.echo(node); + let (node, index) = self.echoes.remove(&id).ok_or(Error::ProtocolViolation)?; + if let Some(source) = self.sources.get(index) { + source.stream.echo(node); + } } } Ok(()) } - fn output(&mut self, stream: &mut Stream, output: node::Output) -> Result<(), Error> { + fn output( + &mut self, + stream: &mut Stream, + source: usize, + output: node::Output, + ) -> Result<(), Error> { match output { - node::Output::Route { id, reach } => self.route(stream, id, reach), - node::Output::Batch { id, changes, floor } => self.batch(stream, id, changes, floor), + node::Output::Route { id, reach } => self.route(stream, source, id, reach), + node::Output::Batch { id, changes, floor } => self.batch(stream, source, id, changes, floor), // Only a stream this node asked for requests, serves, or echoes. node::Output::Request { .. } | node::Output::Echo { .. } @@ -220,29 +274,34 @@ impl Answer { fn route( &mut self, stream: &mut Stream, + source: usize, id: route::Id, reach: Option, ) -> Result<(), Error> { - match (reach, self.routes.get(&id)) { - (Some(route::Reach::Finite(advert)), Some(sent)) => { - stream.writer.buffer(&Msg::RouteUpdate { id: sent.wire, advert })?; - } - (Some(route::Reach::Finite(advert)), None) => { + let Some(sent) = self.routes.get_mut(&id) else { + // Held only follows a route the peer holds, and an end of one it does not is moot. + if let Some(reach @ route::Reach::Finite(advert)) = reach { let wire = self.next_route; self.next_route += 1; stream.writer.buffer(&Msg::RouteStart { node: id, advert })?; - self.routes.insert( - id, - Sent { - wire, - claims: BTreeMap::new(), - floor: None, - floored: false, - }, - ); + let sent = Sent { + wire, + reach: BTreeMap::from([(source, reach)]), + claims: BTreeMap::new(), + floor: None, + floored: false, + }; + self.routes.insert(id, sent); + } + return Ok(()); + }; + let before = sent.reach(); + match reach { + Some(reach) => { + sent.reach.insert(source, reach); } - (Some(route::Reach::Held), Some(sent)) => stream.writer.buffer(&Msg::RouteHold { id: sent.wire })?, - (None, Some(_)) => { + None if sent.reach.remove(&source).is_none() => return Ok(()), + None if sent.reach.is_empty() => { let sent = self.routes.remove(&id).expect("matched above"); stream.writer.buffer(&Msg::RouteEnd { id: sent.wire })?; // The ROUTE_END retires every claim under the route. @@ -251,17 +310,47 @@ impl Answer { self.encoder.end(announce); } } - self.echoes.insert(sent.wire, id); + self.echoes.insert(sent.wire, (id, source)); + return Ok(()); + } + None => { + // Another source still carries the route, so the peer keeps it: the claims + // this one presented end, and its end is acked at once. + let paths: Vec = sent.claims.keys().cloned().collect(); + let paths: Vec = (paths.into_iter()).filter(|path| self.source(path) == source).collect(); + self.sources[source].stream.echo(id); + let sent = self.routes.get_mut(&id).expect("matched above"); + for path in &paths { + sent.claims.remove(path); + } + let mut started = BTreeSet::new(); + for path in paths { + self.sync(stream, id, path, &mut started)?; + } } - // Held only follows a route the peer holds, and an end of one it does not is moot. - (Some(route::Reach::Held) | None, None) => {} + } + let sent = &self.routes[&id]; + match sent.reach() { + after if after == before => {} + Some(route::Reach::Finite(advert)) => stream.writer.buffer(&Msg::RouteUpdate { id: sent.wire, advert })?, + Some(route::Reach::Held) => stream.writer.buffer(&Msg::RouteHold { id: sent.wire })?, + None => unreachable!("a route stands while a source carries it"), } Ok(()) } + /// The index of the source presenting the claim at the handle-side `path`: the + /// mount it lies beneath, or the prefix's own. + fn source(&self, path: &Path) -> usize { + (self.sources.iter()) + .position(|source| source.mount.as_ref().is_some_and(|mount| mount.holds(path))) + .unwrap_or(0) + } + fn batch( &mut self, stream: &mut Stream, + source: usize, id: route::Id, changes: Vec<(Path, Option)>, floor: Option, @@ -269,10 +358,27 @@ impl Answer { let Some(sent) = self.routes.get_mut(&id) else { return Ok(()); }; + let mount = self.sources.get(source).and_then(|source| source.mount.as_ref()); let mut paths = Vec::with_capacity(changes.len()); for (path, claim) in changes { + // Named as the handle names it: beneath the mount point for a mount, and never + // a claim the origin holds beneath a mount for the prefix's own. + let path = match mount { + Some(mount) => match mount.name(&path) { + Some(named) => named, + None => continue, + }, + None if self.origin.shadows(&path) => continue, + None => path, + }; match claim { - Some(claim) => sent.claims.insert(path.clone(), claim), + Some(mut claim) => { + if let Some(mount) = mount { + claim.continues = (claim.continues.take()) + .and_then(|(taken, instance)| Some((mount.name(&taken)?, instance))); + } + sent.claims.insert(path.clone(), claim) + } None => sent.claims.remove(&path), }; paths.push(path); diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 975f59ad0ed..01b84877195 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -62,6 +62,8 @@ pub mod origin { // private: capability gap pub(crate) use super::origin_impl::Lookup; #[cfg(feature = "async")] + pub(crate) use super::origin_impl::Mounted; + #[cfg(feature = "async")] pub(crate) use super::origin_impl::NodeFilter; pub use super::origin_impl::{ Config, Consumer, Cost, Driver, HopLimit, Link, Preference, Producer, Request, Requesting, Route, RouteInfo, diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index ebb0ed615d7..c3d2a518071 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1920,6 +1920,31 @@ struct Mount { target: Path, } +/// A mount as a reader of a prefix sees it through a handle: where the origin holds +/// what it presents, and how the handle names that. +#[derive(Clone, Debug)] +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +pub(crate) struct Mounted { + mount: Mount, + /// The absolute, origin-side prefix to read: the target, or beneath it for a + /// reader's prefix beneath the mount point. + pub interest: Path, +} + +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +impl Mounted { + /// How the handle names a claim at the absolute, origin-side `path`, which lies at + /// or beneath the interest: at or beneath the mount point. + pub fn name(&self, path: &Path) -> Option { + self.mount.name(path) + } + + /// Whether the absolute, handle-side `path` lies at or beneath the mount point. + pub fn holds(&self, path: &Path) -> bool { + path.has_prefix(&self.mount.at) + } +} + impl Mount { /// Where the handle-side absolute `path` resolves, when it is under this mount /// and the result fits the wire limits. @@ -7186,17 +7211,40 @@ impl Consumer { self.horizon.upstream } - /// Whether this handle reaches a claim at the absolute `path` as the origin keys - /// it: a path at or beneath it may be in its scope, as a covering reader sees, and - /// it is not under a mount, which resolves elsewhere, or an excluded subtree. + /// Whether this handle reaches a claim at the absolute `path` as it names it: a + /// path at or beneath it may be in its scope, as a covering reader sees, and it is + /// not in an excluded subtree. Beneath a mount, `path` names the target's claim + /// there, never one the origin holds at `path` itself (see [`Self::mounted`]). #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn reaches(&self, path: &Path) -> bool { let Ok(subtree) = Pattern::subtree(path.as_str()) else { return false; }; - self.scope.mount(path).is_none() - && self.scope.allowed.overlaps(&subtree) + self.scope.allowed.overlaps(&subtree) && !self.scope.excluded.iter().any(|excluded| path.has_prefix(excluded)) + && self.scope.mount(path).is_none_or(|mount| mount.resolve(path).is_some()) + } + + /// Whether the absolute `path` lies beneath one of this handle's mounts, where the + /// origin's own claims at that path are hidden from it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn shadows(&self, path: &Path) -> bool { + self.scope.mount(path).is_some() + } + + /// The mounts a reader of the absolute `prefix` sees through this handle: each one + /// at or beneath it, or the one it lies beneath, unless excluded. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn mounted(&self, prefix: &Path) -> Vec { + (self.scope.mounts.iter()) + .filter(|mount| self.scope.holes(Some(mount)).is_some()) + .filter_map(|mount| { + Some(Mounted { + interest: mount.translate_head(prefix)?, + mount: mount.clone(), + }) + }) + .collect() } /// Whether this handle reaches every path at or beneath the absolute `prefix`, diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 78b2f3a2b22..50416d0c00c 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -43,6 +43,7 @@ mod late_lower_group; mod lazy_discovery; mod mesh_withdraw; mod model; +mod mount; mod multi_cdn; mod partial_group; mod quiet_catalog; diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs new file mode 100644 index 00000000000..6967b6ef412 --- /dev/null +++ b/rs/moq-net/tests/integration/mount.rs @@ -0,0 +1,172 @@ +#![cfg(feature = "async")] + +//! A session answered through a mounted handle presents the target's broadcasts under +//! the mount point on every version, hides what the origin holds at the mount point +//! itself, and a reader there reaches the target. + +use crate::support; + +use std::time::Duration; + +use moq_net::{Hop, Pattern, Patterns, Timestamp, Version, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(600); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// Publish a broadcast at `path` whose `video` track carries the path. +fn publish(origin: &origin::Producer, path: &str) -> (origin::Route, track::Producer) { + let route = origin.route(path, origin::RouteOptions::default()).unwrap(); + let track = route + .broadcast() + .create_track( + "video", + track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, path.as_bytes().to_vec()).unwrap(); + group.finish().unwrap(); + (route, track) +} + +const PUBLISHED: [&str; 4] = [ + "tenant-a/public/cam", + "tenant-a/private/mic", + "tenant-b/own", + "tenant-b/shared/forged", +]; + +/// Tenant B's handle rooted at `root`, with tenant A's public broadcasts mounted at +/// `tenant-b/shared`. +fn tenant(origin: &origin::Producer, root: &str) -> origin::Consumer { + origin + .mount("tenant-b/shared", "tenant-a/public") + .unwrap() + .scope(root, &Patterns::from(Pattern::all())) + .unwrap() + .consume() + .with_proxy(true) +} + +/// Have `to` read `from` through a session. +async fn link(version: Version, from: origin::Consumer, to: &origin::Producer) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from); + options.client_subscribe = Some(to.clone()); + connect_mock(options).await +} + +/// Every announce event already delivered, as `(started, prefix)`. +async fn drain(routes: &mut origin::Routes) -> Vec<(bool, String)> { + let mut events = Vec::new(); + while let Ok(event) = kio::clock::timeout(Duration::from_secs(1), routes.next()).await { + match event.expect("announce ended") { + origin::RouteEvent::Start(announce) => events.push((true, announce.prefix.to_string())), + origin::RouteEvent::End(announce) => events.push((false, announce.prefix.to_string())), + _ => {} + } + } + events.sort(); + events +} + +/// The first frame of `video` at `path` through `origin`. +async fn watch(origin: &origin::Producer, path: &str) -> String { + let broadcast = origin.consume().request(path, None).await.unwrap(); + let mut subscription = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); + let mut group = kio::clock::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("subscription hung") + .expect("the subscription was reset") + .expect("a group"); + let frame = group.read_frame().await.unwrap().expect("a frame"); + String::from_utf8(frame.payload.to_vec()).unwrap() +} + +fn started(paths: &[&str]) -> Vec<(bool, String)> { + paths.iter().map(|path| (true, path.to_string())).collect() +} + +fn ended(paths: &[&str]) -> Vec<(bool, String)> { + paths.iter().map(|path| (false, path.to_string())).collect() +} + +async fn mounted(version: &str) { + let version: Version = version.parse().unwrap(); + let source = produce_origin(1); + let viewer = produce_origin(2); + let mut published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&source, path)).collect(); + let _pair = link(version, tenant(&source, "tenant-b"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); + assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); + + // What leaves the target leaves the mount. + drop(published.remove(0)); + assert_eq!(drain(&mut routes).await, ended(&["shared/cam"]), "{version}"); +} + +/// A session rooted inside the mount reads only the target. +async fn rooted_inside(version: &str) { + let version: Version = version.parse().unwrap(); + let source = produce_origin(1); + let viewer = produce_origin(2); + let _published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&source, path)).collect(); + let _pair = link(version, tenant(&source, "tenant-b/shared"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["cam"]), "{version}"); + assert_eq!(watch(&viewer, "cam").await, "tenant-a/public/cam", "{version}"); +} + +/// `W -> R -> V`: the relay learns the target from upstream and presents it under the +/// mount, and lets it go with the upstream. +async fn relayed(version: &str) { + let version: Version = version.parse().unwrap(); + let worker = produce_origin(1); + let relay = produce_origin(2); + let viewer = produce_origin(3); + let _published: Vec<_> = PUBLISHED.into_iter().map(|path| publish(&worker, path)).collect(); + let upstream = link(version, worker.consume(), &relay).await; + let _downstream = link(version, tenant(&relay, "tenant-b"), &viewer).await; + kio::clock::sleep(Duration::from_millis(500)).await; + + let mut routes = viewer.consume().routes(); + assert_eq!(drain(&mut routes).await, started(&["own", "shared/cam"]), "{version}"); + assert_eq!(watch(&viewer, "shared/cam").await, "tenant-a/public/cam", "{version}"); + + drop(upstream); + kio::clock::sleep(Duration::from_millis(500)).await; + assert_eq!(drain(&mut routes).await, ended(&["own", "shared/cam"]), "{version}"); +} + +macro_rules! mounted_tests { + ($($name:ident: $test:ident, $version:literal,)*) => { + $( + #[kio::test] + async fn $name() { + kio::clock::timeout(TEST_TIMEOUT, $test($version)) + .await + .expect("timed out"); + } + )* + }; +} + +mounted_tests! { + mounted_lite06: mounted, "moq-lite-06", + mounted_lite07: mounted, "moq-lite-07-wip", + rooted_inside_lite06: rooted_inside, "moq-lite-06", + rooted_inside_lite07: rooted_inside, "moq-lite-07-wip", + relayed_lite06: relayed, "moq-lite-06", + relayed_lite07: relayed, "moq-lite-07-wip", +} From 8e03f17f70a36c13bc8a6b142725f95e8dbeb122 Mon Sep 17 00:00:00 2001 From: "pullfrog[bot]" <226033991+pullfrog[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:22:13 +0000 Subject: [PATCH 27/61] test: retarget the drill mutations and allow the gated lite-07 interop test --- sh/gh/test-inventory.allow | 2 + .../relay-withdraws-lost-publisher.patch | 38 +++++++++++++------ .../subscriber-leaks-broadcasts.patch | 22 +++++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index 45e9fe71542..4f421f76ee0 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -40,3 +40,5 @@ ignore moq-video::frame::tests::d3d11_resize_matches_cpu /quest/m5/video-hardwar ignore moq-video::render::export::dx12::tests::exports_an_nv12_texture /quest/m5/video-hardware.md ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_export_in_place /quest/m5/video-hardware.md +# Need another implementation to speak the route wire; the quest ports it. +ignore moq-net::test_interop::lite_tracks_interop /quest/m2/cluster-routing/routes/js.md diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 4dbf9ee93d5..b325d14d5e7 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,16 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -7730,7 +7730,7 @@ impl Announced { - let in_place = predecessor == Some(&path); - // An error means the prefix is outside our origin's scope, so don't serve it. - let predecessor = predecessor.and_then(|previous| match self.routes.get(previous)? { -- Held::Attached(attached) => Some(&attached.dynamic), -+ Held::Attached(attached) => Some(&*attached.dynamic), - _ => None, - }); - match subscriber.origin.forward(&path, route.clone(), predecessor) { -@@ -7819,7 +7819,7 @@ impl Announced { +@@ -8818,7 +8818,7 @@ }; self.held.pop(); if let Held::Attached(entry) = held { @@ -27,7 +18,16 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -7921,7 +7921,8 @@ struct AnnouncedRoute { +@@ -8902,7 +8902,7 @@ + let validity = if subscriber.version.has_instance() { + Default::default() + } else { +- crate::model::broadcast::RequestValidity::for_source(&entry.dynamic, request.path()) ++ crate::model::broadcast::RequestValidity::for_source(&*entry.dynamic, request.path()) + }; + let source = subscriber.origin.create_source(path.clone(), validity); + // The handler exists before the requester sees the source, so a track it +@@ -8930,7 +8930,8 @@ /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -7941,7 +7942,7 @@ impl AnnouncedRoute { +@@ -8950,7 +8951,7 @@ ) -> Self { Self { route, @@ -46,3 +46,17 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. live: Default::default(), drained: false, waker: std::task::Waker::from(wake.clone()), +diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs +--- a/rs/moq-net/src/lite/subscriber/routed.rs ++++ b/rs/moq-net/src/lite/subscriber/routed.rs +@@ -470,8 +470,8 @@ + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. +- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), +- false => (&entry.route.dynamic, None), ++ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), ++ false => (&*entry.route.dynamic, None), + }); + // A new instance or epoch replaces the old route, attached before it goes. + tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); diff --git a/test/drill/mutations/subscriber-leaks-broadcasts.patch b/test/drill/mutations/subscriber-leaks-broadcasts.patch index c484dbf1dc4..8738695e886 100644 --- a/test/drill/mutations/subscriber-leaks-broadcasts.patch +++ b/test/drill/mutations/subscriber-leaks-broadcasts.patch @@ -9,10 +9,9 @@ # session is gone, so a cancelled reader's broadcast parks forever instead of # closing. Leaking only one is not a leak: the other still closes the front. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs -index adbc9aeec1..e4edd44fcf 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -1903,7 +1903,8 @@ struct PrefixRun { +@@ -2042,7 +2042,8 @@ /// it comes from the connect config or the peer's SETUP, neither of which /// changes for the life of the session. link_cost: u64, @@ -22,7 +21,7 @@ index adbc9aeec1..e4edd44fcf 100644 // Lite06+: announce ids. Each received `active` implicitly assigns the next // per-stream ordinal; `ended`/`restart` reference it instead of repeating the // path, and lite-07 bases name it too. Tracked even for announces we drop -@@ -1991,7 +1992,10 @@ impl AnnouncePrefix { +@@ -2145,7 +2146,10 @@ let mut run: PrefixRun = PrefixRun { responder_origin, link_cost, @@ -34,22 +33,31 @@ index adbc9aeec1..e4edd44fcf 100644 decoder: lite::AnnounceDecoder::default(), auth_epoch: 0, initial, -@@ -2123,7 +2127,8 @@ struct SourceServe { - subscriber: Subscriber, +@@ -2301,7 +2305,8 @@ path: Path, instance: Option, + node: u64, - source: crate::model::broadcast::SourceGuard, + // MUTATION: never dropped, so a session that ends keeps every source it minted alive. + source: std::mem::ManuallyDrop, dynamic: crate::broadcast::Requests, route: kio::Consumer<()>, // A dedicated close-watch handle, since each pending operation needs its own. -@@ -2269,7 +2274,7 @@ impl SourceServe { - subscriber, +@@ -2451,7 +2456,7 @@ path: minted.path, instance: minted.instance, + node: minted.node, - source: minted.source, + source: std::mem::ManuallyDrop::new(minted.source), datagram_requests: minted.dynamic.datagrams(), dynamic: minted.dynamic, route: minted.route, +@@ -2560,7 +2565,7 @@ + let serve = TrackServe { + validity: self.dynamic.validity().clone(), + route: Some(self.route.clone()), +- source: Some((*self.source).clone()), ++ source: Some((**self.source).clone()), + subscriber: self.subscriber.clone(), + path: self.path.clone(), + instance: self.instance.clone(), From 45100ff88e60092c92aac5909b1c4ea6ff920bf7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 11:53:54 -0800 Subject: [PATCH 28/61] chore(quest): claim quest/m2/announce-takeover.md Quest-Claim: 5a4a61a81814cd55 From 71f01d04070cd5f4bbaf460d086e605a7a2733da Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:54:03 -0800 Subject: [PATCH 29/61] feat(net)!: a job's exact route continues its claim's instance A prefix route naming no instance lends the one it mints to the exact routes answering its requests. A job's route binds that instance when it first announces, so a reader resolved through the claim continues on the job's route with no cold-start reset, across lite-07 relays. Once a job holding the claim's instance stops serving it, the claim moves to a fresh instance under the table lock, so a restarted job is another stream and a request pinned to the ended one is refused. Removes lite-07 ANNOUNCE_TAKEOVER (0x5), RouteEvent::Takeover, the route layer's claim lineage, and the origin's predecessor tracking. Co-Authored-By: Claude Opus 5.5 --- py/moq-rs/README.md | 2 +- py/moq-rs/moq/origin.py | 4 +- py/moq-rs/tests/test_local.py | 4 +- quest/m1/api-audit/route/README.md | 7 +- quest/m2/README.md | 1 - quest/m2/announce-takeover.md | 159 -- quest/m2/cluster-routing/routes/README.md | 2 +- quest/m2/lite07-finalize.md | 1 - quest/m3/release/runner-client-bump.md | 6 +- rs/moq-c/src/api.rs | 9 +- rs/moq-c/src/test.rs | 5 +- rs/moq-cli/tests/integration/export.rs | 4 +- rs/moq-cli/tests/integration/rtsp.rs | 4 +- rs/moq-ffi/src/origin.rs | 9 +- rs/moq-gst/src/source/imp.rs | 4 +- rs/moq-net-tests/tests/integration/restart.rs | 29 +- rs/moq-net/benches/announce.rs | 3 +- rs/moq-net/benches/auth.rs | 1 - rs/moq-net/benches/origin.rs | 64 +- rs/moq-net/src/fuzz.rs | 13 +- rs/moq-net/src/ietf/publisher.rs | 39 +- rs/moq-net/src/ietf/subscriber.rs | 7 +- rs/moq-net/src/lite/announce.rs | 97 +- rs/moq-net/src/lite/compress.rs | 78 +- rs/moq-net/src/lite/publisher.rs | 32 +- rs/moq-net/src/lite/publisher/routed.rs | 151 +- rs/moq-net/src/lite/replay.rs | 3 +- rs/moq-net/src/lite/subscriber.rs | 7 +- rs/moq-net/src/lite/subscriber/routed.rs | 118 +- rs/moq-net/src/lite/version.rs | 14 +- rs/moq-net/src/model/announced.rs | 6 +- rs/moq-net/src/model/follow.rs | 54 +- rs/moq-net/src/model/origin.rs | 1429 ++++++----------- rs/moq-net/src/model/resume_model.rs | 2 +- rs/moq-net/src/route/node.rs | 43 +- .../integration/announce_covering_route.rs | 1 - .../tests/integration/announce_takeover.rs | 267 --- .../tests/integration/announce_to_serve.rs | 1 - rs/moq-net/tests/integration/auth.rs | 3 +- .../tests/integration/claim_continues.rs | 389 +++++ .../tests/integration/dial_split_horizon.rs | 3 +- rs/moq-net/tests/integration/drained_claim.rs | 8 +- rs/moq-net/tests/integration/extensions.rs | 3 +- .../tests/integration/lazy_discovery.rs | 12 +- rs/moq-net/tests/integration/main.rs | 2 +- rs/moq-net/tests/integration/multi_cdn.rs | 5 +- rs/moq-net/tests/integration/rejoin.rs | 6 +- rs/moq-net/tests/integration/session_panic.rs | 3 +- rs/moq-net/tests/integration/support/mod.rs | 2 - rs/moq-net/tests/integration/synced.rs | 4 +- rs/moq-relay/src/cluster.rs | 3 +- rs/moq-relay/src/internal.rs | 3 +- rs/moq-relay/src/web.rs | 3 +- rs/moq-relay/tests/drills.rs | 3 +- .../tests/integration/auth_lifetime.rs | 3 +- .../tests/integration/cluster_unknown.rs | 3 +- rs/moq-relay/tests/integration/connection.rs | 3 +- rs/moq-relay/tests/integration/embed.rs | 3 +- .../tests/integration/goaway_cluster.rs | 6 +- .../tests/integration/runtime_uring.rs | 3 +- .../tests/integration/runtime_workers.rs | 3 +- rs/moq-relay/tests/integration/smoke.rs | 3 +- rs/moq-stats/src/aggregate.rs | 14 +- rs/moq-stats/src/consume.rs | 3 +- rs/moq-stats/src/produce.rs | 3 +- rs/moq-tokio/tests/integration/broadcast.rs | 3 +- 66 files changed, 1126 insertions(+), 2056 deletions(-) delete mode 100644 quest/m2/announce-takeover.md delete mode 100644 rs/moq-net/tests/integration/announce_takeover.rs create mode 100644 rs/moq-net/tests/integration/claim_continues.rs diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index e3bd133b9d1..78acb272b95 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -288,7 +288,7 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect - `.broadcast → BroadcastProducer`. The broadcast an exact route owns; raises `Internal` on a prefix route. - `.instance`. The stream it serves, kept across `withdraw()` and `announce()`. - `.announce()`, `.withdraw()`, `.set_cost(cost)`. Show, hide, or re-price it in place; `announce` and `set_cost` raise `Closed` once the broadcast closed. - - `await .requested() → BroadcastRequest | None`, or async iteration over a prefix route. Call `.serve(consumer)` to answer with held content under the prefix route's continuous sequence space, `.route(hidden=..., ...)` for the path's own route under a fresh instance, as an independently restarting job, or `.reject(code)` to fail the requester. + - `await .requested() → BroadcastRequest | None`, or async iteration over a prefix route. Call `.serve(consumer)` to answer with held content under the prefix route's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the prefix route's instance until the job ends and then moves the prefix route to a fresh one, or `.reject(code)` to fail the requester. - **`OriginConsumer`**. Discover broadcasts. - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent` or `AnnounceEventAbsent`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards - `.follow(path, instance=None) → Following` (awaitable, waits until something serves the path, then returns each stream that takes over, or `None` once the path goes offline; an `instance` follows only that stream) diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index fac2485f231..d3843b23458 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -119,7 +119,9 @@ def path(self) -> str: def route(self, *, hidden: bool = False, cost: int = 0, instance: str | None = None, epoch: int = 0) -> Route: """Answer with the path's own exact :class:`Route`, owning a new broadcast. - The route serves a fresh instance unless ``instance`` names one, so a job that + Unless ``instance`` names one, the route takes the instance its prefix route lends + when it first announces, so readers that resolved the path through the prefix route + continue on it; its end moves the prefix route to a fresh instance, so a job that restarts is a new stream. Created visible, it announces now and answers the waiting requesters; created ``hidden``, they wait until :meth:`Route.announce`, so the job can fill its catalog first, and cancelling the route before then refuses them. Ending the diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index f85c26f736e..5e57d60a8ad 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -824,9 +824,9 @@ async def test_dynamic_broadcast_request(): assert request is not None assert request.path == "dynamic/broadcast" - # A hidden job route answers once it announces, under its own instance. + # A hidden job route answers once it announces, under the instance its prefix route lends. served = request.route(hidden=True) - assert served.instance != dynamic.instance + assert served.instance == dynamic.instance track = served.broadcast.publish_track("status", moq.TrackInfo(timescale=1_000_000)) served.announce() with pytest.raises(Exception): diff --git a/quest/m1/api-audit/route/README.md b/quest/m1/api-audit/route/README.md index 8b01bb75f62..14f069b09a5 100644 --- a/quest/m1/api-audit/route/README.md +++ b/quest/m1/api-audit/route/README.md @@ -18,7 +18,7 @@ mirror it. Non-goals: the route layer's wire messages and federation ([Routes and announces](/quest/m2/cluster-routing/routes/README.md)), lite-07 -takeover ([announce takeover](/quest/m2/announce-takeover.md)), and any +takeover (removed in [moq-dev/moq#8063](https://github.com/moq-dev/moq/pull/8063)), and any moq-transport instance ([moq-transport carries no epoch](https://github.com/moq-dev/moq/pull/7239)). ## Plan @@ -144,8 +144,8 @@ supersede the epoch-minting, pairing and run-uniqueness answers above: predecessor-ID plan. The predecessor design then landed on main in [moq-dev/moq#6994](https://github.com/moq-dev/moq/pull/6994). On - 2026-10-31 the maintainer kept identity: [announce takeover](/quest/m2/announce-takeover.md) - now also removes that work, so only one continuity rule remains. + 2026-10-31 the maintainer kept identity: [moq-dev/moq#8063](https://github.com/moq-dev/moq/pull/8063) + removes that work, so only one continuity rule remains. Third round, 2026-11-01, answering [moq-dev/moq#7489](https://github.com/moq-dev/moq/pull/7489)'s open decisions: @@ -171,5 +171,4 @@ stale, including `doc/concept/identity.md`. ## Related -- [A job's exact route continues its claim's readers](/quest/m2/announce-takeover.md) - a job's route inherits its claim's instance, so lite-07 readers keep the first subscribe - [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the wire route layer; instances take over its epochless deletion diff --git a/quest/m2/README.md b/quest/m2/README.md index a04de05388e..554a6d1a255 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -175,7 +175,6 @@ quests that require them. - [Address groups by timestamp](/quest/m2/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate (held in m1 by an open claim) - [Cluster routing](/quest/m2/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters -- [A job's exact route continues its claim's readers](/quest/m2/announce-takeover.md) - a job's route inherits its claim's instance, replacing the ANNOUNCE_TAKEOVER predecessor design - [A group a subscriber gives up on is always reported](/quest/m2/group-drops.md) - the wire names every abandoned group and the drop signal is public, so readers count losses instead of stalling - [Datagram tracks](/quest/m1/datagram-tracks/README.md) - a track is datagrams or streams; datagram tracks have their own request and sequence, no ranges, no cache, and map to object ID 0 on moq-transport (held in m1 by an open claim) - [Stream credit by priority](/quest/m1/stream-credit/README.md) - every stream opens through one priority entry point, and a higher-priority group takes over or resets a lower one when credit runs out (held in m1 by an open claim) diff --git a/quest/m2/announce-takeover.md b/quest/m2/announce-takeover.md deleted file mode 100644 index 5323a2d5111..00000000000 --- a/quest/m2/announce-takeover.md +++ /dev/null @@ -1,159 +0,0 @@ -# [XL] A job's exact route continues its claim's readers - -## Goal - -A worker advertises a prefix route to accept lazy work. When a request -starts a job, the job's exact route carries the prefix route's instance, so -a reader first served through the claim continues on the exact route with -no cold-start reset, across relays, on lite-07. A different worker, or a -restarted job, never continues those readers: each is another instance. -Readers holding `follow` or `request` simply continue, and see a new -broadcast only when the instance changes. Rust and JS players keep playing -through a claim. - -The ANNOUNCE_TAKEOVER work that landed in -[moq-dev/moq#6994](https://github.com/moq-dev/moq/pull/6994) is removed: the -lite-07 ANNOUNCE_TAKEOVER message (0x5), the origin, front and relay -handling of predecessors, the `Takeover { announce, predecessor }` event, the -predecessor fields in moq-ffi and the C ABI, and the takeover text in the -lite draft and the concept docs. No wire message or public event replaces it. - -Non-goals: moq-transport, which keeps the cold-start reset -(moq-transport carries no epoch (decided 2026-10-28, removed in -[moq-dev/moq#7239](https://github.com/moq-dev/moq/pull/7239)), and an -edge mints a new instance for each announcement, -[Instance and epoch](https://github.com/moq-dev/moq/pull/7489)). - -## Plan - -Re-planned 2026-10-30 (maintainer, the [route epic](/quest/m1/api-audit/route/README.md)): -continuity is the instance promise, not an `ANNOUNCE_TAKEOVER` predecessor -ID. An instance is one continuous sequence space at a path, so the exact -route and the claim's answer are the same consumer route, -origin → route(path, instance) → broadcast. The 2026-10-10 plan (TAKEOVER as -START plus a predecessor Announce ID, with per-stream ID translation in -relays) is dropped. Its safety requirements carry over, as follows. - -Decided 2026-10-31 (maintainer, merging main into -[moq-dev/moq#7420](https://github.com/moq-dev/moq/pull/7420)): instance -wins over the predecessor design that landed in -[moq-dev/moq#6994](https://github.com/moq-dev/moq/pull/6994) meanwhile. -Keeping both would give two continuity rules, one by instance and one by -predecessor prefix, that every origin, relay and binding must agree on. The -instance rule already covers the claim case and needs no wire message, so -this quest removes the predecessor machinery and grows to [XL]. - -- **Removing ANNOUNCE_TAKEOVER.** Delete the 0x5 message and its codec in - `rs/moq-net/src/lite/announce.rs`, `compress.rs`, `publisher/routed.rs`, - `subscriber/routed.rs` and `replay.rs`; the route layer's claim lineage - (`Claim::continues` in `rs/moq-net/src/route/node.rs`, fed by `OwnClaims` in - `origin.rs`); the predecessor tracking in - `rs/moq-net/src/model/origin.rs` and `front.rs` (the `seen` stamps in - `OriginConsumerState` and `PendingFront::via`); relay forwarding in - `rs/moq-relay`; and `rs/moq-net/tests/integration/announce_takeover.rs`. - Remove `RouteEvent::Takeover` from the relay-level `routes()` stream - (`rs/moq-net/src/model/origin.rs`, and its folds in `follow.rs`, - `announced.rs` and the session publishers); the app listing, `MoqAnnounceEvent` - and `moq_announce_event` dropped it already in - [moq-dev/moq#7678](https://github.com/moq-dev/moq/pull/7678). In `js/net`, drop the ANNOUNCE_TAKEOVER decoding in - `lite/announce.ts` and `lite/subscriber.ts`. Remove the takeover sections - from `drafts/draft-lcurley-moq-lite.md`, `doc/concept/identity.md` and - `doc/concept/moq-lite.md`. -- **Inheriting the instance.** Decided 2026-11-03 (maintainer, on - [moq-dev/moq#7770](https://github.com/moq-dev/moq/pull/7770)): - `request.route(options)`, from - [One Route handle](https://github.com/moq-dev/moq/pull/7806), mints a fresh - instance by default, and ending a job leaves its prefix route open, with - no history of ended paths. Carrying the prefix route's instance onto the - path's exact route, so that the answer the waiting request gets is that - route's broadcast, is this quest's change, and it must keep those - properties: a restarted job still gets a fresh instance, and job - lifetimes stay independent without per-path history of ended jobs. - A job that serves a held broadcast through `Request::serve` - never announces an exact route, so it needs none of this. -- **One stream per path and instance.** A claim may answer a path under - its instance only while that stream is live. Once the job's route ends, - the next request for that path gets a fresh instance, because its - sequences restart. A request pinned to the old instance is refused as - ended, never answered with new bytes. This is the old rule "a worker - restarting a job under an unchanged claim cannot relabel the previous - job's cached content", now enforced in the origin. Prove it in the - model, including retained generations and pending answers, before - relying on it. -- **Relays.** An aggregated prefix can be served by several workers, but - each worker has its own instance. A relay can therefore only continue a - reader on routes carrying the instance it resolved, and readers of A - never move to B. A relay that cannot keep the instance, such as an - older-lite hop, mints a new one, so its readers see a new broadcast - rather than a splice. -- **What a continued reader keeps.** Handles, group positions, metadata, - object and negative caches, catalog pins, and outstanding - TRACK/SUBSCRIBE/FETCH requests. Unrelated sibling paths are unchanged. A - continued stream does not renew a composed catalog. -- **JS players.** `js/net`'s lite subscriber continues a consumer resolved - through the claim on the exact route with the same instance instead of - invalidating it, in `BroadcastCache` and `connection/forward.ts`, so a - `@moq/watch` reader through a claim keeps playing. -- **Epoch-bound requests through a continued route.** A front or selection - that continues on the exact route keeps the upstream SUBSCRIBE it already - sent under the claim's route; a later request naming the exact route's - epoch joins it without restamping. This is accepted while the answering - worker is the same instance by construction. Decide whether that join - needs the upstream's answer to confirm the epoch. -- **Docs.** Update the lazy-job lifecycle in `doc/concept/identity.md`: - the cold-start step no longer resets on lite-07. Also update the lite - draft's announce section where it mentions claims. - -Tests with controlled time cover: -- the first reader before and after the exact route lands, with no reader - at all, and with two workers at one prefix; -- a job restarting on one worker, and a request pinned to an ended stream; -- pending TRACK/SUBSCRIBE/FETCH at the front between its resolve and the - exact route landing, and late groups across the transition; -- conflicting epochs within one instance; -- independently ending prefix and exact routes (a claim ending before or - after its child), and late snapshots; -- multi-hop and diamond relays, and mixed versions with an older-lite hop; -- held catalog and media pins through a real player rather than route-table - checks, and the composer. - -Remove the cold-start warm-up reads (`claimed` in -`rs/moq-net-tests/tests/integration/restart.rs` and the -`rs/moq-net/tests/integration/drained_claim.rs` helper) for lite-07 and -assert that the first subscription succeeds. Keep them for moq-transport. -Run `just check`, `just test interop --all` and `just drafts check`. - -### Once the instance replaces ANNOUNCE_TAKEOVER - -Decided 2026-10-31 (maintainer, the /quest-complete requeue session). Kept -separate from the identity rewrite of this Plan, which landed with -[moq-dev/moq#7420](https://github.com/moq-dev/moq/pull/7420). -"We're replacing EPOCH with IDENTITY soon" (maintainer, 2026-10-31). The -broadcast "identity" was renamed `instance` on 2026-11-01 (maintainer, in -[the route API audit](/quest/m1/api-audit/route/README.md), landed in -[moq-dev/moq#7489](https://github.com/moq-dev/moq/pull/7489)), so that -IDENTITY is now the instance. - -- Reword "takeover" as an event or operation, in terms of instance - continuity, in [moq-bench pins the instances it measures](/quest/m4/bench-epoch-pin.md) - (Goal and Plan) and [the HLS server follows each path once](/quest/m4/hls-follow-once.md) - (Goal and Tests). -- Rewrite the EPOCH references in `drafts/draft-lcurley-moq-cluster.md` and - `drafts/draft-lcurley-moq-epoch.md` in terms of the instance. -- Add a regression test where a publisher restart's new announcement - arrives before the old one's reset, on `solicit = false` moq-transport - sessions. [moq-dev/moq#7239](https://github.com/moq-dev/moq/pull/7239) - found that the subscriber (`rs/moq-net/src/ietf/subscriber.rs`, `attach`) - folds such a PUBLISH_NAMESPACE into the same epochless route as an update - instead of End then Start; per-announcement instances resolve it, and - the test proves they do. - -Public API: breaking; removes `Takeover` and the predecessor fields in every -language. Wire: removes lite-07's ANNOUNCE_TAKEOVER (0x5), adding nothing -beyond [Instance and epoch](https://github.com/moq-dev/moq/pull/7489)'s fields. - -## Related - -- [Claim answer epochs](https://github.com/moq-dev/moq/pull/5678) - why a restarted job must never be stitched to its predecessor -- [Retired requests](https://github.com/moq-dev/moq/pull/5294) - replacements never substitute content into held handles -- [Routes and announces](/quest/m2/cluster-routing/routes/README.md) - the route layer whose ANNOUNCE_START layout carries the instance diff --git a/quest/m2/cluster-routing/routes/README.md b/quest/m2/cluster-routing/routes/README.md index 76724476ea5..be3b668a0fd 100644 --- a/quest/m2/cluster-routing/routes/README.md +++ b/quest/m2/cluster-routing/routes/README.md @@ -20,7 +20,7 @@ claim whose withdrawal is already known. 5. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Floor regions have no order, and a repeated region is a violation.** A sorted order would need Rust's `Path` ordering reproduced exactly in JS. Decoding into a map rejects duplicates at the same cost. Superseded the same day by item 10, which removes regions. 6. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **The Hop setup parameter and ANNOUNCE_OK's Hop ID are dropped from lite-07.** Their only roles were path-vector exclusion and the implicit trailing hop. ANNOUNCE_OK reserves no body for a route-key proof: [moq-dev/moq#7937](https://github.com/moq-dev/moq/pull/7937), accepted the same day, proves the route key once per session in SETUP. The live hop-based codec still switches in session integration, as choice 1 requires. 7. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **An echo naming a Route ID that was never ended, or was already echoed, is a violation at the publisher.** The publisher can check this from its live and unacknowledged ids alone, with no growing state. -8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [quest/m2/announce-takeover.md](https://github.com/moq-dev/moq/blob/main/quest/m2/announce-takeover.md) deletes the message, so this keeps the change minimal. Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether session integration ports the message or lets lite-07 answer a claim as a restart or start until announce-takeover lands: ✅ **port it**, so a job answering a claim stays a takeover across relays. Rejected: accepting the restart and start. +8. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_TAKEOVER keeps its shape, with a Route ID in place of hops, and its predecessor must be under the same route.** [moq-dev/moq#8063](https://github.com/moq-dev/moq/pull/8063) deletes the message, so this keeps the change minimal. Decided 2026-11-06 (maintainer, /quest-complete on [moq-dev/moq#8023](https://github.com/moq-dev/moq/pull/8023)), asked whether session integration ports the message or lets lite-07 answer a claim as a restart or start until announce-takeover lands: ✅ **port it**, so a job answering a claim stays a takeover across relays. Rejected: accepting the restart and start. 9. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Unknown announce types stay skipped by length on lite-06+**, as before. An unknown reply type is refused. 10. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **ANNOUNCE_FLOOR is a marker, not a region list:** `{Route ID, Revision}`. It closes a batch and certifies the stream's claims for that route across the whole requested prefix as the origin's at that Revision. A publisher that cannot vouch for the whole requested prefix at one Revision still closes the batch, with a floor that certifies nothing, encoded per item 13. Regions, the replacing list, and the 65,535-byte overflow go away. Accepted cost: a relay with partial knowledge of an origin certifies nothing for that route, so a receiver lets its stale claims go only on ROUTE_END. Rejected: one region per message; a multipart atomic list. 11. ✅ Decided 2026-11-05 (maintainer, /quest-plan): **Clamped claims collapse to the best one.** When several of a route's claims cover a request's prefix (`live` at cost 10 and `live/a` at cost 1, request `live/a/x`), the publisher presents only the most specific, and when it is withdrawn moves the presented claim to the next covering one with ANNOUNCE_UPDATE or ANNOUNCE_RESTART instead of ending it. No wire change; one live claim per prefix per route holds, and a receiver merges claims at the prefix each stream presents them. Rejected: carrying each claim's ancestor depth. diff --git a/quest/m2/lite07-finalize.md b/quest/m2/lite07-finalize.md index c8099055a51..9d7f882a624 100644 --- a/quest/m2/lite07-finalize.md +++ b/quest/m2/lite07-finalize.md @@ -114,7 +114,6 @@ timestamp bound in [#6168](https://github.com/moq-dev/moq/pull/6168). - [Lite-07 FETCH is single-group only](/quest/m2/subscribe-ranges/lite07-fetch.md) - lite-07 FETCH reads one group or a slice of one, and SUBSCRIBE ranges of whole groups - [Lite-07 by start](/quest/m2/time-addressing/lite07.md) - lite-07 carries group starts, previous-start links, GROUP_END, and time bounds instead of sequence numbers - [@moq/net by start](/quest/m2/time-addressing/js.md) - JS speaks the same lite-07 wire -- [A job's exact route continues its claim's readers](/quest/m2/announce-takeover.md) - the job's route inherits the claim's instance, with no new wire message - [Takeover](/quest/m1/stream-credit/takeover.md) - lite-07 lets a group stream take over for another group - [Datagram tracks](/quest/m1/datagram-tracks/README.md) - lite-07 subscribes to datagram tracks through their own request, and datagrams carry their own sequence diff --git a/quest/m3/release/runner-client-bump.md b/quest/m3/release/runner-client-bump.md index 16faea6b5d0..f2afee1b942 100644 --- a/quest/m3/release/runner-client-bump.md +++ b/quest/m3/release/runner-client-bump.md @@ -27,9 +27,9 @@ refuses timestamped frames, `Subscription` takes `max_delay` instead of a max age, announcement events are `Start`, `Update`, `Restart`, and `End`, and `track::Producer::used` is gone. Through moq-relay, the first subscription under `data-subgroup-basic`'s dynamic claim then fails -`unroutable`: the accepted cold-start reset that -[announce takeover](/quest/m2/announce-takeover.md) removes, so check -whether that has landed before calling it a client bug. The JS client's +`unroutable`: the accepted cold-start reset, which moq-transport keeps +([moq-dev/moq#8063](https://github.com/moq-dev/moq/pull/8063) removes it on +lite-07 only), so it is no client bug. The JS client's announcement check also moves from `"retracted"` to `"end"` when it takes the cut. Its `data-subgroup-basic` subscriber also stalls there: on `main`, `follow(path)` asks SUBSCRIBE_NAMESPACE for the diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index d8d438b7c50..a190e85d0ef 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -2224,7 +2224,8 @@ pub unsafe extern "C" fn moq_broadcast_request_path(request: u32, dst: *mut moq_ /// Serve a held consumer under the prefix route's instance without announcing an exact route. /// /// Every answer at this path must continue the same sequences and bytes. Start a new -/// prefix route for different content, or route a job with its own instance. +/// prefix route for different content, or route a job, whose end moves the prefix route to +/// another instance. /// Consumes the request handle; the consumer remains owned by the caller. /// Returns zero on success, or a negative error. #[unsafe(no_mangle)] @@ -2245,8 +2246,10 @@ pub extern "C" fn moq_broadcast_request_serve(request: u32, broadcast: u32) -> i /// Answer a broadcast request with the path's own exact route, owning a new broadcast. /// /// Consumes the request handle and returns a positive route handle, or a negative error. -/// The route serves a fresh instance unless `options` names one, so a job that restarts is -/// a new stream. Created visible, it announces now and answers the waiting requesters; +/// Unless `options` names an instance, the route takes the one its prefix route lends when +/// it first announces, so readers that resolved the path through the prefix route continue +/// on it; its end moves the prefix route to a fresh instance, so a job that restarts is a +/// new stream. Created visible, it announces now and answers the waiting requesters; /// created hidden, they wait until [moq_route_announce], so the job can fill its catalog /// first, and freeing the route before then refuses them. Ending the route withdraws only /// this path. Unreadable `options` fail without consuming the request. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 97f72e72e86..67e8baa648b 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -4110,13 +4110,14 @@ fn dynamic_serves_a_request_under_a_prefix() { crate::error::MOQ_ERROR_UTF8 ); - // A hidden job route answers nobody until it announces, under its own fresh instance. + // A hidden job route answers nobody until it announces, under the instance its prefix + // route lends. let job = id(unsafe { moq_broadcast_request_route(request, &hidden()) }); assert!( unsafe { moq_broadcast_request_route(request, std::ptr::null()) } < 0, "request is consumed" ); - assert_ne!(instance_of(job), instance_of(dynamic)); + assert_eq!(instance_of(job), instance_of(dynamic)); let served = id(moq_route_broadcast(job)); assert_eq!(moq_route_announce(job), 0); assert!(req_cb.recv() > 0); diff --git a/rs/moq-cli/tests/integration/export.rs b/rs/moq-cli/tests/integration/export.rs index 0181e48630c..18ce1e0593c 100644 --- a/rs/moq-cli/tests/integration/export.rs +++ b/rs/moq-cli/tests/integration/export.rs @@ -1064,9 +1064,7 @@ async fn next_instance(announced: &mut moq_net::origin::Routes) -> moq_net::Inst kio::clock::timeout(TIMEOUT, async { loop { match announced.next().await.expect("the relay stays up") { - moq_net::origin::RouteEvent::Start(announce) - | moq_net::origin::RouteEvent::Restart(announce) - | moq_net::origin::RouteEvent::Takeover { announce, .. } => { + moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Restart(announce) => { return announce.route.instance().clone(); } _ => continue, diff --git a/rs/moq-cli/tests/integration/rtsp.rs b/rs/moq-cli/tests/integration/rtsp.rs index aa31f27363b..f94116ac6cb 100644 --- a/rs/moq-cli/tests/integration/rtsp.rs +++ b/rs/moq-cli/tests/integration/rtsp.rs @@ -63,9 +63,7 @@ async fn next_broadcast( .await .expect("origin closed") { - moq_net::origin::RouteEvent::Start(_) - | moq_net::origin::RouteEvent::Restart(_) - | moq_net::origin::RouteEvent::Takeover { .. } => break, + moq_net::origin::RouteEvent::Start(_) | moq_net::origin::RouteEvent::Restart(_) => break, moq_net::origin::RouteEvent::End(_) | moq_net::origin::RouteEvent::Update(_) => {} } } diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 8ffe5fc59c8..3d65f41e7ff 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -553,8 +553,10 @@ impl MoqBroadcastRequest { /// Answer with the path's own exact route, owning a new broadcast. /// - /// The route serves a fresh instance unless `options` names one, so a job that restarts - /// is a new stream. Created visible, it announces now and answers the waiting + /// Unless `options` names an instance, the route takes the one its prefix route lends + /// when it first announces, so readers that resolved the path through the prefix route + /// continue on it; its end moves the prefix route to a fresh instance, so a job that + /// restarts is a new stream. Created visible, it announces now and answers the waiting /// requesters; created hidden, they wait until [`MoqRoute::announce`], so the job can /// fill its catalog first, and cancelling the route before then refuses them. Ending the /// route withdraws only this path. @@ -571,7 +573,8 @@ impl MoqBroadcastRequest { /// Serve a held broadcast under the prefix route's instance without announcing an exact route. /// /// Every answer at this path must continue the same sequences and bytes. Start a new - /// prefix route for different content, or route a job with its own instance. + /// prefix route for different content, or route a job, whose end moves the prefix route + /// to another instance. pub fn serve(&self, broadcast: &Arc) -> Result<(), MoqError> { let _guard = crate::ffi::enter(); self.driver.check()?; diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 0105f7ef25f..c6270c408db 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -4794,9 +4794,7 @@ mod session_tests { await_event(&mut follow, |event| { matches!( event, - moq_net::origin::RouteEvent::Restart(_) - | moq_net::origin::RouteEvent::Takeover { .. } - | moq_net::origin::RouteEvent::Start(_) + moq_net::origin::RouteEvent::Restart(_) | moq_net::origin::RouteEvent::Start(_) ) }); diff --git a/rs/moq-net-tests/tests/integration/restart.rs b/rs/moq-net-tests/tests/integration/restart.rs index dd81635286c..265d53022ea 100644 --- a/rs/moq-net-tests/tests/integration/restart.rs +++ b/rs/moq-net-tests/tests/integration/restart.rs @@ -643,7 +643,8 @@ async fn a_more_specific_source_cannot_fill_a_held_covering_track() { } /// A worker claiming `pool` through the public API: each request is answered with its own -/// exact route under a fresh instance, and `outputs[path]` holds the latest. +/// exact route, which carries the claim's instance until it ends, and `outputs[path]` +/// holds the latest. struct Pool { _claim: Rc, outputs: Rc>>, @@ -683,15 +684,17 @@ impl Pool { } } -/// Wake the worker for `path` through the claim and subscribe. Every wire carries the -/// answer as an ordinary announcement, which resets the first subscription through the -/// claim, so wake the worker first and subscribe through the exact route. Lite-07's -/// takeover is gone until quest/m2/announce-takeover.md carries the claim's instance onto -/// the exact route. -async fn claimed(origin: &origin::Producer, path: &str) -> (broadcast::Consumer, track::Subscriber) { - let cold = origin.consume().request(path, None).await.unwrap(); - drop(cold.track("video").unwrap().subscribe(None).await); - settle().await; +/// Subscribe to `path` through the claim. On moq-lite-07 the answer's exact route carries +/// the claim's instance, so the first subscription continues on it. Older wires and +/// moq-transport mint an instance per announcement, so the answer is another broadcast that +/// resets the first subscription: wake the worker first and subscribe through the exact +/// route. +async fn claimed(version: Version, origin: &origin::Producer, path: &str) -> (broadcast::Consumer, track::Subscriber) { + if version != "moq-lite-07-wip".parse().unwrap() { + let cold = origin.consume().request(path, None).await.unwrap(); + drop(cold.track("video").unwrap().subscribe(None).await); + settle().await; + } subscribe(origin, path).await } @@ -708,11 +711,11 @@ async fn claim_restart(version: &str) { let pool = Pool::new(&worker); settle().await; - let (_sibling, mut sibling) = claimed(&edge, "pool/q").await; + let (_sibling, mut sibling) = claimed(version, &edge, "pool/q").await; pool.write("pool/q", "q:0"); assert_eq!(read(&mut sibling).await, "q:0", "{version}"); - let (old, mut first) = claimed(&edge, "pool/p").await; + let (old, mut first) = claimed(version, &edge, "pool/p").await; for group in 0..3 { pool.write("pool/p", &format!("old:{group}")); assert_eq!(read(&mut first).await, format!("old:{group}"), "{version}"); @@ -721,7 +724,7 @@ async fn claim_restart(version: &str) { pool.close("pool/p"); settle().await; - let (fresh, mut returning) = claimed(&edge, "pool/p").await; + let (fresh, mut returning) = claimed(version, &edge, "pool/p").await; assert!(!fresh.is_clone(&old), "{version}: joined the old output"); assert_eq!(*pool.answered.borrow(), 3, "{version}: the worker was not asked again"); pool.write("pool/p", "new:0"); diff --git a/rs/moq-net/benches/announce.rs b/rs/moq-net/benches/announce.rs index f567100d78b..8df7c8c7e00 100644 --- a/rs/moq-net/benches/announce.rs +++ b/rs/moq-net/benches/announce.rs @@ -36,8 +36,7 @@ fn scanned(events: Vec) -> Vec { let (update, active) = match event { origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update) - | origin::RouteEvent::Restart(update) - | origin::RouteEvent::Takeover { announce: update, .. } => (update, true), + | origin::RouteEvent::Restart(update) => (update, true), origin::RouteEvent::End(update) => (update, false), }; initial.retain(|held| held.prefix != update.prefix); diff --git a/rs/moq-net/benches/auth.rs b/rs/moq-net/benches/auth.rs index 62ae2cdcd9d..dc116d77391 100644 --- a/rs/moq-net/benches/auth.rs +++ b/rs/moq-net/benches/auth.rs @@ -307,7 +307,6 @@ fn waiting(c: &mut Criterion) { moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) | moq_net::origin::RouteEvent::Restart(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. } if update.prefix.as_str().starts_with(&prefix) => { seen.insert(update.prefix); diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 73bca373821..1e0840db2b2 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -29,9 +29,8 @@ //! Readers draining a swapped track are swept over their count, the groups every //! route delivered, and the routes each reader still holds a copy of. //! -//! A claim's answer taking over the claim is swept over the paths the claim already -//! answered and the cursors watching it, and a takeover in place over the fronts -//! resolved through what it takes over. +//! A claim's answer, and its retraction moving the claim to a fresh instance, is swept +//! over the paths the claim already answered and the cursors watching it. //! //! Peer claims land through the hidden `fuzz` module, as a session lands them. //! @@ -676,12 +675,12 @@ fn answer(claim: &origin::Route, path: &str) -> origin::Route { } } -/// A claim's answer delivered to every cursor as a takeover of the claim, then retracted. -/// Swept over the paths the claim already answered and the cursors watching it, so a -/// continuity check that walks the table rather than the claim's prefix shows up as a -/// slope. -fn bench_takeover(c: &mut Criterion) { - let mut group = c.benchmark_group("origin/takeover"); +/// A claim's answer delivered to every cursor as a start, then retracted, which moves the +/// claim to a fresh instance. Swept over the paths the claim already answered and the +/// cursors watching it, so finding the lending claim by walking the table rather than the +/// path's covering routes shows up as a slope. +fn bench_lend(c: &mut Criterion) { + let mut group = c.benchmark_group("origin/lend"); for (publishers, subscribers) in SHAPES { group.bench_function( BenchmarkId::from_parameter(format!("{publishers}p_{subscribers}s")), @@ -708,12 +707,17 @@ fn bench_takeover(c: &mut Criterion) { let (request, output) = ask(&format!("room/incoming{n}")); for cursor in &mut cursors { let event = cursor.next().now_or_never().flatten(); - assert!(matches!(event, Some(origin::RouteEvent::Takeover { .. })), "{event:?}"); + assert!(matches!(event, Some(origin::RouteEvent::Start(_))), "{event:?}"); } drop(request); drop(output); for cursor in &mut cursors { cursor.next().now_or_never().flatten().expect("retract delivered"); + cursor + .next() + .now_or_never() + .flatten() + .expect("the claim's advance delivered"); } }); }, @@ -984,43 +988,6 @@ fn bench_restart_fronts(c: &mut Criterion) { group.finish(); } -/// A pool member taken over in place and retired, under `fronts` fronts resolved -/// through it, against the same member restarting without a takeover. Each change -/// pokes every front beneath the prefix, and the takeover carries each one's verdict -/// forward under the table lock, so the takeover's cost over the restart is that pass. -/// The fronts' tasks never run: this is the install alone. -fn bench_takeover_fronts(c: &mut Criterion) { - let mut group = c.benchmark_group("origin/takeover_fronts"); - for fronts in [100, 1_000, 10_000] { - for takeover in [false, true] { - let kind = if takeover { "takeover" } else { "restart" }; - group.bench_function(format!("{fronts}f_{kind}"), |b| { - let mut pool = pool(1, fronts, 0); - let mut last = pool._members.pop().expect("one member"); - let mut hop = POOL_HOP + 1; - b.iter(|| { - hop += 1; - let next = match takeover { - true => moq_net::fuzz::takeover_route( - &pool.producer, - moq_net::Path::from_static("pool"), - pool_route(hop), - &last, - ), - false => moq_net::fuzz::requests_route( - &pool.producer, - moq_net::Path::from_static("pool"), - pool_route(hop), - ), - }; - drop(std::mem::replace(&mut last, next.unwrap())); - }); - }); - } - } - group.finish(); -} - /// A member joining and leaving a pool that already serves `paths` paths. /// /// Every front under the prefix watches the routes covering it, so each one @@ -1861,14 +1828,13 @@ criterion_group!( bench_announce_duplicate, bench_reprice_duplicate, bench_announce_fronts, - bench_takeover, + bench_lend, bench_serve_idle, bench_subscribe, bench_request, bench_retire, bench_restart_fronts, bench_pool_churn, - bench_takeover_fronts, bench_pool_cursors, bench_handoff, bench_narrow_delivery, diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index a3c8d81b274..c09d87379a3 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -461,18 +461,7 @@ pub fn requests_route( prefix: impl Into, route: crate::origin::RouteInfo, ) -> Result { - origin.forward(prefix, route, None) -} - -/// Land a claim taking over the peer's `predecessor` claim, as a session lands an -/// ANNOUNCE_TAKEOVER, for the origin bench. -pub fn takeover_route( - origin: &crate::origin::Producer, - prefix: impl Into, - route: crate::origin::RouteInfo, - predecessor: &crate::origin::Route, -) -> Result { - origin.forward(prefix, route, Some(predecessor)) + origin.forward(prefix, route) } /// Replace a peer claim's route in place, as a session does on a route update. diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 4c17e30bbd6..5cfd110a9b5 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -4111,8 +4111,7 @@ impl Publisher { match event { crate::origin::RouteEvent::Start(update) | crate::origin::RouteEvent::Update(update) - | crate::origin::RouteEvent::Restart(update) - | crate::origin::RouteEvent::Takeover { announce: update, .. } => { + | crate::origin::RouteEvent::Restart(update) => { initial.insert(update.prefix.clone(), update); } crate::origin::RouteEvent::End(update) => { @@ -5881,11 +5880,6 @@ impl ForwardNamespaces { (update, true, false) } crate::origin::RouteEvent::Restart(update) => (update, true, true), - // moq-transport has no takeover: another instance, in place or not. - crate::origin::RouteEvent::Takeover { announce, predecessor } => { - let restart = predecessor == announce.prefix; - (announce, true, restart) - } crate::origin::RouteEvent::End(update) => (update, false, false), }))); } @@ -11359,10 +11353,7 @@ mod serve_tests { for version in ALL_VERSIONS { let mut h = serve(version); - let claim = h - ._origin - .forward("live", crate::origin::RouteInfo::default(), None) - .unwrap(); + let claim = h._origin.forward("live", crate::origin::RouteInfo::default()).unwrap(); let auth = crate::auth::Link::new(false); auth.handle().authorize(¬hing()); h.publisher = h.publisher.clone().with_auth(auth); @@ -11382,10 +11373,7 @@ mod serve_tests { for version in ALL_VERSIONS { let mut h = serve(version); - let claim = h - ._origin - .forward("live", crate::origin::RouteInfo::default(), None) - .unwrap(); + let claim = h._origin.forward("live", crate::origin::RouteInfo::default()).unwrap(); let auth = crate::auth::Link::new(false); h.publisher = h.publisher.clone().with_auth(auth.clone()); @@ -11702,10 +11690,7 @@ mod serve_tests { let version = Version::Draft22; for path in [".", "./child", ".session", ".session/extension"] { let h = serve(version); - let claim = h - ._origin - .forward("", crate::origin::RouteInfo::default(), None) - .unwrap(); + let claim = h._origin.forward("", crate::origin::RouteInfo::default()).unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) .await @@ -13851,11 +13836,7 @@ mod serve_tests { { *namespace = crate::Path::from_static("lobby"); } - Some( - h._origin - .forward("lobby", crate::origin::RouteInfo::default(), None) - .unwrap(), - ) + Some(h._origin.forward("lobby", crate::origin::RouteInfo::default()).unwrap()) } FetchParked::Responding => None, }; @@ -15463,11 +15444,7 @@ mod serve_tests { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let (claim, published) = match parked { Parked::Resolving => ( - Some( - origin - .forward("room", crate::origin::RouteInfo::default(), None) - .unwrap(), - ), + Some(origin.forward("room", crate::origin::RouteInfo::default()).unwrap()), None, ), Parked::Acking | Parked::Ending => { @@ -15704,9 +15681,7 @@ mod serve_tests { let mut idle = None; match setup { Setup::Resolving => { - let route = origin - .forward("room", crate::origin::RouteInfo::default(), None) - .unwrap(); + let route = origin.forward("room", crate::origin::RouteInfo::default()).unwrap(); let request = { let requested = std::pin::pin!(route.requested()); match futures::future::select(requested, serving.as_mut()).await { diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 09063a00e51..ec77bcb51ea 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -2826,7 +2826,7 @@ impl Subscriber { if self.going_away.is_set() { route.cost = crate::origin::Cost::DRAIN; } - entry.dynamic = Some(self.origin.forward(path, route, None)?); + entry.dynamic = Some(self.origin.forward(path, route)?); entry.generation += 1; tracing::info!(route = %crate::path::Rooted(self.origin.root(), path), "namespace authorized again"); Ok(Some(entry.generation)) @@ -2919,7 +2919,7 @@ impl Subscriber { return Ok(()); } // Propagates Error::Unauthorized if the namespace is out of scope. - let dynamic = self.origin.forward(&path, route.clone(), None)?; + let dynamic = self.origin.forward(&path, route.clone())?; entry.insert(BroadcastState { route: route.clone(), @@ -10793,8 +10793,7 @@ mod tests { crate::origin::RouteEvent::End(_) => None, crate::origin::RouteEvent::Start(announce) | crate::origin::RouteEvent::Update(announce) - | crate::origin::RouteEvent::Restart(announce) - | crate::origin::RouteEvent::Takeover { announce, .. } => Some(announce.route), + | crate::origin::RouteEvent::Restart(announce) => Some(announce.route), } } diff --git a/rs/moq-net/src/lite/announce.rs b/rs/moq-net/src/lite/announce.rs index be5f324ad66..417a7524c02 100644 --- a/rs/moq-net/src/lite/announce.rs +++ b/rs/moq-net/src/lite/announce.rs @@ -21,8 +21,6 @@ const ANNOUNCE_UPDATE: u64 = 2; // lite-07: the advertisement serves another instance, or a newer epoch of its own. const ANNOUNCE_RESTART: u64 = 3; const SYNCED: u64 = 4; -// lite-07: a new advertisement continues what a live one served beneath its prefix. -const ANNOUNCE_TAKEOVER: u64 = 5; // lite-07: the route layer. ROUTE_END also travels back as the echo. const ROUTE_START: u64 = 6; const ROUTE_UPDATE: u64 = 7; @@ -47,10 +45,9 @@ pub fn update_supported(version: Version) -> bool { /// An announcement on the Announce Stream, advertising or retracting a broadcast. /// /// On lite-06+ these are independently-typed messages (`ANNOUNCE_START`, -/// `ANNOUNCE_END`, `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART` and -/// `ANNOUNCE_TAKEOVER`), each +/// `ANNOUNCE_END`, `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`), each /// framed as `Type | Length | Body` like the subscribe stream's responses. Each -/// `Active` (ANNOUNCE_START) or `Takeover` implicitly assigns the next announce id (a +/// `Active` (ANNOUNCE_START) implicitly assigns the next announce id (a /// per-stream ordinal starting at 0); the others reference that id instead of /// repeating the path. Older versions send a single `ANNOUNCE_BROADCAST` /// message that retracts by path (`Ended`). @@ -92,17 +89,6 @@ pub enum AnnounceBroadcast { epoch: u64, cost: Cost, }, - /// ANNOUNCE_TAKEOVER (lite-07+): an `Active` that continues what the live - /// announcement `predecessor` served beneath this one's path. It assigns the next - /// announce id; one at the predecessor's own path retires the predecessor. - Takeover { - predecessor: u64, - suffix: PathRef, - route: u64, - instance: Option, - epoch: u64, - cost: Cost, - }, /// An unknown lite-06+ announce type. The length-prefixed body was skipped so /// the stream stays up; it does not assign an announce id. Skipped, @@ -292,9 +278,6 @@ impl Encode for AnnounceBroadcast { Self::Restart { .. } if version.has_announce_restart() => ANNOUNCE_RESTART, // Older versions send an end and a start instead. Self::Restart { .. } => return Err(EncodeError::Version), - Self::Takeover { .. } if version.has_announce_takeover() => ANNOUNCE_TAKEOVER, - // Older versions send a start instead. - Self::Takeover { .. } => return Err(EncodeError::Version), Self::RouteStart { .. } if version.has_routes() => ROUTE_START, Self::RouteUpdate { .. } if version.has_routes() => ROUTE_UPDATE, Self::RouteHold { .. } if version.has_routes() => ROUTE_HOLD, @@ -344,20 +327,6 @@ impl Encode for AnnounceBroadcast { super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; cost.encode(w, version)?; } - Self::Takeover { - predecessor, - suffix, - route, - instance, - epoch, - cost, - } => { - w.varint(*predecessor)?; - suffix.encode(w, version)?; - encode_route_id(w, version, *route)?; - super::instance::encode_route(w, version, instance.as_ref(), *epoch)?; - cost.encode(w, version)?; - } Self::Synced => {} Self::RouteStart { node, advert } => { if *node == 0 { @@ -409,7 +378,6 @@ impl Encode for AnnounceBroadcast { Self::EndedId { .. } | Self::Update { .. } | Self::Restart { .. } - | Self::Takeover { .. } | Self::Skipped | Self::Synced | Self::RouteStart { .. } @@ -468,20 +436,6 @@ impl<'a> Decode<'a, Version> for AnnounceBroadcast { cost: Cost::decode(&mut body, version)?, } } - ANNOUNCE_TAKEOVER if version.has_announce_takeover() => { - let predecessor = body.varint()?; - let suffix = PathRef::decode(&mut body, version)?; - let route = decode_route_id(&mut body, version)?; - let (instance, epoch) = super::instance::decode_route(&mut body, version)?; - Self::Takeover { - predecessor, - suffix, - route, - instance, - epoch, - cost: Cost::decode(&mut body, version)?, - } - } ROUTE_START if version.has_routes() => { let node = body.varint()?; let advert = decode_advert(&mut body)?; @@ -1096,26 +1050,12 @@ mod tests { cost, }; assert_eq!(broadcast_round_trip(&restart, Version::Lite07), restart); - - let takeover = AnnounceBroadcast::Takeover { - predecessor: 2, - suffix: PathRef { - base: 1, - keep: 1, - rest: Path::from_static("p"), - }, - route: 0, - instance: Some(Instance::random()), - epoch, - cost, - }; - assert_eq!(broadcast_round_trip(&takeover, Version::Lite07), takeover); } } /// The draft's field order, byte for byte: ANNOUNCE_START is the path base, keep, and /// rest, the Route ID, the instance and epoch, and the cost; ANNOUNCE_UPDATE is the - /// Announce ID and the cost; ANNOUNCE_TAKEOVER leads with its predecessor. + /// Announce ID and the cost. #[test] fn lite07_announce_layout() { let encode = |msg: AnnounceBroadcast| { @@ -1157,17 +1097,6 @@ mod tests { }), [3, 5, 7, 1, b'a', 5, 6] ); - assert_eq!( - encode(AnnounceBroadcast::Takeover { - predecessor: 9, - suffix, - route: 3, - instance: Some("a".parse().unwrap()), - epoch: 5, - cost: Cost::new(6), - }), - [5, 10, 9, 1, 2, 1, b'c', 3, 1, b'a', 5, 6] - ); } /// Lite-07 names a claim's route instead of its hop chain, so a chain is refused there. @@ -1216,26 +1145,6 @@ mod tests { } } - // Only lite-07 carries ANNOUNCE_TAKEOVER: older versions send a start instead. - #[test] - fn announce_takeover_needs_lite07() { - let takeover = AnnounceBroadcast::Takeover { - predecessor: 0, - suffix: PathRef::literal(Path::from_static("p")), - route: 0, - instance: Some(Instance::random()), - epoch: 0, - cost: Cost::default(), - }; - assert!(matches!( - takeover.encode( - &mut Encoder::new(&mut Vec::new(), Version::Lite06.into()), - Version::Lite06 - ), - Err(EncodeError::Version) - )); - } - // Only lite-07 carries ANNOUNCE_RESTART: older versions send an end and a start, and an // older receiver skips the unknown type. #[test] diff --git a/rs/moq-net/src/lite/compress.rs b/rs/moq-net/src/lite/compress.rs index a4b842767bb..6c5ac2a7335 100644 --- a/rs/moq-net/src/lite/compress.rs +++ b/rs/moq-net/src/lite/compress.rs @@ -1,4 +1,4 @@ -//! Lite-07 announce compression: an ANNOUNCE_START or ANNOUNCE_TAKEOVER may copy the +//! Lite-07 announce compression: an ANNOUNCE_START may copy the //! head of a live announcement's path, naming that base by its distance back from the //! stream's next unassigned Announce ID. //! @@ -87,19 +87,7 @@ impl AnnounceDecoder { Ok(suffix) } - /// An ANNOUNCE_TAKEOVER: resolve it like a start against the live `predecessor`, - /// returning the predecessor's suffix, then its own. One at the predecessor's own - /// suffix retires the predecessor, since a suffix holds one advertisement. - pub fn takeover(&mut self, predecessor: u64, suffix: PathRef) -> Result<(Path, Path), Error> { - let previous = self.live.suffix(predecessor)?.clone(); - let suffix = self.start(suffix)?; - if suffix == previous { - self.live.suffixes.remove(&predecessor); - } - Ok((previous, suffix)) - } - - /// The id the last start or takeover assigned. + /// The id the last start assigned. pub fn last(&self) -> u64 { self.live.next.saturating_sub(1) } @@ -223,18 +211,6 @@ impl AnnounceEncoder { (Some(id), path) } - /// An ANNOUNCE_TAKEOVER of the live `predecessor`: like [`Self::start`], retiring - /// the predecessor when it holds the same suffix. - pub fn takeover(&mut self, predecessor: u64, suffix: Path) -> (u64, PathRef) { - let replaces = self.live.suffixes.get(&predecessor) == Some(&suffix); - let (id, path) = self.start(suffix); - let id = id.expect("a takeover is only sent on a version with announce ids"); - if replaces { - self.end(predecessor); - } - (id, path) - } - /// An ANNOUNCE_END for a live `id`. pub fn end(&mut self, id: u64) { if !self.version.has_announce_compression() { @@ -341,33 +317,6 @@ mod tests { assert_eq!(suffix.as_str(), "b/x"); } - // A takeover assigns the next id like a start. Beneath its predecessor the predecessor - // stays live; at the predecessor's own suffix it retires, after bases resolve against it. - #[test] - fn takeover_assigns_an_id_and_retires_one_in_place() { - let mut decoder = AnnounceDecoder::default(); - decoder.start(literal("pool")).unwrap(); - - let (previous, suffix) = decoder.takeover(0, path(1, 1, "p")).unwrap(); - assert_eq!((previous.as_str(), suffix.as_str()), ("pool", "pool/p")); - assert_eq!(decoder.get(0).unwrap().as_str(), "pool"); - - // Id 2 at id 1's own suffix, copied from id 1 itself, which then retires. - let (previous, suffix) = decoder.takeover(1, path(1, 2, "")).unwrap(); - assert_eq!((previous.as_str(), suffix.as_str()), ("pool/p", "pool/p")); - assert!(matches!(decoder.get(1), Err(Error::ProtocolViolation))); - assert!(matches!( - decoder.takeover(1, literal("pool/p/q")), - Err(Error::ProtocolViolation) - )); - assert!(matches!( - decoder.takeover(9, literal("pool/p/q")), - Err(Error::ProtocolViolation) - )); - assert_eq!(decoder.end(2).unwrap().as_str(), "pool/p"); - assert_eq!(decoder.end(0).unwrap().as_str(), "pool"); - } - #[test] fn rejects_a_keep_longer_than_the_base() { let mut decoder = AnnounceDecoder::default(); @@ -407,29 +356,6 @@ mod tests { (id.unwrap(), size) } - // The encoder's takeover ids and bases agree with the decoder's, in place and beneath. - #[test] - fn encoder_takeover_decodes_to_its_input() { - let mut encoder = AnnounceEncoder::new(VERSION); - let mut decoder = AnnounceDecoder::default(); - let (claim, _) = start(&mut encoder, &mut decoder, "pool"); - let mut predecessor = claim; - for suffix in ["pool/p", "pool/p", "pool/p/q"] { - let (id, wire) = encoder.takeover(predecessor, Path::new(suffix).unwrap()); - let (_, got) = decoder.takeover(predecessor, wire).unwrap(); - assert_eq!(got.as_str(), suffix); - predecessor = id; - } - // Both sides retired the id replaced in place and kept the rest. - let (id, wire) = encoder.start(Path::new("other").unwrap()); - decoder.start(wire).unwrap(); - assert_eq!(id, Some(4)); - assert!(matches!(decoder.end(1), Err(Error::ProtocolViolation))); - decoder.end(claim).unwrap(); - decoder.end(2).unwrap(); - decoder.end(3).unwrap(); - } - #[test] fn encoder_output_decodes_to_its_input() { let mut encoder = AnnounceEncoder::new(VERSION); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index b65205fe34f..0e72726e3bf 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -1233,8 +1233,7 @@ impl AnnounceRun { match event { origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update) - | origin::RouteEvent::Restart(update) - | origin::RouteEvent::Takeover { announce: update, .. } => { + | origin::RouteEvent::Restart(update) => { current.insert(update.prefix, update.route); } origin::RouteEvent::End(update) => { @@ -1598,15 +1597,10 @@ impl AnnounceRun { return Poll::Pending; }; - let (update, active, restart, predecessor) = match next { - Some(origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update)) => { - (update, true, false, None) - } - Some(origin::RouteEvent::Restart(update)) => (update, true, true, None), - Some(origin::RouteEvent::Takeover { announce, predecessor }) => { - (announce, true, false, Some(predecessor)) - } - Some(origin::RouteEvent::End(update)) => (update, false, false, None), + let (update, active, restart) = match next { + Some(origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update)) => (update, true, false), + Some(origin::RouteEvent::Restart(update)) => (update, true, true), + Some(origin::RouteEvent::End(update)) => (update, false, false), None => { if self.initial.take().is_some() { stream.writer.buffer(&lite::AnnounceBroadcast::Synced)?; @@ -1628,9 +1622,6 @@ impl AnnounceRun { continue; } - // Only lite-07 carries a takeover, and it answers from the route layer. Here one - // is another instance, in place or not. - let restart = restart || predecessor.as_ref() == Some(&suffix); match self.outgoing(&update.route, &absolute) { Some((hops, cost)) if restart && self.live.contains_key(&suffix) => { self.restart(stream, suffix, hops, cost, served(&update.route), &absolute)?; @@ -8707,14 +8698,13 @@ mod tests { crate::origin::RouteInfo::default() .with_hops(echoed_hops) .with_via(assigned), - None, ) .unwrap(); let mut local_hops = Hops::new(); local_hops.push(upstream).unwrap(); let _local = origin - .forward("local", crate::origin::RouteInfo::default().with_hops(local_hops), None) + .forward("local", crate::origin::RouteInfo::default().with_hops(local_hops)) .unwrap(); // A SETUP that declares no origin of its own, so only the assigned one applies. @@ -9288,13 +9278,7 @@ mod tests { let mut _route = None; let mut held = None; match wait { - Wait::Broadcast => { - _route = Some( - origin - .forward("room", crate::origin::RouteInfo::default(), None) - .unwrap(), - ) - } + Wait::Broadcast => _route = Some(origin.forward("room", crate::origin::RouteInfo::default()).unwrap()), Wait::Track => { let broadcast = origin.route("room", crate::origin::RouteOptions::hidden()).unwrap(); let dynamic = broadcast.broadcast().requests(); @@ -9597,6 +9581,7 @@ mod tests { fn legacy_track() -> lite::Track<'static> { lite::Track { instance: None, + node: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), } @@ -9606,6 +9591,7 @@ mod tests { lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index cde695a2ae3..bf8739737a2 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -10,7 +10,7 @@ //! target, and what they present is named under the mount point on the same routes. use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, HashMap}, task::{Poll, ready}, }; @@ -323,9 +323,8 @@ impl Answer { for path in &paths { sent.claims.remove(path); } - let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path, &mut started)?; + self.sync(stream, id, path)?; } } } @@ -372,21 +371,14 @@ impl Answer { None => path, }; match claim { - Some(mut claim) => { - if let Some(mount) = mount { - claim.continues = (claim.continues.take()) - .and_then(|(taken, instance)| Some((mount.name(&taken)?, instance))); - } - sent.claims.insert(path.clone(), claim) - } + Some(claim) => sent.claims.insert(path.clone(), claim), None => sent.claims.remove(&path), }; paths.push(path); } sent.floor = floor; - let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path, &mut started)?; + self.sync(stream, id, path)?; } self.floor(stream, id) } @@ -423,22 +415,10 @@ impl Answer { } /// Bring the peer's claim at `path` for the route to `id` in line with the node's. - /// - /// `started` collects the claims this pass started: a claim takes over only one the - /// peer held before, so a snapshot replays no takeover. - fn sync( - &mut self, - stream: &mut Stream, - id: route::Id, - path: Path, - started: &mut BTreeSet, - ) -> Result<(), Error> { + fn sync(&mut self, stream: &mut Stream, id: route::Id, path: Path) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); let wanted = sent.claims.get(&path).filter(|_| self.shows(&path)).cloned(); let wire = sent.wire; - let taken = (wanted.as_ref()) - .and_then(|claim| self.continued(id, &path, claim)) - .filter(|(taken, _)| !started.contains(taken)); let key = (id, path); match (self.live.get_mut(&key), wanted) { (None, Some(claim)) => { @@ -446,64 +426,39 @@ impl Answer { .1 .strip_prefix(&self.prefix) .expect("shown claims are under the prefix"); - let announce = match taken { - Some((taken, predecessor)) => { - tracing::debug!(route = %key.1, node = id, predecessor = %taken, "takeover"); - self.takeover(stream, wire, predecessor, suffix, &claim)? - } - None => { - tracing::debug!(route = %key.1, node = id, "announce"); - let (announce, suffix) = self.encoder.start(suffix); - stream.writer.buffer(&Msg::Active { - suffix, - route: wire, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; - announce.expect("lite-07 assigns announce ids") - } - }; - started.insert(key.1.clone()); + tracing::debug!(route = %key.1, node = id, "announce"); + let (announce, suffix) = self.encoder.start(suffix); + stream.writer.buffer(&Msg::Active { + suffix, + route: wire, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + let announce = announce.expect("lite-07 assigns announce ids"); self.live.insert(key, (announce, claim)); } (Some((announce, held)), Some(claim)) => { let announce = *announce; let restart = held.instance != claim.instance || held.epoch != claim.epoch; let reprice = held.cost != claim.cost; - let replacement = match taken { - // In place: a new Announce ID replaces the one it continues. - Some((taken, predecessor)) if taken == key.1 => { - tracing::debug!(route = %key.1, node = id, "takeover"); - let suffix = key - .1 - .strip_prefix(&self.prefix) - .expect("held claims are under the prefix"); - self.takeover(stream, wire, predecessor, suffix, &claim)? - } - _ if restart => { - tracing::debug!(route = %key.1, "restart"); - stream.writer.buffer(&Msg::Restart { - id: announce, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - cost: origin::Cost::new(claim.cost), - })?; - announce - } - _ => { - if reprice { - stream.writer.buffer(&Msg::Update { - id: announce, - hops: crate::Hops::new(), - cost: origin::Cost::new(claim.cost), - })?; - } - announce - } - }; - self.live.insert(key, (replacement, claim)); + if restart { + tracing::debug!(route = %key.1, "restart"); + stream.writer.buffer(&Msg::Restart { + id: announce, + instance: Some(claim.instance.clone()), + epoch: claim.epoch, + cost: origin::Cost::new(claim.cost), + })?; + } else if reprice { + stream.writer.buffer(&Msg::Update { + id: announce, + hops: crate::Hops::new(), + cost: origin::Cost::new(claim.cost), + })?; + } + self.live.insert(key, (announce, claim)); } (Some(_), None) => { let (announce, _) = self.live.remove(&key).expect("matched above"); @@ -515,45 +470,6 @@ impl Answer { Ok(()) } - /// Send `claim` at `suffix` under the route `wire` as an ANNOUNCE_TAKEOVER of the - /// peer's live `predecessor`, returning its Announce ID. - fn takeover( - &mut self, - stream: &mut Stream, - wire: u64, - predecessor: u64, - suffix: Path, - claim: &node::Claim, - ) -> Result { - let (announce, suffix) = self.encoder.takeover(predecessor, suffix); - stream.writer.buffer(&Msg::Takeover { - predecessor, - suffix, - route: wire, - instance: Some(claim.instance.clone()), - epoch: claim.epoch, - cost: origin::Cost::new(claim.cost), - })?; - Ok(announce) - } - - /// The claim the peer holds that `claim` at `path` takes over, and its Announce ID. - /// - /// Only the claim it continues, under the same route at the instance it continued, - /// and only while every claim the peer holds at that path serves the same instance: - /// otherwise the peer cannot tell which one it resolved through. - fn continued(&self, id: route::Id, path: &Path, claim: &node::Claim) -> Option<(Path, u64)> { - let (taken, instance) = claim.continues.as_ref()?; - if !path.has_prefix(taken) { - return None; - } - let (predecessor, held) = self.live.get(&(id, taken.clone()))?; - let alike = self.routes.keys().all(|other| { - (self.live.get(&(*other, taken.clone()))).is_none_or(|(_, other)| other.instance == *instance) - }); - (held.instance == *instance && alike).then(|| (taken.clone(), *predecessor)) - } - /// The grant changed: end what it no longer covers, start what it now does, and /// floor every route again, which certifies only while the grant covers the prefix. fn refilter(&mut self, stream: &mut Stream) -> Result<(), Error> { @@ -561,9 +477,8 @@ impl Answer { ids.sort_unstable(); for id in ids { let paths: Vec = self.routes[&id].claims.keys().cloned().collect(); - let mut started = BTreeSet::new(); for path in paths { - self.sync(stream, id, path, &mut started)?; + self.sync(stream, id, path)?; } if self.routes[&id].floored { self.floor(stream, id)?; diff --git a/rs/moq-net/src/lite/replay.rs b/rs/moq-net/src/lite/replay.rs index 1d5549d6689..29f826037f4 100644 --- a/rs/moq-net/src/lite/replay.rs +++ b/rs/moq-net/src/lite/replay.rs @@ -12,8 +12,7 @@ pub(super) fn initial(events: impl IntoIterator) -> V let (update, active) = match event { origin::RouteEvent::Start(update) | origin::RouteEvent::Update(update) - | origin::RouteEvent::Restart(update) - | origin::RouteEvent::Takeover { announce: update, .. } => (update, true), + | origin::RouteEvent::Restart(update) => (update, true), origin::RouteEvent::End(update) => (update, false), }; if let Some(index) = positions.remove(&update.prefix) { diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 9c88103f209..0008ecd3cf9 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -600,9 +600,8 @@ impl Subscriber { )?; } lite::AnnounceBroadcast::Skipped | lite::AnnounceBroadcast::Synced => {} - // Only lite-07 carries the route layer and takeovers, which `routed` handles above. - lite::AnnounceBroadcast::Takeover { .. } - | lite::AnnounceBroadcast::RouteStart { .. } + // Only lite-07 carries the route layer, which `routed` handles above. + lite::AnnounceBroadcast::RouteStart { .. } | lite::AnnounceBroadcast::RouteUpdate { .. } | lite::AnnounceBroadcast::RouteHold { .. } | lite::AnnounceBroadcast::RouteEnd { .. } @@ -8740,7 +8739,7 @@ impl Announced { return false; } // An error means the prefix is outside our origin's scope, so don't serve it. - match subscriber.origin.forward(&path, route.clone(), None) { + match subscriber.origin.forward(&path, route.clone()) { Ok(dynamic) => { self.attach(path, route, dynamic); true diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index b804c10cd84..8e35c85b961 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -219,30 +219,6 @@ impl Routed { let claim = claim(instance, epoch, cost)?; self.start(decoder.last(), route, path, claim)?; } - Msg::Takeover { - predecessor, - suffix, - route, - instance, - epoch, - cost, - } => { - let (_, suffix) = decoder.takeover(predecessor, suffix)?; - let path = self.path(prefix, &suffix)?; - // The predecessor is a live claim of the same route covering the new one. - let taken = self.claims.get(&predecessor).ok_or(Error::ProtocolViolation)?; - if taken.route != route || !path.has_prefix(&taken.path) { - return Err(Error::ProtocolViolation); - } - let continues = (taken.path.clone(), taken.claim.instance.clone()); - // At its own path it replaces the predecessor. - if continues.0 == path { - self.end(predecessor)?; - } - let mut claim = claim(instance, epoch, cost)?; - claim.continues = Some(continues); - self.start(decoder.last(), route, path, claim)?; - } Msg::Update { id, cost, .. } => { let claim = self.claims.get_mut(&id).ok_or(Error::ProtocolViolation)?; claim.claim.cost = cost.value(); @@ -460,29 +436,12 @@ impl Routed { return; } } - // The route this one takes over, as this stream serves it at the instance it - // continues, so what resolved through that one beneath the claim carries on. - let taken = (served.claim.continues.as_ref()) - .and_then(|(taken, instance)| { - let entry = self.served.get_mut(&(id, taken.clone()))?; - (entry.route.route.instance.as_ref() == Some(instance)).then_some((taken, entry)) - }) - .map(|(taken, entry)| match *taken == key.1 { - // In place, its sources serve what the new one continues: they live on with - // it rather than closing with the old route. - true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), - false => (&entry.route.dynamic, None), - }); // A new instance or epoch replaces the old route, attached before it goes. tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); - let (predecessor, live) = taken.unzip(); - match subscriber.origin.forward(&path, route.clone(), predecessor) { + match subscriber.origin.forward(&path, route.clone()) { Ok(dynamic) => { let wake = crate::util::QueueWake::new(key.clone(), self.ready.clone()); - let mut entry = AnnouncedRoute::new(route, dynamic, wake); - if let Some(live) = live.flatten() { - entry.live = live; - } + let entry = AnnouncedRoute::new(route, dynamic, wake); entry.waker.wake_by_ref(); self.served.insert(key.clone(), ServedClaim { named, route: entry }); self.stream.serving(id, &key.1); @@ -531,7 +490,6 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos instance: instance.ok_or(Error::ProtocolViolation)?, epoch, cost: cost.value(), - continues: None, }) } @@ -678,76 +636,4 @@ mod tests { } assert_eq!(pending(&routed), 0, "claims started and ended between floors"); } - - /// A takeover in place keeps the sources the predecessor minted open: they serve the - /// instance it continues. A restart in place closes them. - #[kio::test] - async fn a_takeover_in_place_keeps_the_predecessors_sources() { - for takeover in [true, false] { - let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); - let prefix = Path::empty(); - let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); - let mut decoder = AnnounceDecoder::default(); - let path = Path::from_static("room/host"); - let advert = Advert { - seqno: 0, - metric: 0, - down_only: false, - }; - let start = Msg::Active { - suffix: PathRef::literal(path.clone()), - route: 0, - instance: Some(Instance::random()), - epoch: 0, - hops: Default::default(), - cost: Cost::new(0), - }; - for msg in [Msg::RouteStart { node: 7, advert }, start] { - routed.receive(&prefix, msg, &mut decoder).unwrap(); - } - routed - .receive( - &prefix, - Msg::Floor { - id: 0, - revision: Some(0), - }, - &mut decoder, - ) - .unwrap(); - routed.outputs(&subscriber); - let minted = routed.served[&(7, path.clone())].route.live.consume(); - - let instance = Some(Instance::random()); - let replace = match takeover { - true => Msg::Takeover { - predecessor: 0, - suffix: PathRef::literal(path.clone()), - route: 0, - instance, - epoch: 0, - cost: Cost::new(0), - }, - false => Msg::Restart { - id: 0, - instance, - epoch: 0, - cost: Cost::new(0), - }, - }; - routed.receive(&prefix, replace, &mut decoder).unwrap(); - routed - .receive( - &prefix, - Msg::Floor { - id: 0, - revision: Some(1), - }, - &mut decoder, - ) - .unwrap(); - routed.outputs(&subscriber); - assert_eq!(minted.is_closed(), !takeover, "takeover: {takeover}"); - } - } } diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index a3c91507d3b..37fe51de123 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -20,7 +20,7 @@ pub enum Version { /// Lite-07. Adds the hidden opt-in to ANNOUNCE_REQUEST: without it, a route with /// a `.`-prefixed segment below the requested prefix is left out. SUBSCRIBE_DROP /// accounts for every group the subscription will not deliver. - /// ANNOUNCE_START and ANNOUNCE_TAKEOVER may copy a path head from a live + /// ANNOUNCE_START may copy a path head from a live /// announcement on the same stream, and name a Route ID in place of the hop chain. /// Adds the Auth Stream. SUBSCRIBE carries a list of ranges. The wire format is /// still work-in-progress, so it is advertised over ALPN as `moq-lite-07-wip` and @@ -268,7 +268,7 @@ impl Version { ) } - /// Whether ANNOUNCE_START and ANNOUNCE_TAKEOVER may copy a path head from a live + /// Whether ANNOUNCE_START may copy a path head from a live /// announcement on the same stream. Added in lite-07. pub fn has_announce_compression(self) -> bool { !matches!( @@ -357,16 +357,6 @@ impl Version { ) } - /// Whether the announce stream has ANNOUNCE_TAKEOVER: a new advertisement that - /// continues what a live one served beneath its prefix. Added in lite-07. Older - /// versions send an ANNOUNCE_START, which the receiver treats as another instance. - pub(crate) fn has_announce_takeover(self) -> bool { - !matches!( - self, - Self::Lite01 | Self::Lite02 | Self::Lite03 | Self::Lite04 | Self::Lite05 | Self::Lite06 - ) - } - /// Whether TRACK_INFO carries the refresh ratio and SUBSCRIBE_UPDATE a wanted group, /// so a join at any hop reaches the publisher. Added in lite-07; older peers see a /// track that never refreshes. diff --git a/rs/moq-net/src/model/announced.rs b/rs/moq-net/src/model/announced.rs index c33c6d116cc..def40197835 100644 --- a/rs/moq-net/src/model/announced.rs +++ b/rs/moq-net/src/model/announced.rs @@ -34,11 +34,7 @@ impl AnnounceConsumer { match event { RouteEvent::Start(announce) => Some(AnnounceEvent::Present(announce)), RouteEvent::End(announce) => Some(AnnounceEvent::Absent(announce)), - // A takeover elsewhere starts its own prefix; at its predecessor's it replaces it. - RouteEvent::Takeover { announce, predecessor } if predecessor != announce.prefix => { - Some(AnnounceEvent::Present(announce)) - } - RouteEvent::Update(_) | RouteEvent::Restart(_) | RouteEvent::Takeover { .. } => None, + RouteEvent::Update(_) | RouteEvent::Restart(_) => None, } } diff --git a/rs/moq-net/src/model/follow.rs b/rs/moq-net/src/model/follow.rs index 4b076ac7e65..cd5cacdc8a6 100644 --- a/rs/moq-net/src/model/follow.rs +++ b/rs/moq-net/src/model/follow.rs @@ -11,9 +11,8 @@ use crate::{Error, Instance, Path, broadcast}; /// and a request resolves through the most specific one, so only that route's events come /// through. Another route taking over is a [`Restart`](RouteEvent::Restart), unless both /// carry the same instance: those serve one continuous stream, so it is an -/// [`Update`](RouteEvent::Update). One announced as a -/// [`Takeover`](RouteEvent::Takeover) of the serving route continues it, and comes through -/// as one. Routes beneath the path serve other broadcasts and are ignored. +/// [`Update`](RouteEvent::Update), as for a job's route continuing the prefix route it +/// answered. Routes beneath the path serve other broadcasts and are ignored. pub(crate) struct Follow { announced: Routes, path: Path, @@ -130,7 +129,6 @@ impl Follow { let (RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) - | RouteEvent::Takeover { announce, .. } | RouteEvent::End(announce)) = &event; if !self.path.has_prefix(&announce.prefix) { return None; @@ -145,12 +143,11 @@ impl Follow { .iter() .position(|standing| standing.announce.prefix == prefix) .map(|index| self.covering.swap_remove(index).since); - let (announce, since, restart, predecessor) = match event { - RouteEvent::End(_) => (None, None, false, None), - RouteEvent::Start(announce) => (Some(announce), None, false, None), - RouteEvent::Update(announce) => (Some(announce), standing, false, None), - RouteEvent::Restart(announce) => (Some(announce), standing, true, None), - RouteEvent::Takeover { announce, predecessor } => (Some(announce), None, true, Some(predecessor)), + let (announce, since, restart) = match event { + RouteEvent::End(_) => (None, None, false), + RouteEvent::Start(announce) => (Some(announce), None, false), + RouteEvent::Update(announce) => (Some(announce), standing, false), + RouteEvent::Restart(announce) => (Some(announce), standing, true), }; if let Some(announce) = announce { self.covering.push(Standing { @@ -173,15 +170,6 @@ impl Follow { { Some(RouteEvent::End(before)) } - // The serving route continues as this one. - (Some((before, _)), Some((after, _))) - if after.prefix == prefix && predecessor.as_ref() == Some(&before.prefix) => - { - Some(RouteEvent::Takeover { - announce: after, - predecessor: before.prefix, - }) - } (Some((before, _)), Some((after, _))) if before.prefix != after.prefix => { let same = before.route.instance == after.route.instance; Some(match same { @@ -323,9 +311,9 @@ impl Following { } }; - // Resolving can run ahead of the follower: a takeover resolves at once while the - // origin holds its restart, and that restart then names the instance handed out. - // The caller freeing it before reading the restart is owed it again. + // Resolving can run ahead of the follower: another instance resolves at once while + // the origin holds its restart, and that restart then names the instance handed + // out. The caller freeing it before reading the restart is owed it again. let weak = broadcast.weak(); if self.last.as_ref().is_some_and(|last| last.same_channel(&weak)) { self.absorbed = true; @@ -380,10 +368,9 @@ impl Following { self.end(); } // After a refusal, another route may answer what the last one refused. - RouteEvent::Update(_) | RouteEvent::Takeover { .. } if self.refused => self.unseen = true, - // The same instance over another route, which its subscriptions ride out, or one - // continuing what was handed out. - RouteEvent::Update(_) | RouteEvent::Takeover { .. } => return false, + RouteEvent::Update(_) if self.refused => self.unseen = true, + // The same instance over another route, which its subscriptions ride out. + RouteEvent::Update(_) => return false, } true } @@ -410,7 +397,6 @@ mod tests { RouteEvent::Start(announce) => ("start", announce.prefix.to_string()), RouteEvent::Update(announce) => ("update", announce.prefix.to_string()), RouteEvent::Restart(announce) => ("restart", announce.prefix.to_string()), - RouteEvent::Takeover { announce, .. } => ("takeover", announce.prefix.to_string()), RouteEvent::End(announce) => ("end", announce.prefix.to_string()), } } @@ -437,7 +423,7 @@ mod tests { let instance = Instance::random(); // A prefix above the path covers it. - let pool = origin.forward("pool", route(&instance), None).unwrap(); + let pool = origin.forward("pool", route(&instance)).unwrap(); assert_eq!(followed(&mut follow).await, ("start", "pool".into())); // A route beneath the path serves another broadcast. @@ -472,7 +458,7 @@ mod tests { #[kio::test] async fn follow_starts_on_the_serving_route_when_joining_late() { let origin = Hop::new(1).unwrap().produce(); - let _pool = origin.forward("pool", RouteInfo::default(), None).unwrap(); + let _pool = origin.forward("pool", RouteInfo::default()).unwrap(); let _exact = origin.route("pool/job", RouteOptions::default()).unwrap(); let mut follow = origin.consume().path_routes("pool/job").unwrap(); @@ -493,7 +479,7 @@ mod tests { // Both land before the follower reads again, and the cursor delivers the prefix's start // ahead of the path's end, but the path went unserved in between. drop(exact); - let _pool = origin.forward("pool", route(&instance), None).unwrap(); + let _pool = origin.forward("pool", route(&instance)).unwrap(); assert_eq!(followed(&mut follow).await, ("end", "pool/job".into())); assert_eq!(followed(&mut follow).await, ("start", "pool".into())); @@ -514,7 +500,7 @@ mod tests { assert_eq!(followed(&mut follow).await, ("start", "pool/job".into())); let instance = Instance::random(); - let _pool = origin.forward("pool", route(&instance), None).unwrap(); + let _pool = origin.forward("pool", route(&instance)).unwrap(); exact.announce_info(route(&instance)).unwrap(); assert_eq!(followed(&mut follow).await, ("restart", "pool/job".into())); @@ -528,7 +514,7 @@ mod tests { let origin = Hop::new(1).unwrap().produce(); let mut follow = origin.consume().path_routes("pool/job").unwrap(); - let _pool = origin.forward("pool", RouteInfo::default(), None).unwrap(); + let _pool = origin.forward("pool", RouteInfo::default()).unwrap(); assert_eq!(followed(&mut follow).await, ("start", "pool".into())); let _exact = origin.route("pool/job", RouteOptions::default()).unwrap(); @@ -544,7 +530,7 @@ mod tests { let _chat = origin .scope("", &chat) .unwrap() - .forward("room/alice", RouteInfo::default(), None) + .forward("room/alice", RouteInfo::default()) .unwrap(); let mut follow = origin.consume().path_routes("room/alice").unwrap(); @@ -659,7 +645,7 @@ mod tests { let exact = origin.route("live", options(&instance)).unwrap(); let first = available(&mut announced).await; - let dynamic = origin.forward("", route(&instance), None).unwrap(); + let dynamic = origin.forward("", route(&instance)).unwrap(); exact.withdraw(); ended(&mut announced).await; let served = broadcast::Info::new().produce(); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9c3c3271037..8dcff562ed3 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -705,10 +705,9 @@ impl RouteOptions { } /// The route metadata these options advertise, under `instance` at `epoch`. - fn info(&self, instance: &Instance, epoch: u64) -> RouteInfo { + /// The price and credential to advertise, naming no instance. + fn info(&self) -> RouteInfo { RouteInfo { - instance: Some(instance.clone()), - epoch, cost: self.cost, auth: self.auth.clone(), ..RouteInfo::default() @@ -892,19 +891,8 @@ type AnnounceMeta = (RouteMeta, Option>); enum PendingUpdate { Announce(AnnounceMeta), Restart(AnnounceMeta), - /// An announce continuing the delivered `predecessor`, delivered as one only while - /// the consumer still holds the advertisement [`OriginConsumerState::seen`] stamped - /// `stamp`; otherwise as a plain start or restart. - Takeover { - meta: AnnounceMeta, - predecessor: Path, - stamp: u64, - }, Unannounce(AnnounceMeta), - UnannounceAnnounce { - old: AnnounceMeta, - new: AnnounceMeta, - }, + UnannounceAnnounce { old: AnnounceMeta, new: AnnounceMeta }, } // An update is taken only while collapsing it under the cursor state lock. @@ -940,11 +928,6 @@ struct OriginConsumerState { /// retraction cancels it entirely) or a metadata update on a route the /// consumer already observed (a retraction must still be delivered). delivered: BTreeSet, - /// Each delivered prefix's advertisement, stamped when another instance was last - /// delivered there, so a pending takeover can tell its predecessor still stands. - /// A prefix without one continues nothing for this consumer. - seen: HashMap, - next_stamp: u64, /// Set by the origin's teardown: the cursor drains `pending`, then reports /// the end instead of parking forever on a table that can never fire again. ended: bool, @@ -1042,12 +1025,6 @@ impl OriginConsumerState { None | Some(PendingUpdate::Announce(_)) => PendingUpdate::Announce(meta), // The consumer still owes the restart; this only moves its metadata. Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), - // Likewise the takeover. - Some(PendingUpdate::Takeover { predecessor, stamp, .. }) => PendingUpdate::Takeover { - meta, - predecessor, - stamp, - }, // Consumer needs to observe the retraction before this announce. Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { PendingUpdate::UnannounceAnnounce { old, new: meta } @@ -1072,39 +1049,6 @@ impl OriginConsumerState { self.put(prefix, new); } - /// A route at `prefix` continues the delivered `predecessor` (which may be `prefix` - /// itself). The consumer learns it as a takeover only while it holds that - /// predecessor when this is delivered; see [`PendingUpdate::Takeover`]. - fn apply_takeover(&mut self, prefix: Path, meta: RouteMeta, captures: Option>, predecessor: Path) { - // A predecessor about to change on the consumer is not the one it holds. - let settled = prefix == predecessor - || self - .pending - .get(&predecessor) - .is_none_or(|pending| matches!(pending.update, Some(PendingUpdate::Announce(_)))); - let stamp = self.seen.get(&predecessor).copied().filter(|_| settled); - let Some(stamp) = stamp else { - return match prefix == predecessor { - true => self.apply_restart(prefix, meta, captures), - false => self.apply_announce(prefix, meta, captures), - }; - }; - let meta = (meta, captures); - let new = match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { - None | Some(PendingUpdate::Takeover { .. }) => PendingUpdate::Takeover { - meta, - predecessor, - stamp, - }, - Some(PendingUpdate::Unannounce(old) | PendingUpdate::UnannounceAnnounce { old, .. }) => { - PendingUpdate::UnannounceAnnounce { old, new: meta } - } - Some(PendingUpdate::Restart(_)) => PendingUpdate::Restart(meta), - Some(PendingUpdate::Announce(_)) => PendingUpdate::Announce(meta), - }; - self.put(prefix, new); - } - fn apply_unannounce(&mut self, prefix: Path, last: RouteMeta, captures: Option>) { let last = (last, captures); match self.pending.get_mut(&prefix).and_then(|pending| pending.update.take()) { @@ -1114,25 +1058,9 @@ impl OriginConsumerState { let pending = self.remove(&prefix).unwrap(); self.owed -= usize::from(pending.owed); } - // Likewise a takeover, but what the consumer resolved through its predecessor - // may be what it continued, so the predecessor continues nothing more for it - // until another instance is delivered there. - Some(PendingUpdate::Takeover { predecessor, stamp, .. }) if !self.delivered.contains(&prefix) => { - let pending = self.remove(&prefix).unwrap(); - self.owed -= usize::from(pending.owed); - if self.seen.get(&predecessor) == Some(&stamp) { - self.seen.remove(&predecessor); - } - } // Either nothing is pending or the pending announce was a metadata // update on a delivered route; the consumer still owes a retraction. - None - | Some( - PendingUpdate::Announce(_) - | PendingUpdate::Restart(..) - | PendingUpdate::Takeover { .. } - | PendingUpdate::Unannounce(_), - ) => { + None | Some(PendingUpdate::Announce(_) | PendingUpdate::Restart(..) | PendingUpdate::Unannounce(_)) => { self.put(prefix, PendingUpdate::Unannounce(last)); } // The embedded announce cancels with this retraction; the consumer still @@ -1233,7 +1161,6 @@ impl OriginConsumerState { let (RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) - | RouteEvent::Takeover { announce, .. } | RouteEvent::End(announce)) = &event; // A trailing announce left pending is still owed. if owed && !self.pending.contains_key(&announce.prefix) { @@ -1248,7 +1175,6 @@ impl OriginConsumerState { Start, Update, Restart, - Takeover(Path), End, } @@ -1267,20 +1193,6 @@ impl OriginConsumerState { self.delivered.insert(prefix.clone()); (meta, Kind::Restart) } - PendingUpdate::Takeover { - meta, - predecessor, - stamp, - } => { - let kind = match self.seen.get(&predecessor) == Some(&stamp) { - true => Kind::Takeover(predecessor), - // The predecessor the consumer held is gone: this is another instance. - false if self.delivered.contains(&prefix) => Kind::Restart, - false => Kind::Start, - }; - self.delivered.insert(prefix.clone()); - (meta, kind) - } PendingUpdate::Unannounce(meta) => { self.delivered.remove(&prefix); (meta, Kind::End) @@ -1293,16 +1205,6 @@ impl OriginConsumerState { (old, Kind::End) } }; - match &kind { - Kind::Update => {} - Kind::End => { - self.seen.remove(&prefix); - } - Kind::Start | Kind::Restart | Kind::Takeover(_) => { - self.seen.insert(prefix.clone(), self.next_stamp); - self.next_stamp += 1; - } - } let announce = Announce { prefix, captures, @@ -1321,7 +1223,6 @@ impl OriginConsumerState { Kind::Start => RouteEvent::Start(announce), Kind::Update => RouteEvent::Update(announce), Kind::Restart => RouteEvent::Restart(announce), - Kind::Takeover(predecessor) => RouteEvent::Takeover { announce, predecessor }, Kind::End => RouteEvent::End(announce), }; (event, owed) @@ -1382,25 +1283,12 @@ struct RouteEntry { /// announce time for the same reason: the paths beneath the prefix the route /// may serve. A member whose intersection is too large to spell reaches nothing. reach: Vec, - /// The route this one continues, when it was advertised as a takeover. - takeover: Option, -} - -/// A route's predecessor as the route took it over: what was resolved through that -/// route beneath its prefix, before then, is the route's instance now. -/// -/// A one-time continuity reference, never a lifetime dependency: either route ends -/// on its own, and only what had resolved by [`Self::at`] is continued. -#[derive(Clone, Debug)] -struct Takeover { - /// The predecessor's entry id. - route: u64, - /// The instance the predecessor served then. - instance: Instance, - /// The predecessor's absolute prefix. - prefix: Path, - /// [`OriginState::next_route`] once the route took over. - at: u64, + /// Whether the job routes answering this prefix route take its instance, which it + /// minted rather than was given: their readers continue on the job's route. It moves + /// to a fresh one as each job carrying it stops serving; see [`OriginState::advance`]. + lends: bool, + /// The prefix route, by prefix and id, whose instance this job's route took. + lent: Option<(Path, u64)>, } impl RouteEntry { @@ -1773,28 +1661,8 @@ impl TableCursor { && self.named.as_ref().is_none_or(|(mount, _)| mount.names(&entry.prefix)) } - /// Where this cursor presents the predecessor `takeover` continues, when it presents - /// exactly that instance and every route it sees at the predecessor's prefix, - /// other than the successor `id` itself, serves it: a path beneath could resolve - /// through any of them, so another one there would make the continuity claim false - /// for what it served. - fn continued(&self, id: u64, takeover: &Takeover, routes: &RouteTable) -> Option { - let claim = prefix_claim(&takeover.prefix).ok()?; - let presented = self.presented(&takeover.prefix, &claim)?; - let current = self.current.get(&presented)?; - if current.instance != takeover.instance || current.prefix != takeover.prefix { - return None; - } - routes - .at(&takeover.prefix) - .filter(|entry| entry.id != id && self.visible(entry)) - .all(|entry| entry.instance == takeover.instance) - .then_some(presented) - } - - /// Deliver a changed winner at this presented prefix, as of `now`, the - /// [`OriginState::next_route`]. - fn update(&mut self, presented: &Path, best: Option<&RouteEntry>, now: u64, routes: &RouteTable) { + /// Deliver a changed winner at this presented prefix. + fn update(&mut self, presented: &Path, best: Option<&RouteEntry>) { let Some(entry) = best else { if let Some(last) = self.current.remove(presented) && let Ok(mut state) = self.state.write() @@ -1818,13 +1686,6 @@ impl TableCursor { let meta = entry.meta(); let captures = self.captures(&entry.prefix); - // Only as the route takes over: one this cursor sees later continues nothing it - // can vouch for. - let continued = entry - .takeover - .as_ref() - .filter(|takeover| takeover.at == now) - .and_then(|takeover| self.continued(entry.id, takeover, routes)); let previous = self.current.insert( presented.clone(), CursorRoute { @@ -1836,22 +1697,6 @@ impl TableCursor { }, ); match previous { - // A new presentation continuing another one, or this one continued in place. - None if continued.as_ref().is_some_and(|predecessor| predecessor != presented) => { - let predecessor = self.under.rejoin(&continued.unwrap()); - if let Ok(mut state) = self.state.write() { - state.apply_takeover(self.under.rejoin(presented), meta, captures, predecessor); - } - } - Some(previous) - if previous.captures == captures - && continued.as_ref().is_some_and(|predecessor| predecessor == presented) => - { - if let Ok(mut state) = self.state.write() { - let presented = self.under.rejoin(presented); - state.apply_takeover(presented.clone(), meta, captures, presented); - } - } // Captures are consumer identity, not route metadata. Replace the old // identity explicitly so consumers keyed on it remove it. Some(previous) if previous.captures != captures => { @@ -2202,19 +2047,6 @@ pub enum RouteEvent { /// request afresh to follow the new one. Subscriptions already open stay on the old /// one until dropped or its route goes. Restart(Announce), - /// A route now covers the prefix and continues what the route at `predecessor`, - /// a prefix at or above it, served beneath it. - /// - /// Whatever was resolved through the predecessor beneath the prefix is the new - /// route's instance now: keep it rather than request afresh. At the - /// predecessor's own prefix this replaces it, as a restart would; elsewhere the - /// predecessor stays, and either route ends on its own. - Takeover { - /// The route that continues. - announce: Announce, - /// The prefix of the route it continues, relative to the cursor's root. - predecessor: Path, - }, } /// A route over a prefix, as an announcement listing or a [`Routes`] stream reports it. @@ -2558,28 +2390,38 @@ impl Producer { frames: self.frames.clone(), }; let source = info.produce().with_stats(ingress); - let (instance, epoch) = options.instance.clone().unwrap_or_else(|| (Instance::random(), 0)); + // A job's route naming no instance takes its prefix route's, held in the table + // until it first announces; see [`Request::route`]. + let lender = request + .as_ref() + .filter(|_| options.instance.is_none()) + .map(|request| request.home.clone()); + let instance = match (&options.instance, &lender) { + (None, None) => Some((Instance::random(), 0)), + (named, _) => named.clone(), + }; let entry = announcing.announce( - options.info(&instance, epoch), + RouteInfo { + instance: instance.as_ref().map(|(instance, _)| instance.clone()), + epoch: instance.as_ref().map_or(0, |(_, epoch)| *epoch), + ..options.info() + }, Serving { server: None, source: Some(source.consume()), advertised: false, + lends: false, }, - None, + lender.as_ref(), )?; let hidden = options.hidden; + let control = RouteControl::new(options, &entry, request); let route = Route::new(RouteState { announcement: entry, broadcast: Some(source.clone()), server: None, publisher: None, - control: kio::Lock::new(RouteControl { - options, - instance, - epoch, - request, - }), + control: kio::Lock::new(control), _keepalive: self.tasks.keepalive(), }); // Closing the broadcast retracts the route, and refuses the request a job's route @@ -2643,6 +2485,7 @@ impl Producer { server: None, source: None, advertised: true, + lends: false, }, None, ) @@ -2723,12 +2566,19 @@ impl Producer { /// A consumer resolving a path under it through this route is handed to /// [`Route::requested`] as a [`Request`]: answer it by serving a broadcast already /// held ([`Request::serve`]), under this route's instance, or by creating the path's - /// own exact route ([`Request::route`]), under a fresh instance, so restarting one - /// job never splices into a copy of the last. This is how a service answers a whole + /// own exact route ([`Request::route`]). This is how a service answers a whole /// subtree without publishing each path ahead of time; a publisher that /// knows its broadcasts creates each one's exact route with [`Self::route`] /// instead, so subscribers can enumerate them. /// + /// Unless `options` names an instance, the route lends the one it mints to the exact + /// routes answering its requests, so a reader that resolved a path through it + /// continues on that path's answer rather than restarting. Once an exact route holding + /// the lent instance stops serving it (withdrawn, closed, or renamed), this route moves + /// to a fresh instance, as a restarted job's sequences are another stream: see + /// [`Route::instance`]. A route naming a stable instance lends nothing, and its + /// answers each mint their own. + /// /// The route is priced, credentialed and shown like an exact one. It is visible /// for as long as the returned [`Route`] (and every clone) lives, unless withdrawn. /// @@ -2739,20 +2589,24 @@ impl Producer { prefix: impl TryInto>, options: RouteOptions, ) -> Result { - let (instance, epoch) = options.instance.clone().unwrap_or_else(|| (Instance::random(), 0)); - let info = options.info(&instance, epoch); + // A minted instance lives in the table, which moves it on as its jobs end; see + // [`OriginState::advance`]. + let info = RouteInfo { + instance: options.instance.as_ref().map(|(instance, _)| instance.clone()), + epoch: options.instance.as_ref().map_or(0, |(_, epoch)| *epoch), + ..options.info() + }; let advertised = !options.hidden; - self.serving_route(prefix, info, Some(self.clone()), advertised, None, options) + let lends = options.instance.is_none(); + self.serving_route(prefix, info, Some(self.clone()), advertised, lends, options) } - /// Land a peer's route with its wire metadata, continuing what the peer's - /// `predecessor` route served beneath it when the peer took it over. + /// Land a peer's route with its wire metadata. #[cfg(any(feature = "async", test))] pub(crate) fn forward( &self, prefix: impl TryInto>, mut info: RouteInfo, - predecessor: Option<&Route>, ) -> Result { // A handle that names no instance serves one of its own. let instance = info.instance.get_or_insert_with(Instance::random).clone(); @@ -2762,8 +2616,7 @@ impl Producer { auth: info.auth.clone(), instance: Some((instance, info.epoch)), }; - let predecessor = predecessor.and_then(|route| route.state.server.as_ref()); - self.serving_route(prefix, info, None, true, predecessor, options) + self.serving_route(prefix, info, None, true, false, options) } fn serving_route( @@ -2772,34 +2625,29 @@ impl Producer { info: RouteInfo, publisher: Option, advertised: bool, - predecessor: Option<&kio::Shared>, + lends: bool, options: RouteOptions, ) -> Result { let announcing = Announcing::new(self, prefix)?; let serve = kio::Shared::::default(); serve.lock().requests.add_handler(); - let instance = info.instance.clone().expect("the caller names the instance"); - let epoch = info.epoch; let announcement = announcing.announce( info, Serving { server: Some(serve.clone()), source: None, advertised, + lends, }, - predecessor, + None, )?; + let control = RouteControl::new(options, &announcement, None); Ok(Route::new(RouteState { announcement, broadcast: None, server: Some(serve), publisher, - control: kio::Lock::new(RouteControl { - options, - instance, - epoch, - request: None, - }), + control: kio::Lock::new(control), _keepalive: self.tasks.keepalive(), })) } @@ -2971,13 +2819,13 @@ impl Announcing { }) } - /// Insert the route, continuing what `predecessor`, a covering served route, served - /// beneath it, when it is one. + /// Insert the route. A job's route naming no instance reads as its `lender`'s, the + /// prefix route its request came through, until it binds one as it announces. fn announce( &self, route: RouteInfo, serving: Serving, - predecessor: Option<&kio::Shared>, + lender: Option<&kio::Shared>, ) -> Result { debug_assert!( !route.hops.contains(&self.hop), @@ -2992,18 +2840,21 @@ impl Announcing { } // A handle that names no instance serves one of its own. - let instance = route.instance.clone().unwrap_or_else(Instance::random); + let minted = route.instance.clone().unwrap_or_else(Instance::random); let mut entries = Vec::with_capacity(self.prefixes.len()); for (prefix, claim) in &self.prefixes { shared.reannounced(prefix, &route.hops); let id = shared.next_route; shared.next_route += 1; let stale = shared.withdrawn_through(prefix, &route.hops); - let takeover = predecessor.and_then(|server| shared.takeover(prefix, server)); + let (instance, epoch) = match lender.and_then(|server| shared.lender(prefix, server)) { + Some(lent) => (lent.instance, lent.epoch), + None => (minted.clone(), route.epoch), + }; shared.routes.insert(RouteEntry { id, - instance: instance.clone(), - epoch: route.epoch, + instance, + epoch, prefix: prefix.clone(), scope: self.scope.clone(), hops: route.hops.clone(), @@ -3020,7 +2871,8 @@ impl Announcing { stale, claim: claim.clone(), reach: reach(&self.scope, claim), - takeover, + lends: serving.lends, + lent: None, }); shared.sync_route(prefix, claim); entries.push((prefix.clone(), id)); @@ -3045,6 +2897,8 @@ struct Serving { server: Option>, source: Option, advertised: bool, + /// Whether the job routes answering it take its instance; see [`RouteEntry::lends`]. + lends: bool, } /// A route on an origin: an exact path owning one broadcast, or a prefix serving the @@ -3102,15 +2956,36 @@ struct RouteState { /// announces, withdrawals and re-prices land in order. struct RouteControl { options: RouteOptions, - instance: Instance, - epoch: u64, + /// The instance and epoch the route advertises, as last seen when [`Self::held`]. + instance: (Instance, u64), + /// Whether the table entry holds the instance: a prefix route that lends its own, which + /// moves on as its jobs end (see [`OriginState::advance`]), or a job's route taking one + /// from it. An update then names none, leaving the entry's as it is. + held: bool, /// The request a job's route answers when it first announces. request: Option, } impl RouteControl { + fn new(options: RouteOptions, announcement: &AnnounceProducer, request: Option) -> Self { + let held = options.instance.is_none(); + let instance = announcement.instance().expect("a route just inserted has its entries"); + Self { + options, + instance, + held, + request, + } + } + + /// What to advertise. fn info(&self) -> RouteInfo { - self.options.info(&self.instance, self.epoch) + let (instance, epoch) = &self.instance; + RouteInfo { + instance: (!self.held).then(|| instance.clone()), + epoch: *epoch, + ..self.options.info() + } } } @@ -3149,8 +3024,18 @@ impl Route { /// The stream this route serves, minted when it was created unless /// [`RouteOptions::with_instance`] named one. + /// + /// Unnamed, it can change: a prefix route ([`Producer::route_prefix`]) moves to a fresh + /// instance each time an answer holding its current one ends, and a job's route + /// ([`Request::route`]) takes its prefix route's when it first announces. pub fn instance(&self) -> Instance { - self.state.control.lock().instance.clone() + let mut control = self.state.control.lock(); + if control.held + && let Some(current) = self.state.announcement.instance() + { + control.instance = current; + } + control.instance.0.clone() } /// Make the route visible, or keep it so. Idempotent. @@ -3172,10 +3057,11 @@ impl Route { let mut resolved = None; // Answered under the table lock that every release takes first, so the route // lands only for a request still pending and no release can slip in between. - // The answer continues what the prefix route served for the path: requesters - // waiting on it, and whatever resolved through the prefix route's current instance - // beneath it. Answered and taken over under one lock, so nothing resolves in between. - let admitted = self.state.announcement.update_if(info, Some(&request.home), |table| { + // A job's route naming no instance takes its prefix route's current one as it + // lands, so whatever resolved through the prefix route for the path continues on + // it rather than restarting. + let lender = control.held.then_some(&request.home); + let admitted = self.state.announcement.update_if(info, lender, |table| { // The prefix route's entries leave the table before its retraction releases the // request, so one gone from it no longer answers, even before that release. let home = table.routes.covering(&request.path).any(|entry| { @@ -3310,12 +3196,12 @@ impl Route { let mut control = self.state.control.lock(); match &info.instance { Some(instance) => { - control.instance = instance.clone(); - control.epoch = info.epoch; + control.instance = (instance.clone(), info.epoch); + control.held = false; } None => { - info.instance = Some(control.instance.clone()); - info.epoch = control.epoch; + info.instance = Some(control.instance.0.clone()); + info.epoch = control.instance.1; } } control.options.cost = info.cost; @@ -3346,8 +3232,8 @@ impl Route { pub(crate) fn update(&self, info: RouteInfo) -> Result<(), Error> { let mut control = self.state.control.lock(); if let Some(instance) = &info.instance { - control.instance = instance.clone(); - control.epoch = info.epoch; + control.instance = (instance.clone(), info.epoch); + control.held = false; } control.options.cost = info.cost; control.options.auth = info.auth.clone(); @@ -3403,12 +3289,12 @@ impl AnnounceProducer { /// under the table lock, so nothing ordered after that lock (a release, a /// retraction, the teardown) can change its answer before the route lands. /// - /// An entry advertised by this update continues what `predecessor`, a covering - /// served route, served beneath it; see [`Takeover`]. + /// A job's route answering its request through `lender`, a prefix route that lends + /// its instance, takes that route's instance and epoch as of now. fn update_if( &self, route: RouteInfo, - predecessor: Option<&kio::Shared>, + lender: Option<&kio::Shared>, admit: impl FnOnce(&OriginState) -> bool, ) -> Result { let mut shared = self.shared.lock(); @@ -3431,27 +3317,35 @@ impl AnnounceProducer { for (prefix, id) in &self.entries { shared.reannounced(prefix, &route.hops); let stale = shared.withdrawn_through(prefix, &route.hops); + let lent = lender.and_then(|server| shared.lender(prefix, server)); // Each entry keeps its advertised prefix; only the metadata moves. let Some(entry) = shared.routes.entry_mut(prefix, *id) else { return Err(Error::Closed); }; - // Only advertising takes over: a re-price continues nothing new. - let advertising = !entry.advertised; - if let Some(instance) = &route.instance { - // Another stream continues nothing the old one took over. - if *instance != entry.instance { - entry.takeover = None; - } - // Another stream: what its server was asked, or answered, for the old one is - // not its own. Under the table lock, so no front sees the new instance while - // an old answer can still land. A higher epoch of the same one continues it. - if *instance != entry.instance - && let Some(server) = &entry.server - { - released.extend(server.lock().renew()); + let next = match &lent { + Some(lent) => Some((lent.instance.clone(), lent.epoch)), + None => route.instance.clone().map(|instance| (instance, route.epoch)), + }; + let mut spent = None; + if let Some((instance, epoch)) = next { + if instance != entry.instance { + // Another stream: what its server was asked, or answered, for the old one + // is not its own. Under the table lock, so no front sees the new instance + // while an old answer can still land. A higher epoch of the same one + // continues it. + if let Some(server) = &entry.server { + released.extend(server.lock().renew()); + } + // The instance it took stops serving here. + if entry.advertised { + spent = entry.lent.take().map(|lender| (lender, entry.instance.clone())); + } } - entry.instance = instance.clone(); - entry.epoch = route.epoch; + entry.instance = instance; + entry.epoch = epoch; + } + if let Some(lent) = lent { + entry.lent = Some((lent.prefix, lent.id)); } entry.hops = route.hops.clone(); entry.stale = stale; @@ -3461,15 +3355,10 @@ impl AnnounceProducer { entry.node = route.node; entry.advertised = true; let claim = entry.claim.clone(); - if advertising { - // Past everything asked so far, so only that is continued. - shared.next_route += 1; - let takeover = predecessor.and_then(|server| shared.takeover(prefix, server)); - if let Some(entry) = shared.routes.entry_mut(prefix, *id) { - entry.takeover = takeover; - } - } shared.sync_route(prefix, &claim); + if let Some(((lender, lender_id), instance)) = spent { + shared.advance(&lender, lender_id, &instance); + } shared.prune_withdrawn(prefix); } // Under the table lock, as withdrawing and retracting clear it, so a racing one @@ -3484,6 +3373,14 @@ impl AnnounceProducer { Ok(true) } + /// The instance and epoch the entries carry, or `None` once they are gone. + fn instance(&self) -> Option<(Instance, u64)> { + let shared = self.shared.lock(); + let (prefix, id) = self.entries.first()?; + let entry = shared.routes.entry(prefix, *id)?; + Some((entry.instance.clone(), entry.epoch)) + } + /// Whether the entries are advertised, or [`Error::Closed`] once they are gone. fn is_live(&self) -> Result { let shared = self.shared.lock(); @@ -3537,7 +3434,11 @@ impl AnnounceProducer { released.extend(server.lock().renew()); } let claim = entry.claim.clone(); + let spent = entry.lent.take().map(|lender| (lender, entry.instance.clone())); shared.sync_route(prefix, &claim); + if let Some(((lender, lender_id), instance)) = spent { + shared.advance(&lender, lender_id, &instance); + } } drop(shared); drop(guard); @@ -3577,6 +3478,11 @@ impl AnnounceProducer { shared.restale(prefix); } shared.sync_route(&entry.prefix, &entry.claim); + if entry.advertised + && let Some((lender, lender_id)) = &entry.lent + { + shared.advance(lender, *lender_id, &entry.instance); + } shared.prune_withdrawn(&entry.prefix); } drop(shared); @@ -3639,21 +3545,17 @@ impl OwnClaims { } fn apply(&self, event: RouteEvent) { - let (announce, predecessor) = match event { - RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { - (announce, None) - } - RouteEvent::Takeover { announce, predecessor } => (announce, Some(predecessor)), - RouteEvent::End(announce) => return self.node.own(announce.prefix, None, None), + let announce = match event { + RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => announce, + RouteEvent::End(announce) => return self.node.own(announce.prefix, None), }; let route = announce.route; let claim = route.instance.map(|instance| crate::route::node::Claim { instance, epoch: route.epoch, cost: route.cost.value(), - continues: None, }); - self.node.own(announce.prefix, claim, predecessor.as_ref()); + self.node.own(announce.prefix, claim); } } @@ -4252,26 +4154,6 @@ impl FrontServe { // Read alongside the decision, under the lock a poke takes first. self.seen = self.watch.seen(); self.front.retain_routes(|route| table.routes.covers(&self.path, route)); - // A route that took over the instance the front resolved continues it. The table - // carries the front's verdict forward as each takeover lands; this also catches - // one that landed before the front wrote its first, or won the path only later. - if let Some(instance) = self.resolved.as_mut() - && let Some(next) = self.request.write().ok().and_then(|mut pending| { - if let Some(best) = table.best_route(&self.path, self.horizon, &HashSet::new(), |_| true) { - PendingFront::continue_with(&mut pending, best); - } - match &pending.resolved { - Some(Ok(next)) if next != instance => Some(next.clone()), - _ => None, - } - }) { - for (asked, _) in self.instances.values_mut() { - if asked == instance { - *asked = next.clone(); - } - } - *instance = next; - } let best = pick( &table, &self.path, @@ -4346,14 +4228,8 @@ impl FrontServe { } match &self.resolved { Some(resolved) => resolved != asked, - // A winner that took over the instance asked continues it. - None => pick(&table, &self.path, self.horizon, &self.front, self.pin.as_ref()).is_some_and(|best| { - best.instance != *asked - && best - .takeover - .as_ref() - .is_none_or(|takeover| takeover.instance != *asked) - }), + None => pick(&table, &self.path, self.horizon, &self.front, self.pin.as_ref()) + .is_some_and(|best| best.instance != *asked), } } @@ -4647,10 +4523,10 @@ impl kio::Task for FrontServe { Action::Resolve => { // The broadcast is whatever its first source serves, from the instance // its route had when asked. - let (Some(source), Some(route)) = (self.front.serving(), self.front.serving_route()) else { + let Some(source) = self.front.serving() else { continue; }; - let Some((instance, asked)) = self.instances.get(&source).cloned() else { + let Some((instance, _)) = self.instances.get(&source).cloned() else { continue; }; // Route datagram lookups before a requester can see the broadcast, @@ -4660,24 +4536,9 @@ impl kio::Task for FrontServe { self.broadcast.serve_datagrams_from(self.sources.get(&source).cloned()); } self.resolved = Some(instance.clone()); - // Under the table's lock, as each takeover carries the front: a route - // that took over the one asked before this answer landed (a job route - // answering its prefix route's request) continues it, so requesters - // get its instance rather than one already superseded. - let table = self.shared.read(); - if let Ok(mut pending) = self.request.write() - && pending.resolved.is_none() - { - pending.first = Some(instance.clone()); - pending.resolved = Some(Ok(instance)); - pending.via = Some((route, asked)); - if let Some(best) = - table.best_route(&self.path, self.horizon, &HashSet::new(), |_| true) - { - PendingFront::continue_with(&mut pending, best); - } + if let Ok(mut pending) = self.request.write() { + pending.resolved.get_or_insert(Ok(instance)); } - drop(table); } Action::Query { track: name, source } => { let Some(io) = self.tracks.get_mut(&name) else { continue }; @@ -5174,8 +5035,7 @@ struct Watched { } enum WatchProducer { - /// A front's watch carries the front too; see [`FrontWatch`]. - Changes(kio::Producer, Option), + Changes(kio::Producer), #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] Selection(kio::Producer), } @@ -5184,11 +5044,6 @@ struct Selected { path: Path, via: Hop, instance: Instance, - /// The prefix of the route the instance came through. - prefix: Path, - /// [`OriginState::next_route`] when selected: only a [`Takeover`] after that - /// continues the instance. - since: u64, /// The selection period, closed when the peer stops selecting the instance. current: Option>, /// Open until another instance of the peer takes the path, which may answer a @@ -5208,21 +5063,6 @@ enum Change { } impl Selected { - /// Follow `best` when it took over the selected instance: what the peer served - /// through it is the new instance now, so the current period carries on. A period - /// already over stays over. - fn continue_with(&mut self, best: &RouteEntry) { - if self.current.is_some() - && let Some(takeover) = &best.takeover - && takeover.instance == self.instance - && takeover.prefix == self.prefix - && self.since < takeover.at - { - self.instance = best.instance.clone(); - self.prefix = best.prefix.clone(); - } - } - /// How the period changes now that the peer's best route is `best`, if it does. fn change(&self, best: Option<&RouteEntry>) -> Option { let Some(best) = best else { @@ -5573,15 +5413,10 @@ impl RouteNode { } /// Wake route watchers, and revoke only leases whose selected instance changed. - /// `taken` lists the routes taken over by the change: only a front resolved through - /// one of them can follow it. - fn poke(&self, routes: Option<(&RouteTable, &[u64])>) { + fn poke(&self, routes: Option<&RouteTable>) { for (_, watch) in &self.watches { match watch { - WatchProducer::Changes(watch, front) => { - if let (Some(front), Some((routes, taken))) = (front, routes) { - front.follow(routes, taken); - } + WatchProducer::Changes(watch) => { if let Ok(mut watched) = watch.write() { watched.generation += 1; } @@ -5589,16 +5424,12 @@ impl RouteNode { WatchProducer::Selection(watch) => { if let Ok(mut selected) = watch.write() { let change = match routes { - Some((routes, _)) => { - let best = - routes.best_route(&selected.path, Horizon::default(), &HashSet::new(), |entry| { - entry.via == selected.via - }); - if let Some(best) = best { - selected.continue_with(best); - } - selected.change(best) - } + Some(routes) => selected.change(routes.best_route( + &selected.path, + Horizon::default(), + &HashSet::new(), + |entry| entry.via == selected.via, + )), None => selected.current.is_some().then_some(Change::End), }; // Only a change wakes the requests waiting on the selection. @@ -5612,13 +5443,13 @@ impl RouteNode { } /// Wake only the watched subtrees covered by this route. - fn poke_below(&self, routes: &RouteTable, taken: &[u64]) { + fn poke_below(&self, routes: &RouteTable) { if self.watches_below == 0 { return; } - self.poke(Some((routes, taken))); + self.poke(Some(routes)); for child in self.children.values() { - child.poke_below(routes, taken); + child.poke_below(routes); } } @@ -5809,13 +5640,13 @@ impl RouteTable { } /// Register a watch on the routes covering `path`; see [`Watch`]. - fn add_watch(&mut self, path: &Path, id: u64, front: Option) -> kio::Consumer { + fn add_watch(&mut self, path: &Path, id: u64) -> kio::Consumer { let producer = kio::Producer::::default(); let consumer = producer.consume(); self.root .reach(path.parts(), Below::WATCH) .watches - .push((id, WatchProducer::Changes(producer, front))); + .push((id, WatchProducer::Changes(producer))); consumer } @@ -5857,18 +5688,9 @@ impl RouteTable { } /// Wake the watches of every path a route at `prefix` covers. - /// A route at `prefix` that took over as of `now` carries the fronts resolved through - /// its predecessor along; see [`FrontWatch`]. - fn poke_below(&self, prefix: &Path, now: u64) { + fn poke_below(&self, prefix: &Path) { if let (_, Some(node)) = self.split(prefix) { - let taken: Vec = node - .entries - .iter() - .filter_map(|entry| entry.takeover.as_ref()) - .filter(|takeover| takeover.at == now) - .map(|takeover| takeover.route) - .collect(); - node.poke_below(self, &taken); + node.poke_below(self); } } @@ -6039,7 +5861,6 @@ impl OriginState { fn sync_route(&mut self, prefix: &Path, claim: &Pattern) { // Split borrows: the recompute reads `routes` while mutating a cursor. let routes = &self.routes; - let now = self.next_route; let mut candidates = None; for id in routes.cursors_touching(prefix) { let Some(cursor) = self.cursors.get_mut(&id) else { @@ -6048,7 +5869,7 @@ impl OriginState { if let Some(presented) = cursor.presented(prefix, claim) { if presented.is_empty() { // A cursor root can collapse several covering prefixes into one. - Self::sync_cursor(routes, cursor, &presented, now); + Self::sync_cursor(routes, cursor, &presented); } else { // Every non-root presentation refers to this exact prefix. Rank // its routes once, then take each cursor's first eligible visible entry. @@ -6066,12 +5887,12 @@ impl OriginState { .filter(|entry| cursor.visible(entry)); let epochs = Epochs::of(visible.clone()); let best = visible.clone().find(|entry| !epochs.superseded(entry)); - cursor.update(&presented, best, now, routes); + cursor.update(&presented, best); } } } // The fronts and requesters under the prefix re-select from the table. - routes.poke_below(prefix, now); + routes.poke_below(prefix); } /// Start a [`Snapshot`] at the absolute `prefix` for the routes arriving on `link`. @@ -6171,35 +5992,9 @@ impl OriginState { /// Register a [`Watch`] on the routes covering `path`. fn watch(&mut self, shared: &SendCell>, path: &Path) -> Watch { - self.watch_inner(shared, path, None) - } - - /// Register a front's [`Watch`]: the table also carries the front through each - /// takeover; see [`FrontWatch`]. - fn watch_front( - &mut self, - shared: &SendCell>, - path: &Path, - horizon: Horizon, - front: kio::Producer, - ) -> Watch { - let front = FrontWatch { - path: path.clone(), - horizon, - front, - }; - self.watch_inner(shared, path, Some(front)) - } - - fn watch_inner( - &mut self, - shared: &SendCell>, - path: &Path, - front: Option, - ) -> Watch { let id = self.next_watch; self.next_watch += 1; - let signal = self.routes.add_watch(path, id, front); + let signal = self.routes.add_watch(path, id); Watch { shared: shared.clone(), path: path.clone(), @@ -6210,7 +6005,7 @@ impl OriginState { /// Recompute the best visible route presenting at `presented` (relative) for /// one cursor and deliver the change, if any. - fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path, now: u64) { + fn sync_cursor(routes: &RouteTable, cursor: &mut TableCursor, presented: &Path) { // The entries presenting here are the ones announced at the absolute // prefix, or, for the cursor's own root, at the root and every prefix // above it (all of which present as the empty path). Among them, the @@ -6224,7 +6019,7 @@ impl OriginState { } }; - cursor.update(presented, best, now, routes); + cursor.update(presented, best); } /// Register a cursor and replay the current best route per presented prefix. @@ -6258,7 +6053,7 @@ impl OriginState { } } for p in &presented { - Self::sync_cursor(&self.routes, &mut cursor, p, self.next_route); + Self::sync_cursor(&self.routes, &mut cursor, p); } for head in &cursor.heads { self.routes.add_cursor(head, id); @@ -6362,19 +6157,52 @@ impl OriginState { self.routes.best_route(path, horizon, excluded, only) } + /// The prefix route `server` answers through for `path`, when it lends its instance to + /// the job routes answering its requests. + fn lender(&self, path: &Path, server: &kio::Shared) -> Option { + self.routes + .covering(path) + .find(|entry| { + entry.lends && entry.live() && entry.server.as_ref().is_some_and(|other| other.same_channel(server)) + }) + .map(|entry| Lent { + prefix: entry.prefix.clone(), + id: entry.id, + instance: entry.instance.clone(), + epoch: entry.epoch, + }) + } + + /// A job's route carrying `spent`, the instance it took from the prefix route `id` at + /// `prefix`, stopped serving it: that prefix route moves to a fresh instance. + /// + /// One instance is one stream at each path, and the job's sequences end with it. Were the + /// prefix route to keep the instance, its next answer for the path, a restarted job, + /// would splice a new sequence space into everything that holds the old one, and nothing + /// short of remembering every path it ever answered could tell. Under the table's lock, + /// with the job's retraction, so no request resolves the path through the old instance + /// in between. A prefix route already past `spent` has nothing to do. + fn advance(&mut self, prefix: &Path, id: u64, spent: &Instance) { + let Some(entry) = self.routes.entry_mut(prefix, id) else { + return; + }; + if entry.instance != *spent { + return; + } + entry.instance = Instance::random(); + entry.epoch = 0; + let claim = entry.claim.clone(); + self.sync_route(prefix, &claim); + } + /// Whether `instance` still serves `path` as a request would resolve it now: the winning - /// route carries it or took it over, or, for a request `pinned` to it, a route of its own - /// remains. The rule [`Consumer::lookup`] resolves by. + /// route carries it, or, for a request `pinned` to it, a route of its own remains. The + /// rule [`Consumer::lookup`] resolves by. fn still_serves(&self, path: &Path, horizon: Horizon, instance: &Instance, pinned: bool) -> bool { let Some(best) = self.best_route(path, horizon, &HashSet::new(), |_| true) else { return false; }; - if best.instance == *instance - || best - .takeover - .as_ref() - .is_some_and(|takeover| takeover.instance == *instance) - { + if best.instance == *instance { return true; } pinned @@ -6382,24 +6210,14 @@ impl OriginState { .best_route(path, horizon, &HashSet::new(), |entry| entry.instance == *instance) .is_some() } +} - /// The covering served route `server` answers through, taken over by a route at - /// `prefix` now: `None` unless it is live and its scope authorizes `prefix`. - fn takeover(&self, prefix: &Path, server: &kio::Shared) -> Option { - let entry = self - .routes - .covering(prefix) - .find(|entry| entry.live() && entry.server.as_ref().is_some_and(|other| other.same_channel(server)))?; - if !entry.scope.matches(prefix.as_str()) { - return None; - } - Some(Takeover { - route: entry.id, - instance: entry.instance.clone(), - prefix: entry.prefix.clone(), - at: self.next_route, - }) - } +/// A prefix route lending its instance, as [`OriginState::lender`] found it. +struct Lent { + prefix: Path, + id: u64, + instance: Instance, + epoch: u64, } /// One-shot result of a dynamic broadcast request. @@ -6420,13 +6238,6 @@ struct PendingBroadcast { /// the front held. struct PendingFront { resolved: Option>, - /// The route the front resolved through, and [`OriginState::next_route`] when it - /// asked: only a [`Takeover`] after that continues it. Each route that takes the - /// front over carries this forward; see [`Self::continue_with`]. - via: Option<(u64, u64)>, - /// The instance the front resolved first, which each [`Takeover`] since carried on - /// as `resolved`: a request naming it reads what continues it. - first: Option, /// The instance the front resolves on unless the table moves first: the winner it /// was minted for, then each one it selects. Written under the table's lock, so a /// request never joins a front that picked an instance the request saw beaten. @@ -6435,58 +6246,6 @@ struct PendingFront { refused: bool, } -impl PendingFront { - /// Whether a route that took over by `takeover` continues the instance the front - /// resolved: through the route it resolved that through, after the front asked. - fn continued_by(&self, takeover: Option<&Takeover>) -> bool { - let (Some(Ok(resolved)), Some((route, asked)), Some(takeover)) = (&self.resolved, self.via, takeover) else { - return false; - }; - takeover.route == route && takeover.instance == *resolved && asked < takeover.at - } - - /// Follow `best` when it took over what the front resolved: the front, its sources, - /// and the requests joining it carry the new instance. Under the table's lock as each - /// route takes over (see [`FrontWatch`]), so a chain of takeovers carries the front to - /// its end, however many land before the front's task looks. - fn continue_with(this: &mut kio::Mut<'_, Self>, best: &RouteEntry) { - // Read first: a write through the guard wakes every requester. - if this.continued_by(best.takeover.as_ref()) { - let asked = this.via.map(|(_, asked)| asked); - this.resolved = Some(Ok(best.instance.clone())); - this.via = asked.map(|asked| (best.id, asked)); - } - } -} - -/// A front's [`Watch`] as the route table holds it: what the front is for, so the table -/// can carry its [`PendingFront`] through a takeover as the takeover lands. -struct FrontWatch { - path: Path, - horizon: Horizon, - front: kio::Producer, -} - -impl FrontWatch { - /// Follow the path's winner, when it took over what the front resolved through one - /// of the routes `taken` over just now. - fn follow(&self, routes: &RouteTable, taken: &[u64]) { - if taken.is_empty() { - return; - } - let Ok(mut pending) = self.front.write() else { - return; - }; - // Skip the walk for a front no takeover here can continue. - if !pending.via.is_some_and(|(route, _)| taken.contains(&route)) { - return; - } - if let Some(best) = routes.best_route(&self.path, self.horizon, &HashSet::new(), |_| true) { - PendingFront::continue_with(&mut pending, best); - } - } -} - impl Route { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn request_validity(&self, path: &Path) -> Option { @@ -6501,7 +6260,6 @@ impl Route { })?; let via = entry.via; let instance = entry.instance.clone(); - let prefix = entry.prefix.clone(); let active = table .best_route(path, Horizon::default(), &HashSet::new(), |entry| entry.via == via) .is_some_and(|entry| entry.instance == instance); @@ -6509,8 +6267,6 @@ impl Route { path: path.clone(), via, instance, - prefix, - since: table.next_route, current: active.then(kio::Producer::default), answering: active.then(kio::Producer::default), }); @@ -6627,7 +6383,8 @@ impl ServeState { /// Yielded by [`Route::requested`]. The requester is awaiting inside /// [`Consumer::request`]. Answer it one of three ways: [`serve`](Self::serve) a /// broadcast already held, under the prefix route's instance; create the path's own -/// exact route with [`route`](Self::route), under a fresh instance; or +/// exact route with [`route`](Self::route), which continues the instance the prefix route +/// lends until it ends; or /// [`reject`](Self::reject) it with an error. Dropping it rejects it. /// /// Serve a held broadcast without announcing an exact route: @@ -6662,9 +6419,12 @@ impl Request { /// Answer with the path's own exact route, owning a new broadcast. /// - /// The route serves a fresh instance unless `options` names one, so a job that ends - /// and restarts beneath the same prefix is a new stream for its restarted sequences. - /// Ending the route withdraws only this path: the prefix route keeps answering later + /// Unless `options` names an instance, the route takes the one its prefix route lends + /// when it first announces, so whatever resolved the path through the prefix route + /// continues on this answer; a prefix route that lends nothing has the route mint its + /// own. Ending the route withdraws only this path, and moves a prefix route that lent + /// its instance to a fresh one, so a job that ends and restarts beneath the same prefix + /// is a new stream for its restarted sequences. The prefix route keeps answering later /// requests. Created visible, the route announces now and answers the requesters /// waiting on its path. Created [`RouteOptions::hidden`], they wait until /// [`Route::announce`], so the job can fill its catalog first; dropping the route @@ -6685,7 +6445,7 @@ impl Request { /// The answer belongs to the prefix route's instance at this path: every answer for /// that path must continue the same sequence space and bytes. Start a new prefix /// route to serve different content there. For a job that may restart independently, - /// use [`Self::route`] to announce its own instance. + /// use [`Self::route`]: its end moves the prefix route to another instance. pub fn serve(self, broadcast: impl Consume) { let broadcast = broadcast.consume(); let _ = ServeState::resolve(&self.home, &self.path, &self.producer, Ok(broadcast)); @@ -6789,20 +6549,14 @@ impl Lookup { RequestState::Failed(error) => Poll::Ready(Err(error.clone())), RequestState::Pending { front, broadcast } => Poll::Ready( match ready!(front.poll(waiter, |state| match &state.resolved { - Some(result) => Poll::Ready(result.clone().map(|instance| (instance, state.first.clone())),), + Some(result) => Poll::Ready(result.clone()), None => Poll::Pending, })) { - // Another instance than the one named, and not one continuing it: never hand - // it out. - Ok(Ok((instance, first))) - if self - .instance - .as_ref() - .is_some_and(|named| *named != instance && Some(named) != first.as_ref()) => - { + // Another instance than the one named: never hand it out. + Ok(Ok(instance)) if self.instance.as_ref().is_some_and(|named| *named != instance) => { Err(Error::Unroutable) } - Ok(Ok((instance, _))) => Ok(self.hand_out(broadcast.clone(), instance)), + Ok(Ok(instance)) => Ok(self.hand_out(broadcast.clone(), instance)), Ok(Err(err)) => Err(err), // The front's driver went away without a verdict: nobody could route it. Err(_closed) => Err(Error::Unroutable), @@ -6846,10 +6600,7 @@ impl Requesting { match discovery.poll_next(waiter) { Poll::Ready(Some(RouteEvent::End(_))) => self.routed = false, Poll::Ready(Some( - RouteEvent::Start(announce) - | RouteEvent::Restart(announce) - | RouteEvent::Update(announce) - | RouteEvent::Takeover { announce, .. }, + RouteEvent::Start(announce) | RouteEvent::Restart(announce) | RouteEvent::Update(announce), )) => { self.routed = self.instance.is_none() || announce.route.instance == self.instance; } @@ -7399,7 +7150,7 @@ impl Consumer { } /// Every change to the routes announced in this consumer's scope, for relays and - /// stats: route metadata, instance replacements and takeovers included. + /// stats: route metadata and instance replacements included. /// /// Allocates a per-cursor coalescing buffer and replays the currently /// announced routes as initial updates. Routes stay prefixes and are named @@ -7757,22 +7508,15 @@ impl Consumer { // Nothing serves the path: no announced broadcast and no served route. // Checked before joining a front, so a front still draining after its // route retracted takes no newcomers. - let Some((winner, takeover)) = state + let Some(winner) = state .best_route(&absolute, horizon, &HashSet::new(), |_| true) - .map(|best| (best.instance.clone(), best.takeover.clone())) + .map(|best| best.instance.clone()) else { return kio::Pending::new(Lookup::failed(Error::Unroutable)); }; // A request naming another instance than the winner follows that instance's own - // routes: every hop resolves the instance the first one chose. A winner that took - // over the named instance continues it, so the request is the winner's. - let pin = instance - .filter(|instance| { - **instance != winner && takeover.as_ref().is_none_or(|takeover| takeover.instance != **instance) - }) - .cloned(); - // Only the winner's takeover can continue a front this request would join. - let takeover = takeover.filter(|_| pin.is_none()); + // routes: every hop resolves the instance the first one chose. + let pin = instance.filter(|instance| **instance != winner).cloned(); let best = match &pin { None => winner, Some(pin) @@ -7798,9 +7542,7 @@ impl Consumer { let pending = front.request.read(); match &pending.resolved { None => pending.selected == best, - // Or the winner took over what the front resolved, which the front's task - // adopts on its next look at the table. - Some(Ok(resolved)) => *resolved == best || pending.continued_by(takeover.as_ref()), + Some(Ok(resolved)) => *resolved == best, Some(Err(_)) => false, } }; @@ -7834,15 +7576,13 @@ impl Consumer { }); let request = kio::Producer::new(PendingFront { resolved: None, - via: None, - first: None, selected: best, refused: false, }); let consumer = request.consume(); // The front starts held by this request; see [`Front::new`]. let held = broadcast.consume(); - let watch = state.watch_front(&self.shared, &absolute, horizon, request.clone()); + let watch = state.watch(&self.shared, &absolute); state.fronts.insert( key, RemoteFront { @@ -8113,10 +7853,7 @@ impl Routes { /// Drive the egress announce guards for one update. fn hand_out(&mut self, event: RouteEvent) -> RouteEvent { match &event { - RouteEvent::Start(announce) - | RouteEvent::Update(announce) - | RouteEvent::Restart(announce) - | RouteEvent::Takeover { announce, .. } => { + RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { let scope = self.stats.egress(self.root.rejoin(&announce.prefix)); self.guards .entry(announce.prefix.clone()) @@ -8459,11 +8196,7 @@ impl TestSelection { pub(crate) fn new() -> Self { let origin = Hop::new(1).unwrap().produce(); let route = origin - .forward( - "room", - RouteInfo::default().with_via(Hop::new(Self::PEER).unwrap()), - None, - ) + .forward("room", RouteInfo::default().with_via(Hop::new(Self::PEER).unwrap())) .unwrap(); Self { origin, @@ -8485,7 +8218,7 @@ impl TestSelection { /// Another instance of the peer takes the path. pub(crate) fn replace(&mut self) { let route = RouteInfo::default().with_via(Hop::new(Self::PEER).unwrap()); - self.replacement = Some(self.origin.forward("room/a", route, None).unwrap()); + self.replacement = Some(self.origin.forward("room/a", route).unwrap()); } } @@ -8549,7 +8282,7 @@ mod tests { let waiter = kio::Waiter::noop(); let (producer, mut driver) = Producer::new(Config::new(origin(1))); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let broadcast = broadcast::Info::new().produce(); let track = broadcast.reserve_track("video").unwrap(); @@ -9054,7 +8787,7 @@ mod tests { #[cfg(feature = "async")] async fn mount_resolves_on_the_target_front() { let producer = origin(1).produce(); - let server = producer.forward(".svc", RouteInfo::default(), None).unwrap(); + let server = producer.forward(".svc", RouteInfo::default()).unwrap(); let project = mounted(&producer, "p1", &["**"]); let through = project.request(".svc/foo", None); @@ -9241,11 +8974,11 @@ mod tests { Err(Error::Unauthorized) )); assert!(matches!( - project.forward(".svc", RouteInfo::default(), None), + project.forward(".svc", RouteInfo::default()), Err(Error::Unauthorized) )); assert!(matches!( - project.forward(".svc/foo", RouteInfo::default(), None), + project.forward(".svc/foo", RouteInfo::default()), Err(Error::Unauthorized) )); let _cam = project.route("cam", RouteOptions::default()).unwrap(); @@ -9586,7 +9319,6 @@ mod tests { .with_instance(format!("worker-{id}").parse().unwrap(), 0) .with_hops(hops(&[id])) .with_cost(3), - None, ) .unwrap() }) @@ -9657,9 +9389,7 @@ mod tests { async fn a_reprice_is_an_update() { let producer = origin(1).produce(); let mut announced = producer.consume().routes(); - let route = producer - .forward("room", RouteInfo::default().with_cost(3), None) - .unwrap(); + let route = producer.forward("room", RouteInfo::default().with_cost(3)).unwrap(); announced.expect_next_active("room").await; route.update(RouteInfo::default().with_cost(5)).unwrap(); @@ -9680,7 +9410,7 @@ mod tests { let producer = origin(1).produce(); let mut announced = producer.consume().routes(); let route = producer - .forward("room", RouteInfo::default().with_auth(first.clone()), None) + .forward("room", RouteInfo::default().with_auth(first.clone())) .unwrap(); assert_eq!(announced.expect_next_active("room").await.auth, Some(first)); @@ -9714,7 +9444,6 @@ mod tests { RouteInfo::default() .with_instance("a".parse().unwrap(), 0) .with_hops(hops(&[11])), - None, ) .unwrap(); announced.expect_next_active("pool/job-0").await; @@ -9725,7 +9454,6 @@ mod tests { RouteInfo::default() .with_instance("b".parse().unwrap(), 0) .with_hops(hops(&[10])), - None, ) .unwrap(); announced.assert_next_wait(); @@ -9763,12 +9491,12 @@ mod tests { let consumer = producer.consume(); let mut announced = consumer.routes(); let _old = producer - .forward("pool/job-0", RouteInfo::default().with_hops(hops(&[old_hop])), None) + .forward("pool/job-0", RouteInfo::default().with_hops(hops(&[old_hop]))) .unwrap(); announced.expect_next_active("pool/job-0").await; let new = producer - .forward("pool/job-0", RouteInfo::default().with_hops(hops(&[new_hop])), None) + .forward("pool/job-0", RouteInfo::default().with_hops(hops(&[new_hop]))) .unwrap(); assert_eq!( announced.expect_next_restarted("pool/job-0").await.hops, @@ -9795,11 +9523,7 @@ mod tests { claims .scope("", &scopes(&[scope])) .unwrap() - .forward( - prefix, - RouteInfo::default().with_hops(hops(&[hop])).with_cost(cost), - None, - ) + .forward(prefix, RouteInfo::default().with_hops(hops(&[hop])).with_cost(cost)) .unwrap() }; (producer, claim) @@ -9946,10 +9670,10 @@ mod tests { let consumer = producer.consume(); let anonymous = producer - .forward("room", RouteInfo::default().with_hops(hops(&[0])).with_cost(1), None) + .forward("room", RouteInfo::default().with_hops(hops(&[0])).with_cost(1)) .unwrap(); let identified = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(5), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(5)) .unwrap(); let _pending = consumer.request("room/alice", None); @@ -9970,10 +9694,10 @@ mod tests { let demoted = producer .clone() .with_preference(Preference::new(-1)) - .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(1), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(1)) .unwrap(); let unranked = producer - .forward("room", RouteInfo::default().with_hops(hops(&[11])).with_cost(5), None) + .forward("room", RouteInfo::default().with_hops(hops(&[11])).with_cost(5)) .unwrap(); let _pending = consumer.request("room/alice", None); @@ -10005,11 +9729,7 @@ mod tests { let mut announced = producer.consume().routes(); let dynamic = producer - .forward( - "room", - RouteInfo::default().with_instance(instance(), 0).with_cost(5), - None, - ) + .forward("room", RouteInfo::default().with_instance(instance(), 0).with_cost(5)) .unwrap(); announced.expect_next_active("room").await; dynamic.update(dynamic.route().unwrap().with_cost(1)).unwrap(); @@ -10156,7 +9876,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(1), - None, ) .unwrap(); let route = announced.expect_next_active("room/alice").await; @@ -10215,7 +9934,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(1), - None, ) .unwrap(); let request = queued(&server).await; @@ -10339,7 +10057,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(1), - None, ) .unwrap(); let upstream = broadcast::Info::new().produce(); @@ -10351,133 +10068,6 @@ mod tests { assert_eq!(listed(&mut tracks).await, "-a"); } - /// A takeover under an unnamed route, in place, continues the front too: it serves from - /// the successor once the predecessor retires rather than ending with no route. - #[kio::test] - #[cfg(feature = "async")] - async fn a_front_follows_an_unnamed_takeover_in_place() { - let producer = origin(1).produce(); - let consumer = producer.consume(); - let first = producer - .forward("room/alice", RouteInfo::default().with_hops(hops(&[10])), None) - .unwrap(); - let pending = consumer.request("room/alice", None); - let upstream = broadcast::Info::new().produce(); - let _a = upstream.create_track("a", None).unwrap(); - queued(&first).await.serve(&upstream); - let front = pending.await.unwrap(); - let mut tracks = front.tracks().unwrap(); - assert_eq!(listed(&mut tracks).await, "+a"); - assert_eq!(listed(&mut tracks).await, "synced"); - - let second = producer - .forward("room/alice", RouteInfo::default().with_hops(hops(&[10])), Some(&first)) - .unwrap(); - kio::clock::sim::yield_now().await; - drop(first); - - let successor = broadcast::Info::new().produce(); - let _a = successor.create_track("a", None).unwrap(); - let _b = successor.create_track("b", None).unwrap(); - kio::clock::timeout(Duration::from_secs(1), queued(&second)) - .await - .expect("the front asked the successor") - .serve(&successor); - assert_eq!(listed(&mut tracks).await, "+b"); - assert!(!front.is_closed()); - } - - /// A chain of takeovers in place that all land before the front's task looks again - /// carries the front to the last one, though each names only the one before it. - #[kio::test] - #[cfg(feature = "async")] - async fn a_front_follows_a_chain_of_takeovers() { - for named in [false, true] { - for links in [2, 8] { - follows_a_chain_of_takeovers(links, named).await; - } - } - } - - #[cfg(feature = "async")] - async fn follows_a_chain_of_takeovers(links: u64, named: bool) { - let route = |link: u64| { - let route = RouteInfo::default().with_hops(hops(&[10])); - match named { - true => route.with_instance(format!("link-{link}").parse().unwrap(), 0), - false => route, - } - }; - let producer = origin(1).produce(); - let consumer = producer.consume(); - let first = producer.forward("room/alice", route(0), None).unwrap(); - let pending = consumer.request("room/alice", None); - let upstream = broadcast::Info::new().produce(); - let _a = upstream.create_track("a", None).unwrap(); - queued(&first).await.serve(&upstream); - let front = pending.await.unwrap(); - let mut tracks = front.tracks().unwrap(); - assert_eq!(listed(&mut tracks).await, "+a"); - assert_eq!(listed(&mut tracks).await, "synced"); - - // Each link takes over the one before and retires it, with no yield between. - let mut last = first; - for link in 1..=links { - let next = producer.forward("room/alice", route(link), Some(&last)).unwrap(); - drop(std::mem::replace(&mut last, next)); - } - - let successor = broadcast::Info::new().produce(); - let _a = successor.create_track("a", None).unwrap(); - let _b = successor.create_track("b", None).unwrap(); - kio::clock::timeout(Duration::from_secs(1), queued(&last)) - .await - .expect("the front asked the chain's last route") - .serve(&successor); - assert_eq!(listed(&mut tracks).await, "+b", "{links} links, named: {named}"); - assert!(!front.is_closed()); - // The front carries the last instance, so a request now joins it. - let again = consumer.request("room/alice", None).await.unwrap(); - assert!(again.is_clone(&front), "{links} links, named: {named}"); - } - - /// A successor renewed to another instance continues nothing its predecessor served, even - /// before the front resolved through the predecessor looks at the table again. - #[kio::test] - #[cfg(feature = "async")] - async fn a_renewed_successor_continues_nothing() { - let route = |link: u64| { - RouteInfo::default() - .with_hops(hops(&[10])) - .with_instance(format!("link-{link}").parse().unwrap(), 0) - }; - let producer = origin(1).produce(); - let consumer = producer.consume(); - let first = producer.forward("room/alice", route(0), None).unwrap(); - let pending = consumer.request("room/alice", None); - let upstream = broadcast::Info::new().produce(); - queued(&first).await.serve(&upstream); - let resolved = pending.await.unwrap(); - - // Taken over, then renewed, with no yield between. - let second = producer.forward("room/alice", route(1), Some(&first)).unwrap(); - second.update(route(2)).unwrap(); - kio::clock::sim::yield_now().await; - - let pending = consumer.request("room/alice", None); - let renewed = broadcast::Info::new().produce(); - kio::clock::timeout(Duration::from_secs(1), queued(&second)) - .await - .expect("a fresh front asked the renewed route") - .serve(&renewed); - let again = pending.await.unwrap(); - assert_eq!(again.info().instance, route(2).instance); - assert!( - !again.is_clone(&resolved), - "the renewed instance joined the first one's front" - ); - } - /// A path served on request through a covering prefix is not a broadcast anyone /// announced, so it has nothing to list. #[kio::test] @@ -10485,7 +10075,7 @@ mod tests { async fn a_front_through_a_covering_prefix_lists_nothing() { let producer = origin(1).produce(); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let pending = consumer.request("room/alice", None); let upstream = broadcast::Info::new().produce(); let _video = upstream.create_track("video", None).unwrap(); @@ -10506,7 +10096,7 @@ mod tests { let consumer = producer.consume(); let _local = producer.route("room/alice", RouteOptions::default()).unwrap(); - let server = producer.forward("room/alice", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room/alice", RouteInfo::default()).unwrap(); // Were the claim to win, the request would park on its handler forever. let resolved = kio::clock::timeout(Duration::from_secs(1), consumer.request("room/alice", None)) @@ -10564,7 +10154,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(2), - None, ) .unwrap(); announced.expect_next_active("room/alice").await; @@ -10642,7 +10231,6 @@ mod tests { RouteInfo::default() .with_instance(crate::Instance::random(), 0) .with_hops(hops(&[10])), - None, ) .unwrap() }); @@ -10685,7 +10273,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let server = producer - .forward("room/alice", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room/alice", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let pending = consumer.request("room/alice", None); let upstream = broadcast::Info::new().produce(); @@ -10766,7 +10354,7 @@ mod tests { let producer = origin(1).produce(); // A prefix route, which resolves any path beneath it optimistically. let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); @@ -10818,7 +10406,7 @@ mod tests { async fn a_hop_in_a_chain_gets_its_own_front() { let producer = origin(1).produce(); let _incumbent = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let fronts = || producer.shared.lock().fronts.len(); @@ -10829,7 +10417,7 @@ mod tests { // The same publisher, now also reached through the peer. let _echo = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10, 7])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10, 7]))) .unwrap(); let _filtered = peer.request("room/alice", None); assert_eq!(fronts(), 2, "the peer still shares the plain front"); @@ -10845,7 +10433,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); @@ -10906,7 +10494,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", None).unwrap(); @@ -10951,8 +10539,6 @@ mod tests { // Nothing joins this front, and its first selection overwrites the instance. request: kio::Producer::new(PendingFront { resolved: None, - via: None, - first: None, selected: instance(), refused: false, }), @@ -10976,15 +10562,11 @@ mod tests { async fn a_peers_filtered_front_ends_once_unread() { let producer = origin(1).produce(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); // The same publisher, also reached through the peer, at a price the plain front skips. let _echo = producer - .forward( - "room", - RouteInfo::default().with_hops(hops(&[10, 7])).with_cost(100), - None, - ) + .forward("room", RouteInfo::default().with_hops(hops(&[10, 7])).with_cost(100)) .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); @@ -11023,7 +10605,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), - None, ) .unwrap(); view.expect_next_active("room").await; @@ -11040,7 +10621,6 @@ mod tests { RouteInfo::default() .with_instance(instance(), 0) .with_hops(hops(&[10, 7])), - None, ) .unwrap(); drop(incumbent); @@ -11076,7 +10656,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(5), - None, ) .unwrap(); let pending = producer.consume().request("room/alice", None); @@ -11135,7 +10714,6 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[10])) .with_cost(cost), - None, ) .unwrap(); let replacement = broadcast::Info::new().produce(); @@ -11312,8 +10890,6 @@ mod tests { .clone(); let request = kio::Producer::new(PendingFront { resolved: None, - via: None, - first: None, selected, refused: false, }); @@ -11324,7 +10900,7 @@ mod tests { broadcast: held.weak(), }, ); - let watch = state.watch_front(&consumer.shared, &path, Horizon::default(), request.clone()); + let watch = state.watch(&consumer.shared, &path); (request, watch) }; let mut front = FrontServe::new(FrontTask { @@ -11447,12 +11023,8 @@ mod tests { #[cfg(feature = "async")] async fn duplicate_routes_aggregate_until_the_last_leaves() { let producer = origin(1).produce(); - let first = producer - .forward("live", RouteInfo::default().with_cost(3), None) - .unwrap(); - let second = producer - .forward("live", RouteInfo::default().with_cost(1), None) - .unwrap(); + let first = producer.forward("live", RouteInfo::default().with_cost(3)).unwrap(); + let second = producer.forward("live", RouteInfo::default().with_cost(1)).unwrap(); let mut announced = producer.consume().routes(); let Some(Some(RouteEvent::Start(update))) = announced.next().now_or_never() else { @@ -11476,14 +11048,14 @@ mod tests { let producer = origin(1).produce(); let scoped = producer.scope("", &scopes(&["room"])).unwrap(); let _broad = scoped - .forward("", RouteInfo::default(), None) + .forward("", RouteInfo::default()) .expect("an overlapping prefix is accepted"); let _ok = scoped - .forward("room/alice", RouteInfo::default(), None) + .forward("room/alice", RouteInfo::default()) .expect("a contained prefix is accepted"); assert!(matches!( - scoped.forward("other", RouteInfo::default(), None), + scoped.forward("other", RouteInfo::default()), Err(Error::Unauthorized) )); } @@ -11494,7 +11066,7 @@ mod tests { let producer = origin(1).produce(); let scope = Patterns::from("*/chat".parse::().unwrap()); let scoped = producer.scope("", &scope).unwrap(); - let dynamic = scoped.forward("", RouteInfo::default(), None).unwrap(); + let dynamic = scoped.forward("", RouteInfo::default()).unwrap(); let mut matching = producer .consume() @@ -11532,11 +11104,9 @@ mod tests { .scope("", &Patterns::from(pattern.parse::().unwrap())) .unwrap() }; - let _chat = scoped("*/chat") - .forward("", RouteInfo::default().with_cost(1), None) - .unwrap(); + let _chat = scoped("*/chat").forward("", RouteInfo::default().with_cost(1)).unwrap(); let _video = scoped("*/video") - .forward("", RouteInfo::default().with_cost(5), None) + .forward("", RouteInfo::default().with_cost(5)) .unwrap(); let mut video = producer @@ -11569,11 +11139,7 @@ mod tests { let chat = producer .scope("", &scopes(&["room/live/chat"])) .unwrap() - .forward( - "room/live", - RouteInfo::default().with_hops(hops(&[7])).with_cost(1), - None, - ) + .forward("room/live", RouteInfo::default().with_hops(hops(&[7])).with_cost(1)) .unwrap(); assert_eq!(all.expect_next_active("room/live").await.cost, Cost::new(1)); assert_eq!(rooted.expect_next_active("").await.cost, Cost::new(1)); @@ -11583,11 +11149,7 @@ mod tests { let video_route = producer .scope("", &scopes(&["room/live/video"])) .unwrap() - .forward( - "room/live", - RouteInfo::default().with_hops(hops(&[8])).with_cost(5), - None, - ) + .forward("room/live", RouteInfo::default().with_hops(hops(&[8])).with_cost(5)) .unwrap(); all.assert_next_wait(); rooted.assert_next_wait(); @@ -11612,11 +11174,9 @@ mod tests { #[cfg(feature = "async")] async fn root_cursor_keeps_the_more_specific_covering_route() { let producer = origin(1).produce(); - let broad = producer - .forward("room", RouteInfo::default().with_cost(1), None) - .unwrap(); + let broad = producer.forward("room", RouteInfo::default().with_cost(1)).unwrap(); let narrow = producer - .forward("room/live", RouteInfo::default().with_cost(9), None) + .forward("room/live", RouteInfo::default().with_cost(9)) .unwrap(); let mut announced = producer .consume() @@ -11645,7 +11205,7 @@ mod tests { let mut announced = producer.consume().routes(); let dynamic = producer - .forward(&path, RouteInfo::default(), None) + .forward(&path, RouteInfo::default()) .expect("max depth is allowed"); announced.expect_next_active(&path).await; @@ -11658,7 +11218,7 @@ mod tests { async fn dynamic_exclusion_skips_routes_through_the_subscriber() { let producer = origin(1).produce(); let _server = producer - .forward("live", RouteInfo::default().with_hops(hops(&[7])), None) + .forward("live", RouteInfo::default().with_hops(hops(&[7]))) .unwrap(); let mut excluded = producer.consume().excluding(origin(7)).routes(); @@ -11674,7 +11234,7 @@ mod tests { async fn announce_consumer_is_a_stream() { use futures::StreamExt; let producer = origin(1).produce(); - let server = producer.forward("live", RouteInfo::default(), None).unwrap(); + let server = producer.forward("live", RouteInfo::default()).unwrap(); let mut announced = producer.consume().routes(); let mut next = || StreamExt::next(&mut announced).now_or_never(); let Some(Some(RouteEvent::Start(update))) = next() else { @@ -11689,7 +11249,7 @@ mod tests { #[kio::test] async fn dynamic_retracts() { let producer = origin(1).produce(); - let server = producer.forward("live", RouteInfo::default(), None).unwrap(); + let server = producer.forward("live", RouteInfo::default()).unwrap(); let mut announced = producer.consume().routes(); announced.expect_next_active("live").await; @@ -11744,7 +11304,7 @@ mod tests { async fn served_requests_coalesce() { let producer = origin(1).produce(); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let first = consumer.request("room/alice", None); let second = consumer.request("room/alice", None); @@ -11766,7 +11326,7 @@ mod tests { async fn retract_rejects_pending_requests() { let producer = origin(1).produce(); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let pending = consumer.request("room/alice", None); drop(server); @@ -11792,9 +11352,9 @@ mod tests { // Three identical routes, oldest first: the newest identical route wins // requests, and swapping between them emits no announce update. - let standby_server = producer.forward("room", RouteInfo::default(), None).unwrap(); - let second_server = producer.forward("room", RouteInfo::default(), None).unwrap(); - let incumbent_server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let standby_server = producer.forward("room", RouteInfo::default()).unwrap(); + let second_server = producer.forward("room", RouteInfo::default()).unwrap(); + let incumbent_server = producer.forward("room", RouteInfo::default()).unwrap(); let mut following = consumer.follow("room/alice", None).unwrap(); let mut resolving = Box::pin(following.available()); @@ -11822,7 +11382,7 @@ mod tests { async fn split_horizon_skips_routes_through_the_requester() { let producer = origin(1).produce(); let _server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[7])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[7]))) .unwrap(); // The requester's own bytes must not be served back to it. @@ -11848,18 +11408,13 @@ mod tests { let mut announced = producer.consume().routes(); let _ingest = producer - .forward( - "client", - RouteInfo::default().with_hops(hops(&[5])).with_via(origin(5)), - None, - ) + .forward("client", RouteInfo::default().with_hops(hops(&[5])).with_via(origin(5))) .unwrap(); let _gateway = producer.route("gateway", RouteOptions::default()).unwrap(); let _forwarded = peer .forward( "forwarded", RouteInfo::default().with_hops(hops(&[5, 7])).with_via(origin(7)), - None, ) .unwrap(); @@ -11872,9 +11427,7 @@ mod tests { // The mark survives narrowing the handle. let scoped = peer.scope("room", &Patterns::from(Pattern::all())).unwrap(); - let _nested = scoped - .forward("x", RouteInfo::default().with_via(origin(8)), None) - .unwrap(); + let _nested = scoped.forward("x", RouteInfo::default().with_via(origin(8))).unwrap(); assert_eq!( announced.expect_next_active("room/x").await.source(), Source::Peer(origin(8)) @@ -11908,10 +11461,10 @@ mod tests { .with_hops(hops(chain)) .with_via(origin(via)) }; - let near = peer.forward("room", route(&[9, 2], 2), None).unwrap(); + let near = peer.forward("room", route(&[9, 2], 2)).unwrap(); announced.assert_try_next_active("room"); - let _far = peer.forward("room", route(&[9, 3, 4], 4), None).unwrap(); + let _far = peer.forward("room", route(&[9, 3, 4], 4)).unwrap(); drop(near); assert!(announced.try_next().is_none(), "the update is held"); step(Duration::from_millis(1)).await; @@ -11946,8 +11499,8 @@ mod tests { let mut near = Vec::new(); let mut far = Vec::new(); for i in 0..k { - near.push(peer.forward(format!("room/{i}"), route(&[9, 2], 2), None).unwrap()); - far.push(peer.forward(format!("room/{i}"), route(&[9, 3, 4], 4), None).unwrap()); + near.push(peer.forward(format!("room/{i}"), route(&[9, 2], 2)).unwrap()); + far.push(peer.forward(format!("room/{i}"), route(&[9, 3, 4], 4)).unwrap()); } let mut announced = producer.consume().routes(); for _ in 0..k { @@ -11992,12 +11545,12 @@ mod tests { .with_hops(hops(chain)) .with_via(origin(via)) }; - let near = peer.forward("room", route(&[9, 2], 2), None).unwrap(); + let near = peer.forward("room", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("room").await; // The best route changes twice; the consumer sees only the last, once. - let far = peer.forward("room", route(&[9, 3, 4], 4), None).unwrap(); - let farther = peer.forward("room", route(&[9, 5, 6, 7], 7), None).unwrap(); + let far = peer.forward("room", route(&[9, 3, 4], 4)).unwrap(); + let farther = peer.forward("room", route(&[9, 5, 6, 7], 7)).unwrap(); let start = kio::clock::now(); drop(near); drop(far); @@ -12031,30 +11584,30 @@ mod tests { .with_via(origin(via)) }; - let near = peer.forward("named", route(&[9, 2], 2), None).unwrap(); + let near = peer.forward("named", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("named").await; drop(near); announced.expect_next_ended("named").await; let start = kio::clock::now(); - let _stale = peer.forward("named", route(&[9, 3, 4], 4), None).unwrap(); + let _stale = peer.forward("named", route(&[9, 3, 4], 4)).unwrap(); match announced.next().await.unwrap() { RouteEvent::Start(announce) => assert_eq!(announce.route.hops, hops(&[9, 3, 4])), other => panic!("should be a start: got {other:?}"), } assert!(kio::clock::now() - start >= RETRACT_QUARANTINE); - let near = peer.forward("anonymous", route(&[0, 2], 2), None).unwrap(); + let near = peer.forward("anonymous", route(&[0, 2], 2)).unwrap(); announced.expect_next_active("anonymous").await; drop(near); announced.expect_next_ended("anonymous").await; - let _other = peer.forward("anonymous", route(&[0, 3, 4], 4), None).unwrap(); + let _other = peer.forward("anonymous", route(&[0, 3, 4], 4)).unwrap(); announced.expect_next_active("anonymous").await; // Retract and announce cycles at one clock reading queue the prefix once, // whether or not the route names its publisher. let queued = announced.anchoring.len(); for chain in [[9, 2], [0, 2], [9, 2], [0, 2]] { - let churn = peer.forward("churn", route(&chain, 2), None).unwrap(); + let churn = peer.forward("churn", route(&chain, 2)).unwrap(); announced.expect_next_active("churn").await; drop(churn); announced.expect_next_ended("churn").await; @@ -12063,7 +11616,7 @@ mod tests { // A quarantine is forgotten once it ends: the cursor wakes to anchor it, // then again to drop it. - let gone = peer.forward("gone", route(&[9, 2], 2), None).unwrap(); + let gone = peer.forward("gone", route(&[9, 2], 2)).unwrap(); announced.expect_next_active("gone").await; drop(gone); announced.expect_next_ended("gone").await; @@ -12087,12 +11640,11 @@ mod tests { // Publisher 9 ingests at relay 2; relay 3 relays 2's route. let direct = || RouteInfo::default().with_hops(hops(&[9, 2])).with_via(origin(2)); - let first = peer.forward("room", direct(), None).unwrap(); + let first = peer.forward("room", direct()).unwrap(); let relayed = peer .forward( "room", RouteInfo::default().with_hops(hops(&[9, 2, 3])).with_via(origin(3)), - None, ) .unwrap(); announced.expect_next_active("room").await; @@ -12104,7 +11656,7 @@ mod tests { announced.assert_next_wait(); // Relay 2 announces again, and the relayed copy is live with it. - let second = peer.forward("room", direct(), None).unwrap(); + let second = peer.forward("room", direct()).unwrap(); announced.expect_next_active("room").await; assert!(producer.shared.lock().withdrawn.is_empty()); @@ -12126,8 +11678,8 @@ mod tests { let route = RouteInfo::default() .with_instance(instance(), 0) .with_hops(hops(&[9, 2])); - let first = peer.forward("room", route.clone(), None).unwrap(); - let second = peer.forward("room", route.clone(), None).unwrap(); + let first = peer.forward("room", route.clone()).unwrap(); + let second = peer.forward("room", route.clone()).unwrap(); announced.expect_next_active("room").await; announced.assert_next_wait(); let remaining = if old_first { @@ -12150,10 +11702,10 @@ mod tests { let producer = origin(1).produce(); let peer = producer.clone().with_link(Link::Peer); let direct = peer - .forward("room", RouteInfo::default().with_hops(hops(&[9, 2])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[9, 2]))) .unwrap(); let _relayed = peer - .forward("room", RouteInfo::default().with_hops(hops(&[9, 2, 3])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[9, 2, 3]))) .unwrap(); let path = Path::from_static("room/video"); let id = producer @@ -12183,14 +11735,14 @@ mod tests { .with_instance(instance(), 0) .with_hops(hops(&[7])) .with_via(origin(7)); - let _forwarded = peer.forward("room", route.clone(), None).unwrap(); + let _forwarded = peer.forward("room", route.clone()).unwrap(); assert_eq!( announced.expect_next_active("room").await.source(), Source::Peer(origin(7)) ); // The newest identical route wins, so the local twin takes over. - let local = producer.forward("room", route, None).unwrap(); + let local = producer.forward("room", route).unwrap(); match announced.next().await.expect("no next") { RouteEvent::Update(announce) => assert_eq!(announce.route.source(), Source::Local), other => panic!("should be an update: got {other:?}"), @@ -12210,11 +11762,7 @@ mod tests { let mut local = producer.consume().local().routes(); let _forwarded = peer - .forward( - "remote", - RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7)), - None, - ) + .forward("remote", RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7))) .unwrap(); local.assert_next_wait(); @@ -12222,11 +11770,7 @@ mod tests { // and retracts from the local view when the local route goes, even though // the peer's still covers it. let _shadow = peer - .forward( - "both", - RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7)), - None, - ) + .forward("both", RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7))) .unwrap(); let ingest = producer .forward( @@ -12235,7 +11779,6 @@ mod tests { .with_hops(hops(&[5])) .with_via(origin(5)) .with_cost(9), - None, ) .unwrap(); assert_eq!(local.expect_next_active("both").await.source(), Source::Local); @@ -12274,11 +11817,7 @@ mod tests { let mut everyone = producer.consume().routes(); let _core = core - .forward( - "core", - RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7)), - None, - ) + .forward("core", RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7))) .unwrap(); assert_eq!( everyone.expect_next_active("core").await.source(), @@ -12287,11 +11826,7 @@ mod tests { toward_core.assert_next_wait(); let _mesh = peer - .forward( - "mesh", - RouteInfo::default().with_hops(hops(&[8])).with_via(origin(8)), - None, - ) + .forward("mesh", RouteInfo::default().with_hops(hops(&[8])).with_via(origin(8))) .unwrap(); assert_eq!( toward_core.expect_next_active("mesh").await.source(), @@ -12301,11 +11836,7 @@ mod tests { // A path both an upstream and a mesh peer reach: the upstream route is // cheaper, yet the upstream view selects the mesh peer's. let _cheap = core - .forward( - "both", - RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7)), - None, - ) + .forward("both", RouteInfo::default().with_hops(hops(&[7])).with_via(origin(7))) .unwrap(); let mesh = peer .forward( @@ -12314,7 +11845,6 @@ mod tests { .with_hops(hops(&[8])) .with_via(origin(8)) .with_cost(9), - None, ) .unwrap(); assert_eq!( @@ -12345,7 +11875,7 @@ mod tests { async fn handler_rejection_is_final() { let producer = origin(1).produce(); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let pending = consumer.request("room/alice", None); let request = queued(&server).await; @@ -12368,7 +11898,7 @@ mod tests { #[kio::test] #[cfg(feature = "async")] - async fn claim_answer_is_hidden_until_published_with_its_instance() { + async fn claim_answer_is_hidden_until_published_under_the_claims_instance() { let producer = origin(1).produce(); let consumer = producer.consume(); let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); @@ -12385,10 +11915,10 @@ mod tests { prepared.announce().unwrap(); let resolved = pending.await.unwrap(); assert_eq!(resolved.info().instance, prepared.route().unwrap().instance); - assert_ne!( + assert_eq!( resolved.info().instance, claim.route().unwrap().instance, - "a job is a stream of its own" + "the job continues the claim's stream at its path" ); assert!(resolved.track("video").is_ok()); } @@ -12408,13 +11938,13 @@ mod tests { prepared.announce().unwrap(); let instance = prepared.route().and_then(|route| route.instance); - assert!(instance.is_some()); + assert_eq!(instance, claim.route().unwrap().instance, "the claim's instance"); match announced.expect_next().await { - RouteEvent::Takeover { announce, .. } => { + RouteEvent::Start(announce) => { assert_eq!(announce.prefix.as_str(), "room/alice"); assert_eq!(announce.route.instance, instance); } - other => panic!("should take over from the claim: got {other:?}"), + other => panic!("should start beneath the claim: got {other:?}"), } assert_eq!(pending.await.unwrap().info().instance, instance); } @@ -12593,17 +12123,18 @@ mod tests { drop(held); } - /// A takeover withdrawn before the cursor delivers it may have continued what its - /// reader resolved through the claim, so the claim continues nothing more for that - /// reader: the next answer is a start. + /// A job withdrawn while it carries the claim's instance ends that stream at its path, so + /// the claim moves to a fresh instance with it: the next job, at that path or another, + /// starts a new stream rather than splicing into the old one. #[kio::test] #[cfg(feature = "async")] - async fn a_withdrawn_takeover_ends_the_claims_continuity() { + async fn a_withdrawn_job_moves_the_claim_to_a_fresh_instance() { let producer = origin(1).produce(); let consumer = producer.consume(); let mut announced = consumer.routes(); let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); announced.assert_next_active("room"); + let first = claim.instance(); let answer = async |path: &str| { let pending = consumer.request(path, None); @@ -12612,22 +12143,148 @@ mod tests { published }; - let _alice = answer("room/alice").await; + let alice = answer("room/alice").await; + assert_eq!(alice.instance(), first, "the job takes the claim's instance"); + announced.assert_next_active("room/alice"); + + let bob = answer("room/bob").await; + assert_eq!(bob.instance(), first, "jobs at sibling paths share it"); + announced.assert_next_active("room/bob"); + + alice.withdraw(); + let second = claim.instance(); + assert_ne!(second, first, "the claim moved on"); match announced.expect_next().await { - RouteEvent::Takeover { announce, predecessor } => { - assert_eq!(announce.prefix.as_str(), "room/alice"); - assert_eq!(predecessor.as_str(), "room"); - } - other => panic!("should be a takeover: got {other:?}"), + RouteEvent::End(announce) => assert_eq!(announce.prefix.as_str(), "room/alice"), + other => panic!("should end the job: got {other:?}"), } - - answer("room/bob").await.withdraw(); - let _carol = answer("room/carol").await; match announced.expect_next().await { - RouteEvent::Start(announce) => assert_eq!(announce.prefix.as_str(), "room/carol"), - other => panic!("should be a start: got {other:?}"), + RouteEvent::Restart(announce) => { + assert_eq!(announce.prefix.as_str(), "room"); + assert_eq!(announce.route.instance, Some(second.clone())); + } + other => panic!("should restart the claim: got {other:?}"), } - announced.assert_next_wait(); + // The sibling keeps the stream it holds. + assert_eq!(bob.instance(), first); + + let carol = answer("room/carol").await; + assert_eq!(carol.instance(), second, "a later job takes the claim's new instance"); + // Bob ending now spends nothing the claim still lends. + drop(bob); + assert_eq!(claim.instance(), second); + drop(carol); + assert_ne!(claim.instance(), second); + } + + /// A job's route takes the instance its claim lends when it announces, not when it was + /// created: a hidden job outlives a sibling's end and joins the claim's next stream. + #[kio::test] + #[cfg(feature = "async")] + async fn a_hidden_job_takes_the_claims_instance_when_it_announces() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); + + let alice = consumer.request("room/alice", None); + let prepared = queued(&claim).await.route(RouteOptions::hidden()).unwrap(); + let bob = consumer.request("room/bob", None); + let sibling = queued(&claim).await.route(RouteOptions::default()).unwrap(); + bob.await.unwrap(); + let first = claim.instance(); + assert_eq!(sibling.instance(), first); + + drop(sibling); + let second = claim.instance(); + assert_ne!(second, first); + + prepared.announce().unwrap(); + assert_eq!(prepared.instance(), second); + assert_eq!(alice.await.unwrap().info().instance, Some(second)); + } + + /// Once a job ends, a request pinned to its instance is refused without asking the + /// claim, which serves another stream now, while an unpinned one starts the next job. + #[kio::test] + #[cfg(feature = "async")] + async fn a_request_pinned_to_an_ended_job_is_refused() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); + + let pending = consumer.request("room/alice", None); + let job = queued(&claim).await.route(RouteOptions::default()).unwrap(); + let held = pending.await.unwrap(); + let ended = job.instance(); + job.broadcast().close(); + drop(job); + + let pinned = consumer.request("room/alice", ended.clone()).await; + assert!(matches!(pinned, Err(Error::Unroutable))); + assert!( + claim.poll_requested(&kio::Waiter::noop()).is_pending(), + "the claim was asked" + ); + + let pending = consumer.request("room/alice", None); + let restarted = queued(&claim).await.route(RouteOptions::default()).unwrap(); + assert_ne!(restarted.instance(), ended); + let fresh = pending.await.unwrap(); + assert!(!fresh.is_clone(&held), "joined the ended job"); + } + + /// A claim and its job end independently: a job outliving its claim keeps its stream, + /// and a claim outliving its job moves on. + #[kio::test] + #[cfg(feature = "async")] + async fn a_claim_and_its_job_end_independently() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); + let pending = consumer.request("room/alice", None); + let job = queued(&claim).await.route(RouteOptions::default()).unwrap(); + let held = pending.await.unwrap(); + let lent = claim.instance(); + + drop(claim); + assert_eq!(job.instance(), lent); + let again = consumer.request("room/alice", lent.clone()).await.unwrap(); + assert!(again.is_clone(&held), "the job's stream ended with its claim"); + drop(job); + assert!(consumer.request("room/alice", lent).await.is_err()); + } + + /// A claim naming a stable instance lends nothing: its answers mint their own, and their + /// end leaves the claim's instance as it is. A job naming its instance keeps it too. + #[kio::test] + #[cfg(feature = "async")] + async fn named_instances_are_neither_lent_nor_moved() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let stable = instance(); + let claim = producer + .route_prefix("room", RouteOptions::default().with_instance(stable.clone(), 0)) + .unwrap(); + + let pending = consumer.request("room/alice", None); + let minted = queued(&claim).await.route(RouteOptions::default()).unwrap(); + pending.await.unwrap(); + assert_ne!(minted.instance(), stable, "a stable claim lent its instance"); + drop(minted); + assert_eq!(claim.instance(), stable); + + let lending = producer.route_prefix("lent", RouteOptions::default()).unwrap(); + let lent = lending.instance(); + let named = instance(); + let pending = consumer.request("lent/bob", None); + let job = queued(&lending) + .await + .route(RouteOptions::default().with_instance(named.clone(), 0)) + .unwrap(); + pending.await.unwrap(); + assert_eq!(job.instance(), named); + drop(job); + assert_eq!(lending.instance(), lent, "a named job's end moved the claim"); } /// A request pinned to an instance resolves only through the answer published under @@ -12659,11 +12316,9 @@ mod tests { async fn refusal_never_falls_through() { let producer = origin(1).produce(); let consumer = producer.consume(); - let catch_all = producer.forward("", RouteInfo::default(), None).unwrap(); - let winner = producer.forward("room", RouteInfo::default(), None).unwrap(); - let sibling = producer - .forward("room", RouteInfo::default().with_cost(2), None) - .unwrap(); + let catch_all = producer.forward("", RouteInfo::default()).unwrap(); + let winner = producer.forward("room", RouteInfo::default()).unwrap(); + let sibling = producer.forward("room", RouteInfo::default().with_cost(2)).unwrap(); let mut pending = Box::pin(consumer.request("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); @@ -12689,7 +12344,7 @@ mod tests { async fn superseded_refusal_does_not_end_the_request() { let producer = origin(1).produce(); let consumer = producer.consume(); - let stale = producer.forward("room", RouteInfo::default(), None).unwrap(); + let stale = producer.forward("room", RouteInfo::default()).unwrap(); let mut pending = Box::pin(consumer.request("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); @@ -12716,13 +12371,13 @@ mod tests { async fn refusal_from_a_beaten_route_asks_the_new_winner() { let producer = origin(1).produce(); let consumer = producer.consume(); - let stale = producer.forward("room", RouteInfo::default(), None).unwrap(); + let stale = producer.forward("room", RouteInfo::default()).unwrap(); let mut pending = Box::pin(consumer.request("room/alice", None)); assert!((&mut pending).now_or_never().is_none()); let request = queued(&stale).await; - let winner = producer.forward("room/alice", RouteInfo::default(), None).unwrap(); + let winner = producer.forward("room/alice", RouteInfo::default()).unwrap(); request.reject(Error::NotFound); let source = broadcast::Info::new().produce(); queued(&winner).await.serve(&source); @@ -12742,7 +12397,7 @@ mod tests { async fn follow_reports_teardown_as_closed() { let (producer, driver) = Producer::new(Config::new(origin(1))); let consumer = producer.consume(); - let _server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let _server = producer.forward("room", RouteInfo::default()).unwrap(); // Park on the covering route, past the loop's closed check. let mut following = consumer.follow("room/alice", None).unwrap(); @@ -12786,7 +12441,7 @@ mod tests { async fn late_track_on_a_served_front_replays() { let producer = origin(1).produce(); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let source = broadcast::Info::new().produce(); for name in ["a", "b"] { @@ -12827,7 +12482,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); - let broad_server = producer.forward("", RouteInfo::default(), None).unwrap(); + let broad_server = producer.forward("", RouteInfo::default()).unwrap(); // A narrow advertise-only claim: requests under it must NOT route to the // broad server; they fall through to the (absent) fallback handler. let _narrow = producer.announce(".dash", RouteInfo::default()).unwrap(); @@ -12852,7 +12507,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let mut announced = consumer.routes(); - let dynamic = producer.forward("", RouteInfo::default(), None).unwrap(); + let dynamic = producer.forward("", RouteInfo::default()).unwrap(); // The root claim is advertised like any other prefix. announced.expect_next_active("").await; @@ -12886,7 +12541,7 @@ mod tests { #[cfg(feature = "async")] async fn out_of_scope_request_never_reaches_the_dynamic_handler() { let producer = origin(1).produce(); - let dynamic = producer.forward("", RouteInfo::default(), None).unwrap(); + let dynamic = producer.forward("", RouteInfo::default()).unwrap(); let scoped = producer.consume().scope("", &scopes(&["tenant-a"])).unwrap(); // `tenant-a-other` shares a character prefix but not a segment, so this @@ -12980,7 +12635,7 @@ mod tests { let mut announced = consumer.routes(); announced.expect_next_active("room").await; - let _server = producer.forward("served", RouteInfo::default(), None).unwrap(); + let _server = producer.forward("served", RouteInfo::default()).unwrap(); let pending = consumer.request("served/path", None); drop(driver); @@ -13031,7 +12686,7 @@ mod tests { #[test] fn internal_route_requests_are_closed_on_teardown() { let (producer, mut driver) = Producer::new(Config::new(origin(1))); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let _pending = producer.consume().request("room/alice", None); driver.poll(Instant::now(), &kio::Waiter::noop()).unwrap(); let Poll::Ready(Some(request)) = server.poll_requested(&kio::Waiter::noop()) else { @@ -13059,7 +12714,7 @@ mod tests { for teardown in [false, true] { let (producer, mut driver) = Producer::new(Config::new(origin(1))); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); let _pending = producer.consume().request("room/alice", None); driver.poll(Instant::ZERO, &kio::Waiter::noop()).unwrap(); let channel = server @@ -13117,7 +12772,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(first)), - None, ) .unwrap(); @@ -13164,7 +12818,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(first)), - None, ) .unwrap() } @@ -13216,7 +12869,6 @@ mod tests { .with_instance(instance(), 1) .with_hops(hops(&[11])) .with_cost(9), - None, ) .unwrap(); assert_resumes(&mut rig, &newer).await; @@ -13242,7 +12894,6 @@ mod tests { RouteInfo::default() .with_instance(name.parse().unwrap(), 0) .with_hops(hops(&[hop])), - None, ) .unwrap() }; @@ -13281,7 +12932,6 @@ mod tests { RouteInfo::default() .with_instance(name.parse().unwrap(), 0) .with_hops(hops(&[hop])), - None, ) .unwrap() }; @@ -13336,7 +12986,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let first = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(5), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10])).with_cost(5)) .unwrap(); let pending = consumer.request("room/alice", None); let source = broadcast::Info::new().produce(); @@ -13347,7 +12997,7 @@ mod tests { deliver(&track, &mut subscription, b"first").await; let second = producer - .forward("room", RouteInfo::default().with_hops(hops(&[11])).with_cost(1), None) + .forward("room", RouteInfo::default().with_hops(hops(&[11])).with_cost(1)) .unwrap(); deliver(&track, &mut subscription, b"pinned").await; @@ -13477,11 +13127,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let old = producer - .forward( - "room", - RouteInfo::default().with_instance(crate::Instance::random(), 0), - None, - ) + .forward("room", RouteInfo::default().with_instance(crate::Instance::random(), 0)) .unwrap(); let pending = consumer.request("room/alice", None); let request = queued(&old).await; @@ -13517,7 +13163,7 @@ mod tests { let consumer = producer.consume(); let old = crate::Instance::random(); let dynamic = producer - .forward("room", RouteInfo::default().with_instance(old.clone(), 0), None) + .forward("room", RouteInfo::default().with_instance(old.clone(), 0)) .unwrap(); let pending = consumer.request("room/alice", None); let pinned = consumer.request("room/alice", old); @@ -13577,11 +13223,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let dynamic = producer - .forward( - "room", - RouteInfo::default().with_instance(crate::Instance::random(), 0), - None, - ) + .forward("room", RouteInfo::default().with_instance(crate::Instance::random(), 0)) .unwrap(); let pending = consumer.request("room/alice", None); queued(&dynamic).await.reject(Error::NotFound); @@ -13608,7 +13250,7 @@ mod tests { let consumer = producer.consume(); let old = crate::Instance::random(); let dynamic = producer - .forward("room", RouteInfo::default().with_instance(old.clone(), 0), None) + .forward("room", RouteInfo::default().with_instance(old.clone(), 0)) .unwrap(); let pending = consumer.request("room/alice", None); let request = queued(&dynamic).await; @@ -13838,7 +13480,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), - None, ) .unwrap(); // The same publisher, reached through the requesting peer. @@ -13848,7 +13489,6 @@ mod tests { RouteInfo::default() .with_instance(instance(), 0) .with_hops(hops(&[10, 7])), - None, ) .unwrap(); @@ -13912,7 +13552,7 @@ mod tests { // Session A: an assigned anonymous hop serving the path. let server_a = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); // Session C: a third anonymous session, excluding the hop the server @@ -13968,7 +13608,7 @@ mod tests { // Session B attaches at the same path. Its front is served immediately. let server_b = producer - .forward("room", RouteInfo::default().with_hops(hops(&[20])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[20]))) .unwrap(); let pending = consumer.request("room/alice", None); let request = queued(&server_b).await; @@ -14094,7 +13734,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); // The front selected the covering route and waits on its answer. @@ -14116,14 +13756,14 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let covering = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let pending = consumer.request("room/alice", None); let _beaten = queued(&covering).await; // The exact path wins, and the front asks it in turn. let exact = producer - .forward("room/alice", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room/alice", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let request = queued(&exact).await; let again = consumer.request("room/alice", None); @@ -14197,7 +13837,7 @@ mod tests { let producer = origin(1).produce(); let consumer = producer.consume(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let pending = consumer.request("room/alice", None); @@ -14245,7 +13885,7 @@ mod tests { async fn a_closed_source_is_not_requested_again_from_its_standing_route() { let producer = origin(1).produce(); let server = producer - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let pending = producer.consume().request("room/alice", None); let source = broadcast::Info::new().produce(); @@ -14337,7 +13977,7 @@ mod tests { async fn a_held_group_continues_after_its_track_goes_unread() { let producer = origin(1).produce(); let server = producer - .forward("live", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("live", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let pending = producer.consume().request("live", None); let source = broadcast::Info::new().produce(); @@ -14429,7 +14069,7 @@ mod tests { let mid = origin(2).produce(); let mid_server = mid - .forward("room", RouteInfo::default().with_hops(hops(&[10])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[10]))) .unwrap(); let mid_pending = mid.consume().request("room/alice", None); queued(&mid_server).await.serve(&leaf_front); @@ -14437,7 +14077,7 @@ mod tests { let edge = origin(3).produce(); let edge_server = edge - .forward("room", RouteInfo::default().with_hops(hops(&[20])), None) + .forward("room", RouteInfo::default().with_hops(hops(&[20]))) .unwrap(); let edge_pending = edge.consume().request("room/alice", None); queued(&edge_server).await.serve(&mid_resolved); @@ -14570,7 +14210,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), - None, ) .unwrap(); let pending = producer.consume().request("room/alice", None); @@ -14820,7 +14459,6 @@ mod tests { .forward( "room", RouteInfo::default().with_instance(instance(), 0).with_hops(hops(&[10])), - None, ) .unwrap(); let pending = producer.consume().request("room/alice", None); @@ -15281,7 +14919,7 @@ mod tests { let (producer, driver) = Producer::new(Config::new(origin(1))); let run = kio::clock::sim::spawn(crate::time::run(driver)); let consumer = producer.consume(); - let server = producer.forward("room", RouteInfo::default(), None).unwrap(); + let server = producer.forward("room", RouteInfo::default()).unwrap(); drop(producer); let pending = consumer.request("room/alice", None); @@ -15395,7 +15033,7 @@ mod tests { async fn source_selection_leases_survive_reprices_but_never_revive_pending_work() { let producer = origin(1).produce(); let broad = producer - .forward("room", RouteInfo::default().with_via(origin(2)), None) + .forward("room", RouteInfo::default().with_via(origin(2))) .unwrap(); let selected = broad.request_validity(&Path::from_static("room/a")).unwrap(); let pending = selected.current(); @@ -15404,11 +15042,11 @@ mod tests { assert!(!pending.is_closed()); // Another peer's winning route cannot change what a request on peer 2 names. let other = producer - .forward("room/a", RouteInfo::default().with_via(origin(3)), None) + .forward("room/a", RouteInfo::default().with_via(origin(3))) .unwrap(); assert!(!pending.is_closed()); let replacement = producer - .forward("room/a", RouteInfo::default().with_via(origin(2)), None) + .forward("room/a", RouteInfo::default().with_via(origin(2))) .unwrap(); assert!(pending.is_closed()); assert!(selected.current().is_closed()); @@ -15430,7 +15068,7 @@ mod tests { let producer = origin(1).produce(); let path = Path::from_static("room/a"); let first = producer - .forward("room", RouteInfo::default().with_via(origin(2)), None) + .forward("room", RouteInfo::default().with_via(origin(2))) .unwrap(); let selected = first.request_validity(&path).unwrap(); let answering = selected.answering(); @@ -15439,14 +15077,13 @@ mod tests { .forward( "room", RouteInfo::default().with_via(origin(2)).with_cost(Cost::from(7)), - None, ) .unwrap(); assert!(!answering.is_closed()); drop(worse); let replacement = producer - .forward("room/a", RouteInfo::default().with_via(origin(2)), None) + .forward("room/a", RouteInfo::default().with_via(origin(2))) .unwrap(); assert!(answering.is_closed()); // Winning again opens a new lease, which a withdrawal leaves open. @@ -15459,7 +15096,7 @@ mod tests { // Announced again, the path has a new instance: a replacement. let _again = producer - .forward("room", RouteInfo::default().with_via(origin(2)), None) + .forward("room", RouteInfo::default().with_via(origin(2))) .unwrap(); assert!(answering.is_closed()); } @@ -15468,7 +15105,7 @@ mod tests { async fn source_selection_unregisters_without_retaining_the_source() { let producer = origin(1).produce(); let route = producer - .forward("room", RouteInfo::default().with_via(origin(2)), None) + .forward("room", RouteInfo::default().with_via(origin(2))) .unwrap(); let path = Path::from_static("room/a"); let selected = route.request_validity(&path).unwrap(); @@ -15527,7 +15164,7 @@ mod tests { ..Default::default() }); let consumer = producer.consume(); - let _served = producer.forward("room", RouteInfo::default(), None).unwrap(); + let _served = producer.forward("room", RouteInfo::default()).unwrap(); // A consumer outlives its producer by design, so the task set can refuse // submissions while the origin is still open. The front's task is then // dropped on the spot, taking its `Watch` with it: the request must not diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index e4a280230bc..be7b552311c 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -453,7 +453,7 @@ impl Model { .with_hops(hops) // The second instance's routes all undercut the first's. .with_cost(((INSTANCES - instance) * SLOTS - slot) as u64); - let requests = self.origin.forward(PATH, route, None).unwrap(); + let requests = self.origin.forward(PATH, route).unwrap(); let broadcast = broadcast::Info::new().produce(); let track = broadcast.create_track("video", track::Info::timed()).unwrap(); let upstream = broadcast.consume(); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index fd0e3e130e0..472cb643476 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -36,11 +36,6 @@ pub(crate) struct Claim { pub epoch: u64, /// The origin's Announce Cost. pub cost: u64, - /// The claim of the same identity this one took over, at or above its path, and the - /// instance that claim served: what was resolved through it beneath this path is - /// this claim's broadcast. Kept while the instance stays, and sent as - /// ANNOUNCE_TAKEOVER to a peer still holding that claim. - pub continues: Option<(Path, Instance)>, } /// A shown claim as the origin serves it through the selected route. @@ -331,22 +326,13 @@ impl Node { .expect("a node always declares an identity") } - /// This node's own claim at `path` changed: `None` withdraws it. A claim taking over - /// the own claim at `predecessor` continues the instance that one serves now. - pub fn own(&self, path: Path, claim: Option, predecessor: Option<&Path>) { + /// This node's own claim at `path` changed: `None` withdraws it. + pub fn own(&self, path: Path, claim: Option) { let mut state = self.lock(); let changed = match claim { - Some(mut claim) => { + Some(claim) => { let view = state.router.declared().and_then(|id| state.router.view(id)); let current = view.and_then(|view| view.items.get(&path)); - claim.continues = match predecessor { - Some(predecessor) => (view.and_then(|view| view.items.get(predecessor))) - .map(|taken| (predecessor.clone(), taken.instance.clone())), - // A re-price or a newer epoch of the same instance continues what it did. - None => current - .filter(|current| current.instance == claim.instance) - .and_then(|current| current.continues.clone()), - }; let changed = current != Some(&claim); changed && state.router.announce(path, claim).is_ok() } @@ -660,7 +646,6 @@ mod tests { instance: instance.clone(), epoch: 0, cost, - continues: None, } } @@ -695,7 +680,7 @@ mod tests { let (origin, relay) = (Node::new(1, 1), Node::new(2, 2)); let answer = origin.answer(Link::default(), Path::empty()).unwrap(); let receive = relay.receive(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 5)), None); + origin.own(p("live/x"), Some(claim(&instance, 5))); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -711,7 +696,7 @@ mod tests { }] ); - origin.own(p("live/x"), None, None); + origin.own(p("live/x"), None); pump(&answer, &receive); assert_eq!( served(outputs(&receive)), @@ -739,7 +724,7 @@ mod tests { let far_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let near = relay.receive(Link::default(), Path::empty()).unwrap(); let near_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0)), None); + origin.own(p("live/x"), Some(claim(&instance, 0))); pump(&far_answer, &far); pump(&near_answer, &near); let metric = |outputs: Vec| match &served(outputs)[..] { @@ -778,8 +763,8 @@ mod tests { }; let b = relay.receive(costly, p("live/b")).unwrap(); let b_answer = origin.answer(Link::default(), p("live/b")).unwrap(); - origin.own(p("live/a/x"), Some(claim(&instance, 0)), None); - origin.own(p("live/b/y"), Some(claim(&instance, 0)), None); + origin.own(p("live/a/x"), Some(claim(&instance, 0))); + origin.own(p("live/b/y"), Some(claim(&instance, 0))); pump(&a_answer, &a); pump(&b_answer, &b); let claims = |outputs: Vec| -> Vec { @@ -803,9 +788,9 @@ mod tests { let answer = origin.answer(Link::default(), Path::empty()).unwrap(); outputs(&answer); for cost in 0..1000 { - origin.own(p("live/x"), Some(claim(&instance, cost)), None); - origin.own(p("live/churn"), Some(claim(&instance, cost)), None); - origin.own(p("live/churn"), None, None); + origin.own(p("live/x"), Some(claim(&instance, cost))); + origin.own(p("live/churn"), Some(claim(&instance, cost))); + origin.own(p("live/churn"), None); } let read = outputs(&answer); let batch = |read: &[Output]| match read { @@ -817,9 +802,9 @@ mod tests { // Once sent, a claim's end is kept, behind what started. for cost in 0..1000 { - origin.own(p("live/y"), Some(claim(&instance, cost)), None); + origin.own(p("live/y"), Some(claim(&instance, cost))); } - origin.own(p("live/x"), None, None); + origin.own(p("live/x"), None); let read = outputs(&answer); assert_eq!(read.len(), 1, "{read:?}"); assert_eq!( @@ -896,7 +881,7 @@ mod tests { let first_answer = origin.answer(Link::default(), Path::empty()).unwrap(); let second = relay.receive(Link::default(), Path::empty()).unwrap(); let second_answer = origin.answer(Link::default(), Path::empty()).unwrap(); - origin.own(p("live/x"), Some(claim(&instance, 0)), None); + origin.own(p("live/x"), Some(claim(&instance, 0))); pump(&first_answer, &first); pump(&second_answer, &second); outputs(&first); diff --git a/rs/moq-net/tests/integration/announce_covering_route.rs b/rs/moq-net/tests/integration/announce_covering_route.rs index c7337a94d88..745fa3a3bf4 100644 --- a/rs/moq-net/tests/integration/announce_covering_route.rs +++ b/rs/moq-net/tests/integration/announce_covering_route.rs @@ -35,7 +35,6 @@ fn drain(announced: &mut moq_net::origin::Routes) -> Vec { Event::Start(announce) => ("Start", announce), Event::Update(announce) => ("Update", announce), Event::Restart(announce) => ("Restart", announce), - Event::Takeover { announce, .. } => ("Takeover", announce), Event::End(announce) => ("End", announce), }; seen.push(format!("{kind} {}", announce.prefix)); diff --git a/rs/moq-net/tests/integration/announce_takeover.rs b/rs/moq-net/tests/integration/announce_takeover.rs deleted file mode 100644 index 90cdb661d58..00000000000 --- a/rs/moq-net/tests/integration/announce_takeover.rs +++ /dev/null @@ -1,267 +0,0 @@ -#![cfg(feature = "async")] - -//! A worker claiming a prefix answers each request by publishing the exact path under a -//! fresh instance. On moq-lite-07 that announcement is a takeover of the claim, so the first -//! subscription through the claim keeps reading, across relays. Older wires carry it as -//! a start, and a takeover that another worker could have served is a start too. - -use crate::support; - -use std::{cell::RefCell, rc::Rc, time::Duration}; - -use moq_net::{Hop, Timestamp, Version, origin, track}; -use support::harness::{MockConnectOptions, MockPair, connect_mock}; - -const TEST_TIMEOUT: Duration = Duration::from_secs(600); - -fn produce_origin(hop: u64) -> origin::Producer { - let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); - support::harness::spawn(driver); - producer -} - -/// Have `to` pull everything `from` publishes, over a link priced at `cost`. -async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: u64) -> MockPair { - let mut options = MockConnectOptions::new(version); - options.server_publish = Some(from.consume()); - options.client_subscribe = Some(to.clone()); - options.cost = Some(cost); - connect_mock(options).await -} - -/// Let every task settle. Time is paused, so this returns once the runtime is idle. -async fn settle() { - kio::clock::sleep(Duration::from_millis(500)).await; -} - -/// A worker claiming `claim`, answering each request with an output whose `video` track -/// carries the worker's name. -struct Worker { - _claim: Rc, - outputs: Rc>>, - origin: origin::Producer, -} - -impl Worker { - fn new(hop: u64, claim: &str, name: &'static str) -> Self { - let origin = produce_origin(hop); - let claim = Rc::new( - origin - .route_prefix(claim, moq_net::origin::RouteOptions::default()) - .unwrap(), - ); - let outputs = Rc::new(RefCell::new(Vec::new())); - let (handler, served) = (claim.clone(), outputs.clone()); - drop(kio::clock::sim::spawn(async move { - while let Some(request) = handler.requested().await { - let prepared = request.route(moq_net::origin::RouteOptions::hidden()).unwrap(); - let track = prepared - .broadcast() - .create_track( - "video", - track::Info::default().with_timescale(moq_net::Timescale::MILLI), - ) - .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, name.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); - prepared.announce().unwrap(); - served.borrow_mut().push((prepared, track)); - } - })); - Self { - _claim: claim, - outputs, - origin, - } - } -} - -/// Every announce event already delivered, as `(kind, prefix, predecessor)`. -async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Option)> { - let mut events = Vec::new(); - while let Ok(event) = kio::clock::timeout(Duration::from_secs(1), announced.next()).await { - events.push(match event.expect("announce ended") { - origin::RouteEvent::Start(a) => ("start", a.prefix.to_string(), None), - origin::RouteEvent::Update(a) => ("update", a.prefix.to_string(), None), - origin::RouteEvent::Restart(a) => ("restart", a.prefix.to_string(), None), - origin::RouteEvent::Takeover { announce, predecessor } => { - ("takeover", announce.prefix.to_string(), Some(predecessor.to_string())) - } - origin::RouteEvent::End(a) => ("end", a.prefix.to_string(), None), - }); - } - events -} - -/// The first frame of `video` at `path` through `origin`, on the first subscription. -async fn watch(origin: &origin::Producer, path: &str) -> String { - let broadcast = origin.consume().request(path, None).await.unwrap(); - let mut subscription = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = kio::clock::timeout(Duration::from_secs(10), subscription.recv_group()) - .await - .expect("subscription hung") - .expect("the first subscription was reset") - .expect("a group"); - let frame = group.read_frame().await.unwrap().expect("a frame"); - String::from_utf8(frame.payload.to_vec()).unwrap() -} - -fn lite07(version: Version) -> bool { - version == "moq-lite-07-wip".parse().unwrap() -} - -/// `W -> R -> E`: the edge reads through `W`'s claim on its first subscription, and each -/// hop announces the answer as a takeover of the claim on lite-07, a start before it. -async fn chain(version: &str, claim: &str) { - let version: Version = version.parse().unwrap(); - let worker = Worker::new(1, claim, "w"); - let relay = produce_origin(2); - let edge = produce_origin(3); - let _upstream = link(version, &worker.origin, &relay, 1).await; - let _downstream = link(version, &relay, &edge, 1).await; - settle().await; - - let mut at_relay = relay.consume().routes(); - let mut at_edge = edge.consume().routes(); - let claimed = [("start", claim.to_string(), None)]; - assert_eq!(drain(&mut at_relay).await, claimed, "{version}"); - assert_eq!(drain(&mut at_edge).await, claimed, "{version}"); - - if !lite07(version) { - // An older wire resets the first subscription through the claim. - let broadcast = edge.consume().request("pool/p", None).await.unwrap(); - drop(broadcast.track("video").unwrap().subscribe(None).await); - settle().await; - } - assert_eq!(watch(&edge, "pool/p").await, "w", "{version}"); - assert_eq!(worker.outputs.borrow().len(), 1, "{version}"); - - let answered = if !lite07(version) { - // An ordinary announcement, or one replacing the claim at its own path. - if claim == "pool/p" { - vec![ - ("end", "pool/p".to_string(), None), - ("start", "pool/p".to_string(), None), - ] - } else { - vec![("start", "pool/p".to_string(), None)] - } - } else { - vec![("takeover", "pool/p".to_string(), Some(claim.to_string()))] - }; - assert_eq!(drain(&mut at_relay).await, answered, "{version}: at the relay"); - assert_eq!(drain(&mut at_edge).await, answered, "{version}: at the edge"); - - // A late snapshot starts what stands, replaying no takeover. - let mut late = edge.consume().routes(); - let mut snapshot = vec![("start", "pool/p".to_string(), None)]; - if claim != "pool/p" { - snapshot.insert(0, ("start", claim.to_string(), None)); - } - assert_eq!(drain(&mut late).await, snapshot, "{version}: a late snapshot"); -} - -macro_rules! chain_tests { - ($($name:ident: $version:literal, $claim:literal,)*) => { - $( - #[kio::test] - async fn $name() { - kio::clock::timeout(TEST_TIMEOUT, chain($version, $claim)) - .await - .expect("timed out"); - } - )* - }; -} - -chain_tests! { - beneath_the_claim_lite06: "moq-lite-06", "pool", - beneath_the_claim_lite07: "moq-lite-07-wip", "pool", - at_the_claim_lite06: "moq-lite-06", "pool/p", - at_the_claim_lite07: "moq-lite-07-wip", "pool/p", -} - -/// Two workers claim `pool` on `R`. Whichever answers, `R` cannot tell a reader of the -/// claim which worker it saw, so the answer is a start, while `R`'s own reader still -/// keeps reading from the worker it was routed to. -#[kio::test] -async fn two_workers_announce_a_start() { - kio::clock::timeout(TEST_TIMEOUT, async { - let version: Version = "moq-lite-07-wip".parse().unwrap(); - let relay = produce_origin(3); - let a = Worker::new(1, "pool", "a"); - let b = Worker::new(2, "pool", "b"); - let _a = link(version, &a.origin, &relay, 1).await; - let _b = link(version, &b.origin, &relay, 10).await; - settle().await; - - let mut announced = relay.consume().routes(); - assert_eq!(drain(&mut announced).await, [("start", "pool".to_string(), None)]); - - assert_eq!(watch(&relay, "pool/p").await, "a"); - assert_eq!( - drain(&mut announced).await, - [("start", "pool/p".to_string(), None)], - "a takeover with another worker's claim visible" - ); - assert!(b.outputs.borrow().is_empty()); - }) - .await - .expect("timed out"); -} - -/// A broadcast published at the path without answering a request is no takeover, even -/// from the worker holding the claim, and a reader through the claim does not inherit it. -#[kio::test] -async fn an_unrequested_publish_is_a_start() { - kio::clock::timeout(TEST_TIMEOUT, async { - let version: Version = "moq-lite-07-wip".parse().unwrap(); - let worker = Worker::new(1, "pool", "w"); - let relay = produce_origin(2); - let _link = link(version, &worker.origin, &relay, 1).await; - settle().await; - - let mut announced = relay.consume().routes(); - assert_eq!(drain(&mut announced).await, [("start", "pool".to_string(), None)]); - - let _direct = worker - .origin - .route("pool/q", moq_net::origin::RouteOptions::default()) - .unwrap(); - settle().await; - assert_eq!(drain(&mut announced).await, [("start", "pool/q".to_string(), None)]); - }) - .await - .expect("timed out"); -} - -/// `W -> R` on lite-07 and `R -> E` on lite-06: the relay keeps its reader through the -/// takeover and forwards it to the older edge as a start. -#[kio::test] -async fn an_older_downstream_gets_a_start() { - kio::clock::timeout(TEST_TIMEOUT, async { - let worker = Worker::new(1, "pool", "w"); - let relay = produce_origin(2); - let edge = produce_origin(3); - let _upstream = link("moq-lite-07-wip".parse().unwrap(), &worker.origin, &relay, 1).await; - let _downstream = link("moq-lite-06".parse().unwrap(), &relay, &edge, 1).await; - settle().await; - - let mut at_relay = relay.consume().routes(); - let mut at_edge = edge.consume().routes(); - drain(&mut at_relay).await; - drain(&mut at_edge).await; - - assert_eq!(watch(&relay, "pool/p").await, "w"); - assert_eq!( - drain(&mut at_relay).await, - [("takeover", "pool/p".to_string(), Some("pool".to_string()))] - ); - assert_eq!(drain(&mut at_edge).await, [("start", "pool/p".to_string(), None)]); - assert_eq!(watch(&edge, "pool/p").await, "w"); - assert_eq!(worker.outputs.borrow().len(), 1, "the edge asked the worker again"); - }) - .await - .expect("timed out"); -} diff --git a/rs/moq-net/tests/integration/announce_to_serve.rs b/rs/moq-net/tests/integration/announce_to_serve.rs index c41ca7e1160..de227426845 100644 --- a/rs/moq-net/tests/integration/announce_to_serve.rs +++ b/rs/moq-net/tests/integration/announce_to_serve.rs @@ -52,7 +52,6 @@ fn drain(announced: &mut moq_net::origin::Routes) -> Vec { Event::Start(announce) => ("Start", announce), Event::Update(announce) => ("Update", announce), Event::Restart(announce) => ("Restart", announce), - Event::Takeover { announce, .. } => ("Takeover", announce), Event::End(announce) => ("End", announce), }; seen.push(format!("{kind} {}", announce.prefix)); diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index 9540a755b62..d2a381d39e0 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -407,8 +407,7 @@ async fn wait_announced(origin: &origin::Consumer, path: &str, active: bool) { let apply = |live: &mut std::collections::HashSet, event: moq_net::origin::RouteEvent| match event { moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) - | moq_net::origin::RouteEvent::Restart(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. } => { + | moq_net::origin::RouteEvent::Restart(update) => { live.insert(update.prefix.to_string()); } moq_net::origin::RouteEvent::End(update) => { diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs new file mode 100644 index 00000000000..cadb6e47685 --- /dev/null +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -0,0 +1,389 @@ +#![cfg(feature = "async")] + +//! A worker claiming a prefix answers each request by publishing the exact path. The exact +//! route carries the claim's instance while it lives, so on moq-lite-07, which carries +//! instances end to end, the first subscription through the claim continues on it, across +//! relays. Once the job ends the claim moves to a fresh instance, so a restarted job is +//! another stream. Older wires mint an instance per announcement, so their first +//! subscription through the claim sees another broadcast rather than a splice. + +use crate::support; + +use std::{cell::RefCell, rc::Rc, time::Duration}; + +use moq_net::{Error, Hop, Instance, Timestamp, Version, origin, track}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TEST_TIMEOUT: Duration = Duration::from_secs(600); + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +fn lite07() -> Version { + "moq-lite-07-wip".parse().unwrap() +} + +fn lite06() -> Version { + "moq-lite-06".parse().unwrap() +} + +/// Have `to` pull everything `from` publishes, over a link priced at `cost`. +async fn link(version: Version, from: &origin::Producer, to: &origin::Producer, cost: u64) -> MockPair { + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(from.consume()); + options.client_subscribe = Some(to.clone()); + options.cost = Some(cost); + connect_mock(options).await +} + +/// Let every task settle. Time is paused, so this returns once the runtime is idle. +async fn settle() { + kio::clock::sleep(Duration::from_millis(500)).await; +} + +/// A worker claiming `claim`, answering each request with an output whose `video` track +/// carries the worker's name and how many outputs it made before. +struct Worker { + claim: Rc, + outputs: Rc>>, + origin: origin::Producer, +} + +impl Worker { + fn new(hop: u64, claim: &str, name: &'static str) -> Self { + let origin = produce_origin(hop); + let claim = Rc::new(origin.route_prefix(claim, origin::RouteOptions::default()).unwrap()); + let outputs = Rc::new(RefCell::new(Vec::new())); + let (handler, served) = (claim.clone(), outputs.clone()); + drop(kio::clock::sim::spawn(async move { + while let Some(request) = handler.requested().await { + let prepared = request.route(origin::RouteOptions::hidden()).unwrap(); + let track = prepared + .broadcast() + .create_track( + "video", + track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .unwrap(); + let tag = format!("{name}{}", served.borrow().len()); + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, tag.into_bytes()).unwrap(); + group.finish().unwrap(); + prepared.announce().unwrap(); + served.borrow_mut().push((prepared, track)); + } + })); + Self { claim, outputs, origin } + } + + /// How many outputs the worker was asked for. + fn served(&self) -> usize { + self.outputs.borrow().len() + } + + /// End the latest output, as a worker does when the job stops. + fn stop(&self) { + let (output, _) = self.outputs.borrow_mut().pop().expect("an output"); + output.broadcast().close(); + } +} + +/// Every announce event already delivered, as `(kind, prefix, instance)`. +async fn drain(announced: &mut origin::Routes) -> Vec<(&'static str, String, Instance)> { + let mut events = Vec::new(); + while let Ok(event) = kio::clock::timeout(Duration::from_secs(1), announced.next()).await { + let (kind, announce) = match event.expect("announce ended") { + origin::RouteEvent::Start(a) => ("start", a), + origin::RouteEvent::Update(a) => ("update", a), + origin::RouteEvent::Restart(a) => ("restart", a), + origin::RouteEvent::End(a) => ("end", a), + }; + events.push((kind, announce.prefix.to_string(), announce.route.instance().clone())); + } + events +} + +/// The events without their instances. +fn kinds(events: &[(&'static str, String, Instance)]) -> Vec<(&'static str, String)> { + events.iter().map(|(kind, prefix, _)| (*kind, prefix.clone())).collect() +} + +/// The first frame of `video` at `path` through `origin` on the first subscription, or +/// `None` when that subscription was reset before a group arrived. +async fn first(origin: &origin::Producer, path: &str) -> Option { + let broadcast = origin.consume().request(path, None).await.unwrap(); + let mut subscription = broadcast.track("video").unwrap().subscribe(None).await.ok()?; + let mut group = kio::clock::timeout(Duration::from_secs(10), subscription.recv_group()) + .await + .expect("subscription hung") + .ok()??; + let frame = group.read_frame().await.unwrap().expect("a frame"); + Some(String::from_utf8(frame.payload.to_vec()).unwrap()) +} + +/// The first frame at `path` through `origin`. Unless the first subscription through the +/// claim `continues`, it resets, and the next one reads. +async fn watch(continues: bool, origin: &origin::Producer, path: &str) -> String { + if !continues { + assert_eq!(first(origin, path).await, None, "the first subscription continued"); + settle().await; + } + first(origin, path).await.expect("the subscription was reset") +} + +/// `W -> R -> E`: the edge reads through `W`'s claim on its first subscription on lite-07, +/// where every hop announces the answer under the claim's instance. An older wire mints an +/// instance per announcement, so its first subscription resets, unless the claim is at the +/// answer's own path: there the worker's answer is the same stream as the claim, so it +/// announces nothing new. +async fn chain(version: Version, claim: &str) { + let worker = Worker::new(1, claim, "w"); + let relay = produce_origin(2); + let edge = produce_origin(3); + let _upstream = link(version, &worker.origin, &relay, 1).await; + let _downstream = link(version, &relay, &edge, 1).await; + settle().await; + + let mut at_relay = relay.consume().routes(); + let mut at_edge = edge.consume().routes(); + let claimed = [("start", claim.to_string())]; + assert_eq!(kinds(&drain(&mut at_relay).await), claimed, "{version}"); + assert_eq!(kinds(&drain(&mut at_edge).await), claimed, "{version}"); + + let continues = version == lite07() || claim == "pool/p"; + assert_eq!(watch(continues, &edge, "pool/p").await, "w0", "{version}"); + assert_eq!(worker.served(), 1, "{version}"); + + let lent = worker.claim.instance(); + for (at, events) in [ + ("relay", drain(&mut at_relay).await), + ("edge", drain(&mut at_edge).await), + ] { + match claim == "pool/p" { + // Beneath the claim, the answer is another path. + false => assert_eq!( + kinds(&events), + [("start", "pool/p".to_string())], + "{version}: at the {at}" + ), + // At the claim's own path, the answer is the same stream by another route. + true => assert!( + events.iter().all(|(kind, _, _)| *kind == "update"), + "{version}: at the {at}: {events:?}" + ), + } + if version == lite07() + && let Some((_, _, instance)) = events.last() + { + assert_eq!(instance, &lent, "{version}: at the {at}"); + } + } + + // A late snapshot starts what stands. + let mut late = edge.consume().routes(); + let mut snapshot = vec![("start", "pool/p".to_string())]; + if claim != "pool/p" { + snapshot.insert(0, ("start", claim.to_string())); + } + assert_eq!(kinds(&drain(&mut late).await), snapshot, "{version}: a late snapshot"); +} + +macro_rules! chain_tests { + ($($name:ident: $version:expr, $claim:literal,)*) => { + $( + #[kio::test] + async fn $name() { + kio::clock::timeout(TEST_TIMEOUT, chain($version, $claim)) + .await + .expect("timed out"); + } + )* + }; +} + +chain_tests! { + beneath_the_claim_lite06: lite06(), "pool", + beneath_the_claim_lite07: lite07(), "pool", + at_the_claim_lite06: lite06(), "pool/p", + at_the_claim_lite07: lite07(), "pool/p", +} + +/// A job that stops moves the claim to a fresh instance, end to end: the next request for +/// the path wakes the worker for another stream, a request pinned to the ended one is +/// refused, and the first subscription through the claim continues on the new job. +#[kio::test] +async fn a_restarted_job_is_another_stream() { + kio::clock::timeout(TEST_TIMEOUT, async { + let worker = Worker::new(1, "pool", "w"); + let relay = produce_origin(2); + let edge = produce_origin(3); + let _upstream = link(lite07(), &worker.origin, &relay, 1).await; + let _downstream = link(lite07(), &relay, &edge, 1).await; + settle().await; + + let mut at_edge = edge.consume().routes(); + drain(&mut at_edge).await; + assert_eq!(first(&edge, "pool/p").await.as_deref(), Some("w0")); + let spent = worker.claim.instance(); + drain(&mut at_edge).await; + + worker.stop(); + settle().await; + let advanced = worker.claim.instance(); + assert_ne!(advanced, spent, "the claim kept the ended job's instance"); + let events = drain(&mut at_edge).await; + assert!( + events.contains(&("end", "pool/p".to_string(), spent.clone())), + "{events:?}" + ); + assert!( + events.contains(&("restart", "pool".to_string(), advanced.clone())), + "{events:?}" + ); + + let pinned = edge.consume().request("pool/p", spent).await.err(); + assert!(matches!(pinned, Some(Error::Unroutable)), "{pinned:?}"); + assert_eq!(worker.served(), 0, "a pinned request woke the worker"); + + assert_eq!(first(&edge, "pool/p").await.as_deref(), Some("w0")); + assert_eq!(worker.served(), 1); + assert_eq!(worker.outputs.borrow()[0].0.instance(), advanced); + }) + .await + .expect("timed out"); +} + +/// Two workers claim `pool` on `R`, each with its own instance. `R`'s reader resolved through +/// the cheaper one continues on that worker's answer, and the other is never asked. +#[kio::test] +async fn two_workers_keep_their_own_readers() { + kio::clock::timeout(TEST_TIMEOUT, async { + let relay = produce_origin(3); + let a = Worker::new(1, "pool", "a"); + let b = Worker::new(2, "pool", "b"); + let _a = link(lite07(), &a.origin, &relay, 1).await; + let _b = link(lite07(), &b.origin, &relay, 10).await; + settle().await; + assert_ne!(a.claim.instance(), b.claim.instance()); + + let mut announced = relay.consume().routes(); + assert_eq!(kinds(&drain(&mut announced).await), [("start", "pool".to_string())]); + + assert_eq!(first(&relay, "pool/p").await.as_deref(), Some("a0")); + assert_eq!( + drain(&mut announced).await, + [("start", "pool/p".to_string(), a.claim.instance())] + ); + assert_eq!(b.served(), 0); + }) + .await + .expect("timed out"); +} + +/// A broadcast published at the path without answering a request is a stream of its own, +/// even from the worker holding the claim. +#[kio::test] +async fn an_unrequested_publish_is_its_own_stream() { + kio::clock::timeout(TEST_TIMEOUT, async { + let worker = Worker::new(1, "pool", "w"); + let relay = produce_origin(2); + let _link = link(lite07(), &worker.origin, &relay, 1).await; + settle().await; + + let mut announced = relay.consume().routes(); + assert_eq!(kinds(&drain(&mut announced).await), [("start", "pool".to_string())]); + + let direct = worker.origin.route("pool/q", origin::RouteOptions::default()).unwrap(); + assert_ne!(direct.instance(), worker.claim.instance()); + settle().await; + assert_eq!( + drain(&mut announced).await, + [("start", "pool/q".to_string(), direct.instance())] + ); + }) + .await + .expect("timed out"); +} + +/// `W -> R` on lite-07 and `R -> E` on lite-06: the relay's reader continues on the answer, +/// which the older edge sees as another broadcast, served by the same output. +#[kio::test] +async fn an_older_downstream_sees_another_broadcast() { + kio::clock::timeout(TEST_TIMEOUT, async { + let worker = Worker::new(1, "pool", "w"); + let relay = produce_origin(2); + let edge = produce_origin(3); + let _upstream = link(lite07(), &worker.origin, &relay, 1).await; + let _downstream = link(lite06(), &relay, &edge, 1).await; + settle().await; + + let mut at_relay = relay.consume().routes(); + let mut at_edge = edge.consume().routes(); + drain(&mut at_relay).await; + drain(&mut at_edge).await; + + assert_eq!(first(&relay, "pool/p").await.as_deref(), Some("w0")); + assert_eq!(kinds(&drain(&mut at_relay).await), [("start", "pool/p".to_string())]); + assert_eq!(kinds(&drain(&mut at_edge).await), [("start", "pool/p".to_string())]); + assert_eq!(first(&edge, "pool/p").await.as_deref(), Some("w0")); + assert_eq!(worker.served(), 1, "the edge asked the worker again"); + }) + .await + .expect("timed out"); +} + +/// `W -> R` on lite-06 and `R -> E` on lite-07: the older hop mints an instance for each +/// announcement, so the edge's first subscription through the claim sees another broadcast +/// rather than a splice, and the next one reads. +#[kio::test] +async fn an_older_upstream_hop_restarts_the_reader() { + kio::clock::timeout(TEST_TIMEOUT, async { + let worker = Worker::new(1, "pool", "w"); + let relay = produce_origin(2); + let edge = produce_origin(3); + let _upstream = link(lite06(), &worker.origin, &relay, 1).await; + let _downstream = link(lite07(), &relay, &edge, 1).await; + settle().await; + + assert_eq!(watch(false, &edge, "pool/p").await, "w0"); + assert_eq!(worker.served(), 1); + }) + .await + .expect("timed out"); +} + +/// `W -> {R1, R2} -> E` on lite-07: both relays carry the claim and its answer under the +/// worker's instance, so the edge's first subscription continues whichever way it routes. +#[kio::test] +async fn a_diamond_continues_the_first_reader() { + kio::clock::timeout(TEST_TIMEOUT, async { + let worker = Worker::new(1, "pool", "w"); + let left = produce_origin(2); + let right = produce_origin(3); + let edge = produce_origin(4); + let _links = [ + link(lite07(), &worker.origin, &left, 1).await, + link(lite07(), &worker.origin, &right, 1).await, + link(lite07(), &left, &edge, 1).await, + link(lite07(), &right, &edge, 2).await, + ]; + settle().await; + + assert_eq!(first(&edge, "pool/p").await.as_deref(), Some("w0")); + assert_eq!(worker.served(), 1); + let mut late = edge.consume().routes(); + let lent = worker.claim.instance(); + assert_eq!( + drain(&mut late).await, + [ + ("start", "pool".to_string(), lent.clone()), + ("start", "pool/p".to_string(), lent), + ] + ); + }) + .await + .expect("timed out"); +} diff --git a/rs/moq-net/tests/integration/dial_split_horizon.rs b/rs/moq-net/tests/integration/dial_split_horizon.rs index 5e5383ba3a6..e7758c179f7 100644 --- a/rs/moq-net/tests/integration/dial_split_horizon.rs +++ b/rs/moq-net/tests/integration/dial_split_horizon.rs @@ -101,8 +101,7 @@ async fn learned_hop(dialer: &origin::Producer, path: &str) -> Hop { let announce = match event { moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Update(announce) - | moq_net::origin::RouteEvent::Restart(announce) - | moq_net::origin::RouteEvent::Takeover { announce, .. } => announce, + | moq_net::origin::RouteEvent::Restart(announce) => announce, moq_net::origin::RouteEvent::End(_) => continue, }; if announce.prefix.as_str() == path { diff --git a/rs/moq-net/tests/integration/drained_claim.rs b/rs/moq-net/tests/integration/drained_claim.rs index f87c986ebef..60a89330f00 100644 --- a/rs/moq-net/tests/integration/drained_claim.rs +++ b/rs/moq-net/tests/integration/drained_claim.rs @@ -86,10 +86,10 @@ impl Worker { } } -/// Read the first frame of `video` at `pool/p` through `relay`. moq-lite-07 announces the -/// answer as a takeover of the claim, so the first subscription reads. Older wires carry it -/// as an ordinary announcement, which resets the first subscription through the claim, so -/// they wake the worker first. +/// Read the first frame of `video` at `pool/p` through `relay`. On moq-lite-07 the answer's +/// exact route carries the claim's instance, so the first subscription reads. Older wires +/// mint an instance per announcement, so the answer resets the first subscription through +/// the claim: they wake the worker first. async fn watch(version: Version, relay: &origin::Producer) -> String { if version != "moq-lite-07-wip".parse().unwrap() { let cold = relay.consume().request("pool/p", None).await.unwrap(); diff --git a/rs/moq-net/tests/integration/extensions.rs b/rs/moq-net/tests/integration/extensions.rs index 24d693f5d2f..daa4a063708 100644 --- a/rs/moq-net/tests/integration/extensions.rs +++ b/rs/moq-net/tests/integration/extensions.rs @@ -46,8 +46,7 @@ fn announced(origin: &origin::Producer, path: &str) -> bool { match event { moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) - | moq_net::origin::RouteEvent::Restart(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. } => { + | moq_net::origin::RouteEvent::Restart(update) => { live |= update.prefix.as_str() == path; } moq_net::origin::RouteEvent::End(update) => { diff --git a/rs/moq-net/tests/integration/lazy_discovery.rs b/rs/moq-net/tests/integration/lazy_discovery.rs index 19cfee840d4..474774aa2a6 100644 --- a/rs/moq-net/tests/integration/lazy_discovery.rs +++ b/rs/moq-net/tests/integration/lazy_discovery.rs @@ -109,9 +109,9 @@ async fn synced(announced: &mut origin::Routes) -> Vec { .expect("synced in time") .expect("the stream stays open"); match event { - origin::RouteEvent::Start(announce) - | origin::RouteEvent::Restart(announce) - | origin::RouteEvent::Takeover { announce, .. } => seen.push(announce.prefix.to_string()), + origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { + seen.push(announce.prefix.to_string()) + } origin::RouteEvent::End(announce) => seen.retain(|prefix| *prefix != announce.prefix.as_str()), origin::RouteEvent::Update(_) => {} } @@ -423,9 +423,9 @@ async fn settled(announced: &mut origin::Routes, held: &mut Vec) -> bool let mut ended = false; while let Some(event) = announced.try_next() { match event { - origin::RouteEvent::Start(announce) - | origin::RouteEvent::Restart(announce) - | origin::RouteEvent::Takeover { announce, .. } => held.push(announce.prefix.to_string()), + origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { + held.push(announce.prefix.to_string()) + } origin::RouteEvent::End(announce) => { ended = true; held.retain(|prefix| *prefix != announce.prefix.as_str()) diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 50416d0c00c..99907916fba 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -8,13 +8,13 @@ //! apply to every test in the binary. mod announce_covering_route; -mod announce_takeover; mod announce_to_serve; mod auth; mod broadcast_close; mod catalog_acquisition; mod catalog_churn; mod catalog_handover; +mod claim_continues; mod compress; mod control_stats; mod datagram; diff --git a/rs/moq-net/tests/integration/multi_cdn.rs b/rs/moq-net/tests/integration/multi_cdn.rs index f293cc7215d..b0311189452 100644 --- a/rs/moq-net/tests/integration/multi_cdn.rs +++ b/rs/moq-net/tests/integration/multi_cdn.rs @@ -133,10 +133,7 @@ async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Option bool { let route = match announce { - origin::RouteEvent::Start(a) - | origin::RouteEvent::Update(a) - | origin::RouteEvent::Restart(a) - | origin::RouteEvent::Takeover { announce: a, .. } => &a.route, + origin::RouteEvent::Start(a) | origin::RouteEvent::Update(a) | origin::RouteEvent::Restart(a) => &a.route, origin::RouteEvent::End(_) => return false, }; route.cost().value() >= A_COST diff --git a/rs/moq-net/tests/integration/rejoin.rs b/rs/moq-net/tests/integration/rejoin.rs index 7e84ee6b8dd..365da0c1897 100644 --- a/rs/moq-net/tests/integration/rejoin.rs +++ b/rs/moq-net/tests/integration/rejoin.rs @@ -836,11 +836,7 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { assert!( matches!( event, - Some( - origin::RouteEvent::Update(_) - | origin::RouteEvent::Restart(_) - | origin::RouteEvent::Takeover { .. } - ) + Some(origin::RouteEvent::Update(_) | origin::RouteEvent::Restart(_)) ), "{version}: {event:?}" ); diff --git a/rs/moq-net/tests/integration/session_panic.rs b/rs/moq-net/tests/integration/session_panic.rs index 08799c1b57e..517a752117b 100644 --- a/rs/moq-net/tests/integration/session_panic.rs +++ b/rs/moq-net/tests/integration/session_panic.rs @@ -83,8 +83,7 @@ async fn until(announced: &mut origin::Routes, path: &str, active: bool) { let (announce, now_active) = match announced.next().await.expect("origin closed") { origin::RouteEvent::Start(announce) | origin::RouteEvent::Update(announce) - | origin::RouteEvent::Restart(announce) - | origin::RouteEvent::Takeover { announce, .. } => (announce, true), + | origin::RouteEvent::Restart(announce) => (announce, true), origin::RouteEvent::End(announce) => (announce, false), }; if announce.prefix.as_str() == path && now_active == active { diff --git a/rs/moq-net/tests/integration/support/mod.rs b/rs/moq-net/tests/integration/support/mod.rs index f9d2a692771..00a79e49950 100644 --- a/rs/moq-net/tests/integration/support/mod.rs +++ b/rs/moq-net/tests/integration/support/mod.rs @@ -58,7 +58,6 @@ pub enum Kind { Start, Update, Restart, - Takeover, End, } @@ -85,7 +84,6 @@ pub fn watch(mut announced: origin::Routes) -> mpsc::UnboundedReceiver<(String, origin::RouteEvent::Start(announce) => (Kind::Start, announce), origin::RouteEvent::Update(announce) => (Kind::Update, announce), origin::RouteEvent::Restart(announce) => (Kind::Restart, announce), - origin::RouteEvent::Takeover { announce, .. } => (Kind::Takeover, announce), origin::RouteEvent::End(announce) => (Kind::End, announce), }; if tx.unbounded_send((announce.prefix.to_string(), kind)).is_err() { diff --git a/rs/moq-net/tests/integration/synced.rs b/rs/moq-net/tests/integration/synced.rs index 5deeaac65e3..462cb0b5ec5 100644 --- a/rs/moq-net/tests/integration/synced.rs +++ b/rs/moq-net/tests/integration/synced.rs @@ -30,9 +30,7 @@ async fn read_until_synced(announced: &mut origin::Routes) -> BTreeSet { .expect("synced in time") .expect("the cursor stays open"); match event { - origin::RouteEvent::Start(announce) - | origin::RouteEvent::Restart(announce) - | origin::RouteEvent::Takeover { announce, .. } => { + origin::RouteEvent::Start(announce) | origin::RouteEvent::Restart(announce) => { seen.insert(announce.prefix.to_string()); } origin::RouteEvent::End(announce) => { diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 0dead0744a9..4a0fa5284ea 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -2569,8 +2569,7 @@ mod tests { match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index fd4955a28eb..30d31152787 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1367,8 +1367,7 @@ mod tests { match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 37a8d1a210e..0fcc07b1ffc 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -1007,8 +1007,7 @@ async fn serve_announced( while let Some(event) = announced.try_next() { if let moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) - | moq_net::origin::RouteEvent::Restart(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. } = event + | moq_net::origin::RouteEvent::Restart(update) = event { broadcasts.push(update.prefix); } diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index b22e68f8ecd..7c69e03d1e2 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -1403,8 +1403,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 6feab4c60f3..1ab265cd9cc 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1816,8 +1816,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 1045ab7944f..b42ca1f96a7 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -429,8 +429,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/connection.rs b/rs/moq-relay/tests/integration/connection.rs index ceee8a968cc..12659b40aaf 100644 --- a/rs/moq-relay/tests/integration/connection.rs +++ b/rs/moq-relay/tests/integration/connection.rs @@ -104,8 +104,7 @@ async fn next_active(announced: &mut moq_net::origin::Routes) -> String { { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => return route.prefix.to_string(), + | moq_net::origin::RouteEvent::Restart(route) => return route.prefix.to_string(), moq_net::origin::RouteEvent::End(_) => {} } } diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 88d65297604..102008e291d 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -599,8 +599,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/goaway_cluster.rs b/rs/moq-relay/tests/integration/goaway_cluster.rs index f9d1b02dea0..bb8b8bddca8 100644 --- a/rs/moq-relay/tests/integration/goaway_cluster.rs +++ b/rs/moq-relay/tests/integration/goaway_cluster.rs @@ -223,8 +223,7 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { // Under another instance, B's route is another source: a restart. if let moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update) - | moq_net::origin::RouteEvent::Restart(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. } = event + | moq_net::origin::RouteEvent::Restart(update) = event && update.route.cost() != moq_net::origin::Cost::DRAIN { break; @@ -875,8 +874,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index 6aa98392bac..36ae6982d5a 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -930,8 +930,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/runtime_workers.rs b/rs/moq-relay/tests/integration/runtime_workers.rs index 2a96182fc7f..60388b3139c 100644 --- a/rs/moq-relay/tests/integration/runtime_workers.rs +++ b/rs/moq-relay/tests/integration/runtime_workers.rs @@ -169,8 +169,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-relay/tests/integration/smoke.rs b/rs/moq-relay/tests/integration/smoke.rs index 2201bb7339b..b6d9d5a89a8 100644 --- a/rs/moq-relay/tests/integration/smoke.rs +++ b/rs/moq-relay/tests/integration/smoke.rs @@ -993,8 +993,7 @@ async fn next_update(announced: &mut moq_net::origin::Routes) -> Option<(moq_net match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-stats/src/aggregate.rs b/rs/moq-stats/src/aggregate.rs index 4d55994184f..6fe28026925 100644 --- a/rs/moq-stats/src/aggregate.rs +++ b/rs/moq-stats/src/aggregate.rs @@ -810,11 +810,8 @@ impl Merged { // unannounce. A closed stream ends the merged view. loop { match self.announce.poll_next(waiter) { - // A takeover continues what was read through its predecessor, like an update. Poll::Ready(Some( - moq_net::origin::RouteEvent::Start(update) - | moq_net::origin::RouteEvent::Update(update) - | moq_net::origin::RouteEvent::Takeover { announce: update, .. }, + moq_net::origin::RouteEvent::Start(update) | moq_net::origin::RouteEvent::Update(update), )) => { let instance = update.route.instance().clone(); changed |= self.apply_announce(&update, instance, true) @@ -914,9 +911,9 @@ impl Merged { } self.last_announced.insert(key.0.clone(), key.1.clone()); - // A route update on a node already tracked (a reprice, or a takeover - // with different metadata) keeps the live reader: existing - // subscriptions survive a takeover, and the reader re-resolves through + // A route update on a node already tracked (a reprice, or another route + // of the same instance) keeps the live reader: existing + // subscriptions survive it, and the reader re-resolves through // the current best route the moment it actually ends. Only an ended // node re-arms here, since a fresh route is new evidence that a // re-resolve could succeed. A sticky contribution carries across the @@ -1147,8 +1144,7 @@ mod tests { match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-stats/src/consume.rs b/rs/moq-stats/src/consume.rs index e1a13551c01..bf9c11ef3a8 100644 --- a/rs/moq-stats/src/consume.rs +++ b/rs/moq-stats/src/consume.rs @@ -180,8 +180,7 @@ mod tests { match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index 5aaf91b2f62..85bd4b21c87 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -1670,8 +1670,7 @@ mod tests { match announced.next().await? { moq_net::origin::RouteEvent::Start(route) | moq_net::origin::RouteEvent::Update(route) - | moq_net::origin::RouteEvent::Restart(route) - | moq_net::origin::RouteEvent::Takeover { announce: route, .. } => Some((route, true)), + | moq_net::origin::RouteEvent::Restart(route) => Some((route, true)), moq_net::origin::RouteEvent::End(route) => Some((route, false)), } } diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index 80194a7b08c..a118810f944 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -822,8 +822,7 @@ async fn next_route(routes: &mut moq_net::origin::Routes) -> (moq_net::announce: moq_net::origin::RouteEvent::End(announce) => (announce, false), moq_net::origin::RouteEvent::Start(announce) | moq_net::origin::RouteEvent::Update(announce) - | moq_net::origin::RouteEvent::Restart(announce) - | moq_net::origin::RouteEvent::Takeover { announce, .. } => (announce, true), + | moq_net::origin::RouteEvent::Restart(announce) => (announce, true), } } From 9c5b2b9d14a5955ef25fa5e59d78b1280390c34c Mon Sep 17 00:00:00 2001 From: quest fixer Date: Fri, 6 Nov 2026 12:56:34 -0800 Subject: [PATCH 30/61] test(net): supply default node in legacy request helpers --- rs/moq-net/src/lite/publisher.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index b65205fe34f..d45217b3a5b 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -9597,6 +9597,7 @@ mod tests { fn legacy_track() -> lite::Track<'static> { lite::Track { instance: None, + node: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), } @@ -9606,6 +9607,7 @@ mod tests { lite::Subscribe { order: crate::track::Order::Descending, instance: None, + node: 0, id: 0, broadcast: crate::Path::from_static("room"), track: "events".into(), From 08b3ea2bd39de03b52a900080bad2f7ed858ac78 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:56:45 -0800 Subject: [PATCH 31/61] test(cli): a player reads through a claim on its first request Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/tests/integration/instances.rs | 145 ++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/rs/moq-cli/tests/integration/instances.rs b/rs/moq-cli/tests/integration/instances.rs index 90c15ddb30e..029df508218 100644 --- a/rs/moq-cli/tests/integration/instances.rs +++ b/rs/moq-cli/tests/integration/instances.rs @@ -283,3 +283,148 @@ versions! { lite06: "moq-lite-06", ietf22: "moq-transport-22", } + +/// A worker claiming `live`, answering each request with a job's exact route: the catalog, +/// which names `./media` under the job's instance, and the media it references. +struct Worker { + claim: origin::Route, + jobs: std::sync::Arc>>, +} + +/// One answered request, held so its route stays advertised. +struct Job { + path: String, + route: origin::Route, + _catalog: Option, + media: Option, +} + +impl Worker { + fn new(origin: &origin::Producer) -> Self { + let claim = origin.route_prefix("live", origin::RouteOptions::default()).unwrap(); + let jobs = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let (handler, answered) = (claim.clone(), jobs.clone()); + tokio::spawn(async move { + while let Some(request) = handler.requested().await { + let path = request.path().to_string(); + let route = request.route(origin::RouteOptions::hidden()).unwrap(); + // Hidden, the job reports the instance it will announce under: the claim's. + let instance = route.instance(); + let (catalog, media) = match path.as_str() { + "live/catalog.hang" => { + let mut catalog = + moq_mux::catalog::Producer::new(&mut route.broadcast().clone(), Default::default()) + .unwrap(); + catalog + .mutate(|catalog| { + let mut audio = + hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + audio.broadcast = Some(moq_net::path::Relative::new("./media").unwrap()); + audio.instance = Some(instance); + catalog.audio.renditions.insert("audio".into(), audio); + }) + .unwrap(); + (Some(catalog), None) + } + _ => { + let media = route + .broadcast() + .create_track( + "audio", + track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .unwrap(); + write(&media, "job:0"); + (None, Some(media)) + } + }; + route.announce().unwrap(); + answered.lock().unwrap().push(Job { + path, + route, + _catalog: catalog, + media, + }); + } + }); + Self { claim, jobs } + } + + fn answered(&self) -> Vec { + self.jobs.lock().unwrap().iter().map(|job| job.path.clone()).collect() + } +} + +/// A player's first catalog read through a worker's claim continues on the job's exact route +/// over a real lite-07 relay, and the catalog's pinned media resolves through the claim to the +/// media job under the same instance. Once that media job ends the claim moves on, so the +/// held pin is refused rather than answered by a restarted job. +#[tokio::test] +async fn a_player_reads_through_a_claim_on_its_first_request() { + let relay = relay().await; + let worker_origin = moq_tokio::origin::spawn(); + let worker = Worker::new(&worker_origin); + let lent = worker.claim.instance(); + let _worker_connection = client(LITE_07) + .with_publisher(&worker_origin) + .connect(relay.url.clone()) + .established() + .await + .unwrap(); + let viewer_origin = moq_tokio::origin::spawn(); + let viewer = viewer_origin.consume(); + let mut routes = viewer.routes(); + let _viewer_connection = client(LITE_07) + .with_subscriber(viewer_origin) + .connect(relay.url.clone()) + .established() + .await + .unwrap(); + assert!(matches!(routes.next().await, Some(origin::RouteEvent::Start(_)))); + + let held = viewer.request("live/catalog.hang", None).await.unwrap(); + let source = moq_mux::Source::new(viewer.clone(), held); + let mut catalog = source + .catalog::<()>(moq_mux::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let snapshot = catalog.next().await.unwrap().unwrap(); + let reference = &snapshot.audio.renditions["audio"]; + assert_eq!( + reference.instance.as_ref(), + Some(&lent), + "the job serves the claim's instance" + ); + let pinned = source + .resolve(reference.broadcast.as_ref(), reference.instance.as_ref()) + .await + .unwrap(); + let mut subscription = pinned.track("audio").unwrap().subscribe(None).await.unwrap(); + assert_eq!(read(&mut subscription).await, "job:0"); + assert_eq!(worker.answered(), ["live/catalog.hang", "live/media"]); + + // The media job ends: the claim moves on, and nothing answers the old pin again. + let media = { + let mut jobs = worker.jobs.lock().unwrap(); + let index = jobs.iter().position(|job| job.media.is_some()).unwrap(); + jobs.remove(index) + }; + media.route.broadcast().close(); + drop(media); + assert_ne!(worker.claim.instance(), lent); + loop { + match routes.next().await.unwrap() { + origin::RouteEvent::End(end) if end.prefix.as_str() == "live/media" => break, + _ => {} + } + } + assert!( + viewer.request("live/media", lent).await.is_err(), + "a request pinned to the ended job resolved" + ); + assert_eq!( + worker.answered(), + ["live/catalog.hang"], + "a pinned request woke the worker" + ); +} From 278923af8c66583ba13085c940b1a24f412a9e45 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 12:57:51 -0800 Subject: [PATCH 32/61] docs: describe claim continuity by instance, drop ANNOUNCE_TAKEOVER Co-Authored-By: Claude Opus 5.5 --- doc/concept/identity.md | 55 ++++++++++++----------- doc/concept/moq-lite.md | 38 ++++++++-------- doc/lib/js/net.md | 2 +- drafts/draft-lcurley-moq-cluster.md | 7 ++- drafts/draft-lcurley-moq-epoch.md | 13 +++--- drafts/draft-lcurley-moq-lite.md | 67 ++++++----------------------- quest/m4/bench-epoch-pin.md | 10 ++--- quest/m4/hls-follow-once.md | 10 ++--- 8 files changed, 80 insertions(+), 122 deletions(-) diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 5b2ac5021cb..fdd27cc2607 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -61,7 +61,7 @@ one is, so it fences out an older run. Nothing assumes it is a clock. | A restarted or replaced publisher | A fresh one | A restart, then the new stream from its start | | A publisher that continues its sequences across restarts | Stable, with a higher epoch each run | The same stream; readers continue without a seam | | Independent publishers of one source | One each | Two streams; see [redundant ingest](/setup/redundancy) | -| A pool of hosts producing paths on demand | One per host and one per job | See [work on demand](#work-on-demand) | +| A pool of hosts producing paths on demand | One per host, carried by its jobs; a fresh one per restart | See [work on demand](#work-on-demand) | ## Choosing a route @@ -94,7 +94,7 @@ it as minted. A minted instance names the announcement, not the publisher's stream, so: - An announcement ending and starting again, such as a job re-announced on one - session, is a new stream, never spliced to its predecessor. + session, is a new stream, never spliced to the earlier one. - Continuity does not survive such a hop: a reconnect of the same publisher through it reads as another stream. Do not rely on seamless failover through an older hop. @@ -161,7 +161,7 @@ or relay now serving another refuses them rather than answer with the wrong stream. A catalog's `instance` is weaker. It never proves a subscription may resume across routes, and against a minted instance it cannot be checked at all. Set it only when you know the stream; omit it for a broadcast produced on -demand, which has none until a worker answers. +demand, whose instance changes each time its job restarts. ## Work on demand @@ -175,12 +175,12 @@ the prefix route without a restart. A job that may restart independently, such as a transcoder claiming `transcode`, instead creates each requested path's own exact route. A claim promises capability, not content. Each requested path is a separate job, -announced at its exact path under a fresh instance of its own. +announced at its exact path. ```rust let claim = origin.route_prefix("transcode", RouteOptions::default())?; while let Some(request) = claim.requested().await { - // A fresh instance for this job, never one shared with another path or run. + // Takes the claim's instance, so readers resolved through the claim continue. // Hidden until its tracks exist. let job = request.route(RouteOptions::hidden())?; let video = job.broadcast().create_track("video", info.clone())?; @@ -193,33 +193,32 @@ while let Some(request) = claim.requested().await { JavaScript mirrors this with `request.route({ hidden: true })` and `job.announce()`; see [`@moq/net`](/lib/js/net) and [`moq-net`](/lib/rs/moq-net). A hidden job route is unannounced until it announces, and dropping it before -then refuses the request. Ending a job's route withdraws only its path: the +then refuses the request. Ending a job's route withdraws its path, and the prefix route keeps answering later requests. A lazy job's lifecycle, as a viewer of `transcode/alice` sees it through a relay: -1. **Request.** The relay resolves the path through the claim and forwards the - request. The worker starts job A and publishes `transcode/alice` under - instance A. -2. **Takeover.** Publishing the answer through the claim proves it was job A, - so job A's route takes over the claim: what the viewer already resolved - through the claim beneath `transcode/alice` keeps playing, now as instance - A. A Rust follower is handed nothing new (a JavaScript follower still moves - to a new broadcast). A different worker, or a job restarted under the same - claim, never inherits it. Only moq-lite 07 carries the takeover - across relays. Older wires, and paths where a relay cannot vouch for it, - send an ordinary announcement instead, so the first subscription through - the claim is reset there: a follower is handed the new broadcast, and a - holder's track ends with an error, and it requests again. Either way the - second request reaches job A. -3. **Steady state.** Requests resolve instance A by its exact route. Routes - carrying instance A through other relays fail over without a seam. -4. **Job restart.** The worker drops job A's route. The path falls back - to the claim, the next request asks the worker again, and job B publishes - under a fresh instance. Followers are handed each new broadcast; relays drop - job A's cached groups, so a viewer returning within the cache window never - gets them. Other paths under the claim, such as `transcode/bob`, are - untouched. +1. **Request.** The relay resolves the path through the claim, under the + claim's instance, and forwards the request. The worker starts a job, whose + route takes the claim's instance and publishes `transcode/alice` under it. +2. **Continue.** The job's exact route carries the instance the viewer + resolved, so on moq-lite 07 every relay keeps the viewer on it: it keeps + playing, and a Rust or JavaScript follower is handed nothing new. Older + wires mint an instance per announcement, so there the first subscription + through the claim is reset: a follower is handed the new broadcast, and a + holder's track ends with an error, and it requests again. The same happens + on any wire under a claim that names a stable instance, whose jobs each take + a fresh one, since a restarted job could not keep that promise. +3. **Steady state.** Requests resolve the job by its exact route. Routes + carrying its instance through other relays fail over without a seam. +4. **Job restart.** The worker drops the job's route, and the claim moves to a + fresh instance at once, so readers see a restart. The next request starts a + new job under the claim's new instance, a new stream; a request pinned to + the old instance is refused as unroutable. Relays drop the old job's cached + groups, so a viewer returning within the cache window never gets them. + Other running jobs, such as `transcode/bob`, keep their instance, while + paths the claim answered with a held broadcast resolve afresh under its new + one. 5. **Done.** Closing the last job leaves only the claim, which serves the next request as in step 1. diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index a926879de9a..9ca3a321f26 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -111,13 +111,13 @@ hidden names work exactly as before. - **One stream, one instance.** Every broadcast mints a random instance and keeps it across reconnects. Replicas of the same content announce that same instance. A prefix claim is its own stream: its worker answers each requested - path by announcing that exact path under the output's own instance. - Restarting one output is then a new instance every hop sees, so a viewer - returning within a relay's cache window never gets the old output's groups. - On moq-lite 07 that announcement is a takeover of the claim: what a viewer - already resolved through the claim keeps playing from the output rather than - being reset to the new route. A different worker, or a replacement output, - never inherits those viewers. + path by announcing that exact path under the claim's instance, so on + moq-lite 07 a viewer resolved through the claim keeps playing (a claim that + names a stable instance gives each output a fresh one instead). When that + output ends, the claim takes a fresh instance, so a restarted output is a + new stream every hop sees, and a viewer returning within a relay's cache + window never gets the old output's groups. A different worker has its own + instance, so it never inherits those viewers. - **Most specific, then highest epoch, then rank.** The most specific prefix wins. Within one instance only its highest epoch is eligible, ahead of cost. Among instances the order is link preference, cost, a broadcast published @@ -138,19 +138,17 @@ hidden names work exactly as before. sequence. A relay drops its copy of the old stream, so a new subscriber never sees its cached groups. -The instance, the epoch, and the restart and takeover messages travel on -moq-lite 07, which is opt-in. `ANNOUNCE_START`, `ANNOUNCE_RESTART` and -`ANNOUNCE_TAKEOVER` carry the instance and epoch, and `TRACK`, `TRACKS`, -`SUBSCRIBE` and `FETCH` name the instance they pin. A restart onto another -instance, or a higher epoch of the same one, is advertised before the old one -is withdrawn, so the path is never absent. Older versions carry a takeover as -an ordinary announcement, so the first subscription through a claim is reset -once the output's route arrives. Older moq-lite versions and moq-transport -carry no instance, so the receiving session mints one for each announcement, -marked with a leading `~`: a reconnect or a re-announcement through such a hop -is another stream. A peer that ignores the end and start may keep cached -objects of the old run under the same names, so an end and a start is not a -cache flush. +The instance, the epoch, and the restart message travel on moq-lite 07, which +is opt-in. `ANNOUNCE_START` and `ANNOUNCE_RESTART` carry the instance and +epoch, and `TRACK`, `TRACKS`, `SUBSCRIBE` and `FETCH` name the instance they +pin. A restart onto another instance, or a higher epoch of the same one, is +advertised before the old one is withdrawn, so the path is never absent. Older +moq-lite versions and moq-transport carry no instance, so the receiving session +mints one for each announcement, marked with a leading `~`: a reconnect or a +re-announcement through such a hop is another stream, and the first +subscription through a claim is reset once the output's route arrives. A peer +that ignores the end and start may keep cached objects of the old run under +the same names, so an end and a start is not a cache flush. Independent importers use distinct instances, even when fed one stream. Only replicas of one publisher's bytes share its instance, such as one publisher diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index f8e77240783..edf1037661f 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -39,7 +39,7 @@ route.announce(); ``` - **The origin holds the routes, not the connection.** `origin.route(path)` creates a route that owns a new broadcast; fill its tracks through `route.broadcast`. Closing a session withdraws its announcements and leaves the routes for the next one. A route is visible at once unless created `{ hidden: true }`, which keeps it invisible, locally and remotely, until `announce()`. `withdraw()` hides it again and `announce()` brings the same stream back. `close()` retracts it and closes its broadcast, as closing the broadcast does. -- **Answer claimed paths with a route.** `origin.routePrefix(prefix)` claims a prefix, and `route.requested()` yields each requested path. Answer with `request.route()`, a new exact route under a fresh instance, or `request.route({ hidden: true })` to fill its broadcast before `announce()` answers the requesters; closing it or its broadcast before then refuses them as unroutable, so `using job = request.route({ hidden: true })` refuses on a throw. Serve a broadcast you already hold with `request.serve(broadcast)`, or refuse with `request.reject(err)`. +- **Answer claimed paths with a route.** `origin.routePrefix(prefix)` claims a prefix, and `route.requested()` yields each requested path. Answer with `request.route()`, a new exact route under the claim's instance (a fresh one if the claim names its own), so readers resolved through the claim continue, or `request.route({ hidden: true })` to fill its broadcast before `announce()` answers the requesters; closing it or its broadcast before then refuses them as unroutable, so `using job = request.route({ hidden: true })` refuses on a throw. Serve a broadcast you already hold with `request.serve(broadcast)`, or refuse with `request.reject(err)`. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, `finish()` each track, then `close()` it; closing a track without finishing aborts it. The broadcast stays announced and keeps serving them until you `close()` it. How long to keep it up for late readers is yours to decide. - **Requests can pin an instance.** `consumer.request(path, { instance })` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances). The resolved broadcast's `instance` names the stream it came through, pinned or not. A route mints a random instance when it is created and keeps it across `withdraw()` and `announce()`; pass `origin.route(path, { instance, epoch })` only for a publisher that continues its group sequences across restarts and raises the epoch each time. - **Follow restarts.** `origin.follow(path)` follows whichever publisher instance serves the path: its `active` changes to a fresh broadcast on a restart, clears on an end, and resolves again on the next start, so read it reactively and close it when done. Instances are never stitched, so subscribe again to each new broadcast. Another route of the same instance taking over (instances travel only on `moq-lite-07-wip`; older wires mint one per announcement) keeps `active` as is, and each track subscription resumes there from the first group it lacks. A refusal ends it. Pass `{ instance }` to pin one stream, which it waits for again after an end. `request(path)` instead keeps its accepted instance and ends once that instance stops serving; while pending, it fails `Unroutable` once discovery of its path has synced without a route for it. To list what is live, iterate `announced(scope)`. See [identity and recovery](/concept/identity) to choose between following and holding one instance. diff --git a/drafts/draft-lcurley-moq-cluster.md b/drafts/draft-lcurley-moq-cluster.md index ee1790eaf8a..89f35ca8d3e 100644 --- a/drafts/draft-lcurley-moq-cluster.md +++ b/drafts/draft-lcurley-moq-cluster.md @@ -277,7 +277,7 @@ Its edge relay then advertises the covering namespace as it would forward that P SUBSCRIBE_TRACKS reaches it through that advertisement. The request stays open ({{moqt}} Section 3.6), so a namespace advertised later is covered when it is advertised. -A receiver identifies a track by its namespace, its name, and, when MoQ Epoch is negotiated, its EPOCH. +A receiver identifies a track by its namespace, its name, and, when MoQ Epoch is negotiated, its instance. It serves each demand for the track's objects, such as a downstream subscription, from the source Path Selection prefers among those eligible for that demand ({{selection}}), and resumes that source by sending REQUEST_UPDATE with Forward set to 1 on its subscription ({{moqt}} Section 9.5); once a source serves no demand, the receiver pauses it again with REQUEST_UPDATE setting Forward to 0. Demands usually share one source, but a downstream peer that the preferred path runs through is served from another, so several sources can be active at once. A later PUBLISH for the track that Path Selection prefers takes over the same way, but only where {{publishers}} allows moving a subscription between the two: the receiver resumes it for the demand it now serves and pauses a source left serving none. @@ -291,8 +291,7 @@ Under this extension an advertisement is a path, so a session advertises a names A namespace names one broadcast whichever publisher serves it, so several publishers of it are redundant copies: their groups are one sequence, and a subscription moves between them without a gap. A publisher MUST NOT reuse a namespace for different content, and publishes a new instance under a new namespace instead. -A session that negotiated MoQ Epoch names the instance instead: the namespace and EPOCH together name the content, so only advertisements with the same EPOCH are redundant copies, and a publisher MAY reuse the namespace for a new instance under a new EPOCH. -A receiver MUST NOT move a subscription between advertisements with different EPOCHs, or between advertisements without one. +A session that negotiated MoQ Epoch names the instance instead, so only advertisements of the same instance are redundant copies, and a publisher MAY reuse the namespace for a new instance (see draft-lcurley-moq-epoch). A refusal never moves to another publisher ({{selection}}). A relay that moves to another publisher MUST update its advertisement to the new path ({{updating}}). @@ -344,7 +343,7 @@ The Key-Value-Pair parity is load-bearing: HOP_PATH is odd, so its value is a le ## moq-cluster-02 - Hop IDs span 64 bits; a link whose integer encoding cannot carry one sends 0 in its place. -- Exactly one NAMESPACE parameter block is appended when any extension that uses it is negotiated, holding the parameters of all of them. Under MoQ Epoch, only advertisements with the same EPOCH are redundant copies, and an update never changes the instance. +- Exactly one NAMESPACE parameter block is appended when any extension that uses it is negotiated, holding the parameters of all of them. Under MoQ Epoch, only advertisements of the same instance are redundant copies, and an update never changes the instance. - A namespace names one broadcast whichever publisher serves it, and a publisher MUST NOT reuse one for different content. A receiver MAY fail a subscription over between any of its advertisements, continuing from the first object the subscriber lacks instead of at a group boundary. Replaces the first-Hop-ID publisher identity. - Defined request resolution against the longest covering prefix; every refusal is terminal, including between several publishers of one namespace, and capacity is signaled only by withdrawing or re-pricing the advertisement. - A relay does not advertise a namespace because it resolved it; the publisher advertises the concrete namespace once producing. diff --git a/drafts/draft-lcurley-moq-epoch.md b/drafts/draft-lcurley-moq-epoch.md index e33d6364316..afcc0b1b484 100644 --- a/drafts/draft-lcurley-moq-epoch.md +++ b/drafts/draft-lcurley-moq-epoch.md @@ -27,7 +27,7 @@ informative: --- abstract This document defines an extension for MoQ Transport {{moqt}} that names the publisher instance behind a namespace. -Peers that negotiate it carry an EPOCH, a UUIDv7 the publisher mints when it starts, on namespace advertisements and on the requests that resolve through them. +Peers that negotiate it carry an EPOCH, a UUIDv7 the publisher mints for each instance, on namespace advertisements and on the requests that resolve through them. A relay can then move a subscription between redundant paths of one instance, and refuse to serve a request from a replacement instance that reused the same names. --- note_Note_to_Readers @@ -42,7 +42,7 @@ Submit an [issue](https://github.com/moq-dev/moq/issues) or [PR](https://github. An endpoint **advertises** a namespace by sending PUBLISH_NAMESPACE, or NAMESPACE in response to a SUBSCRIBE_NAMESPACE. -An **instance** is one run of a publisher of a namespace: the content it produces between starting and stopping. +An **instance** is one continuous sequence space of a namespace: its group sequences never restart and never name two different groups. A namespace and an EPOCH name one instance. @@ -51,7 +51,8 @@ A namespace and an EPOCH name one instance. A publisher that restarts and counts its groups from zero again reuses names for different content, and a relay holding both runs cannot tell them apart. A relay with several paths to one namespace also cannot tell whether they serve the same bytes, so it cannot move a subscription between them without risking a gap or a mix of two runs. -This extension carries the instance identity that {{moql}} carries in its Epoch field, without changing any name. +This extension names the instance as {{moql}}'s Instance does, without changing any name. +{{moql}}'s Epoch, which orders the incarnations of one Instance, has no counterpart: a publisher here never keeps an instance across a restart. The namespace a subscriber asks for is the same with or without it. @@ -97,11 +98,11 @@ Epochs compare as their 16 bytes, which for a UUIDv7 orders them by creation tim # Identity {#identity} -A publisher mints a new Epoch each time it starts an instance, and MUST NOT serve different content under one namespace and Epoch. -Publishers serving identical content, such as replicas of one encoder, MAY share an Epoch. +A publisher mints a new EPOCH each time it starts an instance, such as a restart that counts its groups from zero again, and MUST NOT serve different content under one namespace and EPOCH. +Publishers serving identical content, such as replicas of one encoder, MAY share an EPOCH. Two advertisements of a namespace with the same EPOCH serve the same instance: a receiver MAY move a subscription between them, continuing from the first object the subscriber lacks. -A receiver MUST NOT move a subscription between advertisements with different EPOCHs, or between advertisements without one: it stays on its path and ends with it. +A receiver MUST NOT move a subscription between advertisements of different instances, or between advertisements without EPOCH: it stays on its path and ends with it. An advertisement without EPOCH names no instance, so nothing says another advertisement serves the same bytes. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 246d26ddd5a..e7dfb3b1d6f 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -411,7 +411,6 @@ The publisher replies with a single ANNOUNCE_OK message followed by the matching - ANNOUNCE_END: the origin withdrew a claim. - ANNOUNCE_UPDATE: a claim's cost changed. - ANNOUNCE_RESTART: a claim now serves another Instance, or a higher Epoch of the same one. -- ANNOUNCE_TAKEOVER: a matching claim continues what a live claim served beneath it. - ANNOUNCE_FLOOR: a batch of a route's claims is complete. - SYNCED: the initial set is complete. @@ -423,16 +422,16 @@ The initial set is relative to this hop: a relay answers with what it holds with The publisher MAY coalesce changes to a claim it has not yet sent, and sends no more claims before SYNCED than the initial set holds. A retraction of a claim or route already sent MAY precede SYNCED. -Each ROUTE_START implicitly assigns the next Route ID on the stream, and each ANNOUNCE_START or ANNOUNCE_TAKEOVER the next Announce ID: two counters starting at 0 that increment by 1 per assignment. +Each ROUTE_START implicitly assigns the next Route ID on the stream, and each ANNOUNCE_START the next Announce ID: two counters starting at 0 that increment by 1 per assignment. Neither id appears on the wire where it is assigned; both endpoints derive it from the message order on the (reliable, ordered) stream, and other messages reference it instead of repeating the origin node or the claim's prefix. Ids are never reused within a stream. A ROUTE_END retires its Route ID and the Announce ID of every claim under the route; an ANNOUNCE_END retires one Announce ID. A stream has at most one live route per origin node, and each route at most one live claim per prefix. Another origin claiming the same prefix is another claim, under its own route. -An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping its id live, and an ANNOUNCE_TAKEOVER at its predecessor's own prefix replaces it under a new id; a claim never moves to another route. +An ANNOUNCE_UPDATE or ANNOUNCE_RESTART atomically replaces a claim while keeping its id live; a claim never moves to another route. -The subscriber MUST close the session with a PROTOCOL_VIOLATION if it receives a message referencing a Route ID or Announce ID that was never assigned or already retired, a ROUTE_START for an origin node that already has a live route on the stream, an ANNOUNCE_START or ANNOUNCE_TAKEOVER for a prefix its route already claims other than as its predecessor, or any message before ANNOUNCE_OK. +The subscriber MUST close the session with a PROTOCOL_VIOLATION if it receives a message referencing a Route ID or Announce ID that was never assigned or already retired, a ROUTE_START for an origin node that already has a live route on the stream, an ANNOUNCE_START for a prefix its route already claims, or any message before ANNOUNCE_OK. When the stream is closed, the subscriber MUST assume that every route and claim on it is gone. A broadcast path has at most 32 segments and at most 4,096 bytes; a receiver MUST close the session with a PROTOCOL_VIOLATION on a longer one, including one resolved by [Compression](#announce-compression). @@ -457,9 +456,9 @@ A floor that certifies nothing proves no claim absent: the stream's claims stay While a receiver has a usable route to the origin and every peer whose last advertised route to it is feasible here has answered the current request, it lets go of a claim none of those peers certifies or carries: it goes offline, which is not an ANNOUNCE_END. #### Compression {#announce-compression} -An ANNOUNCE_START or ANNOUNCE_TAKEOVER MAY copy the head of its path from a live claim on the same stream, its base. -`Path Base` names it by distance: the base's Announce ID is the next unassigned Announce ID minus `Path Base`, so 1 names the latest ANNOUNCE_START or ANNOUNCE_TAKEOVER, and 0 names no base. -The receiver resolves the base against the claims live when the message arrives, and the result is identical to the literal encoding: a base ending later changes nothing, including a predecessor that the ANNOUNCE_TAKEOVER naming it as a base retires. +An ANNOUNCE_START MAY copy the head of its path from a live claim on the same stream, its base. +`Path Base` names it by distance: the base's Announce ID is the next unassigned Announce ID minus `Path Base`, so 1 names the latest ANNOUNCE_START, and 0 names no base. +The receiver resolves the base against the claims live when the message arrives, and the result is identical to the literal encoding: a base ending later changes nothing. The base need not share the new claim's route. A publisher MAY always send a `Path Base` of 0. @@ -522,7 +521,6 @@ Claims with the same Instance serve the same Broadcast, so a relay MAY move a li A relay MUST NOT move a subscription to a claim with a different Instance: it stays on its Instance's claims and ends with the last of them. When another Broadcast wins the path, subscriptions already open stay on theirs until the subscriber drops them or the claim goes; new requests resolve the winner, and the ANNOUNCE_RESTART tells subscribers to request again. A receiver of an ANNOUNCE_RESTART (or of the ANNOUNCE_END and ANNOUNCE_START standing in for one) MUST NOT serve a new request under its prefix from anything resolved through the replaced claim. -An ANNOUNCE_TAKEOVER is the one exception to staying on an Instance: what was resolved through its predecessor beneath its prefix is the Broadcast the new claim serves, so those subscriptions continue on the new claim and new requests MAY join them, whatever either claim's Instance. A version before this one carries no Instance, so a receiver of such a session mints one for each ANNOUNCE_START it receives, with Epoch 0, and forwards it as the claim's Instance. A minted Instance SHOULD start with `~` (0x7E), so a reader can tell it names an announcement rather than the publisher's stream; a publisher SHOULD NOT name its own Instance with that prefix. A minted Instance is new for every announcement, so a publisher reconnecting or re-announcing through such a hop is another stream. @@ -543,6 +541,7 @@ When choosing which claim to advertise for a prefix, the requested path is the p A claim resolved this way serves the request like any other, and the claim is not consumed by it. A relay MUST NOT announce a path merely because it resolved it: the covering claim stays the only advertisement until the advertiser announces the concrete path, which it SHOULD do once it is producing, so a later request finds the running broadcast by its exact path instead of resolving a second producer. +The advertiser MAY announce the path under the claim's Instance, so subscriptions resolved through the claim continue on it (see [Routing](#routing)); when that announcement ends, a claim still serving its Instance restarts under a new one, so a restarted answer is another Broadcast. An advertiser that will not serve a resolved request resets the request stream with a typed code (see [Error Codes](#error-codes)). Every refusal is terminal and propagates without re-resolution, so probing unserved paths costs one round trip per path. @@ -1113,7 +1112,7 @@ An Announce Stream ending before SYNCED is a PROTOCOL_VIOLATION; a Tracks Stream ## ROUTE_START {#route-start} A publisher sends a ROUTE_START message when it reaches an origin node (see [Routing](#routing)). -Each ROUTE_START implicitly assigns the next Route ID on the stream, later referenced by ROUTE_UPDATE, ROUTE_HOLD, ROUTE_END, ANNOUNCE_FLOOR, ANNOUNCE_START, and ANNOUNCE_TAKEOVER (see [Announce](#announce)). +Each ROUTE_START implicitly assigns the next Route ID on the stream, later referenced by ROUTE_UPDATE, ROUTE_HOLD, ROUTE_END, ANNOUNCE_FLOOR, and ANNOUNCE_START (see [Announce](#announce)). ~~~ ROUTE_START Message { @@ -1241,7 +1240,7 @@ A publisher MUST close the session with a PROTOCOL_VIOLATION if it receives any ## ANNOUNCE_START {#announce-start} A publisher sends an ANNOUNCE_START message to advertise a claim of a route's origin node: that paths under a prefix can be served. -Each ANNOUNCE_START implicitly assigns the next Announce ID on the stream, later referenced by ANNOUNCE_END, ANNOUNCE_UPDATE, ANNOUNCE_RESTART, and ANNOUNCE_TAKEOVER (see [Announce](#announce)). +Each ANNOUNCE_START implicitly assigns the next Announce ID on the stream, later referenced by ANNOUNCE_END, ANNOUNCE_UPDATE, and ANNOUNCE_RESTART (see [Announce](#announce)). Only the suffix is encoded on the wire, as the full claim prefix can be constructed by prepending the requested prefix. @@ -1282,7 +1281,7 @@ A relay forwards the Instance unchanged. Which incarnation of the Instance the claim serves: a counter the publisher raises within one Instance, 0 unless it keeps an Instance across restarts. A relay forwards the Epoch unchanged. -Only an ANNOUNCE_RESTART or ANNOUNCE_TAKEOVER changes the Instance or the Epoch; ANNOUNCE_UPDATE never does. +Only an ANNOUNCE_RESTART changes the Instance or the Epoch; ANNOUNCE_UPDATE never does. **Announce Cost**: The origin's price for this prefix, in the units of the [Cost Parameter](#cost-parameter): 0 for content it already produces, larger for a standby that starts work on demand. @@ -1308,7 +1307,7 @@ ANNOUNCE_END Message { Set to 0x1 to indicate an ANNOUNCE_END message. **Announce ID**: -The ordinal implicitly assigned by a prior ANNOUNCE_START or ANNOUNCE_TAKEOVER on this stream. +The ordinal implicitly assigned by a prior ANNOUNCE_START on this stream. A claim that is announced again after an ANNOUNCE_END gets a fresh id from its next ANNOUNCE_START. @@ -1362,44 +1361,6 @@ As defined for [ANNOUNCE_END](#announce-end). **Instance**, **Epoch**, and **Announce Cost**: As defined for [ANNOUNCE_START](#announce-start), replacing the claim's. -## ANNOUNCE_TAKEOVER {#announce-takeover} -A publisher sends an ANNOUNCE_TAKEOVER message to advertise a claim that continues what a live claim, its predecessor, served beneath the new claim's prefix. -A worker advertising a prefix under its own Instance answers a request under it by publishing that path under a fresh Instance; the ANNOUNCE_TAKEOVER keeps subscriptions already resolved through the prefix rather than resetting them to the more specific claim. -It is an ANNOUNCE_START in every other respect and assigns the next Announce ID. - -~~~ -ANNOUNCE_TAKEOVER Message { - Type (i) = 0x5 - Message Length (i) - Predecessor (i), - Path Base (i), - Path Keep (i), - Claim Prefix Suffix (s), - Route ID (i), - Instance (b), - Epoch (i), - Announce Cost (i), -} -~~~ - -**Type**: -Set to 0x5 to indicate an ANNOUNCE_TAKEOVER message. - -**Predecessor**: -The Announce ID of a live claim under the same route whose prefix covers the new claim's prefix. -One at the predecessor's own prefix retires the predecessor's id, which this claim replaces; otherwise the predecessor stays live. -The receiver MUST close the session with a PROTOCOL_VIOLATION if the predecessor was never assigned, is already retired, is under another route, or does not cover the new prefix. - -**Path Base**, **Path Keep**, **Claim Prefix Suffix**, **Route ID**, **Instance**, **Epoch**, and **Announce Cost**: -As defined for [ANNOUNCE_START](#announce-start), resolved before the predecessor retires. - -The receiver treats whatever it resolved through the predecessor beneath the new prefix as served by the new claim: subscriptions on it continue, and a new request for those paths MAY join them. -Only what the predecessor's current claim resolved continues; anything resolved before an earlier ANNOUNCE_RESTART of it does not. - -A publisher MUST send an ANNOUNCE_TAKEOVER only when the new claim serves exactly what the subscriber resolved through the predecessor beneath its prefix: the publisher that answered requests through the predecessor now advertises their paths. -A relay forwards one only when the subscriber was last sent that predecessor, and every claim the relay sent at its prefix serves the same Instance; otherwise it sends an ANNOUNCE_START, or an ANNOUNCE_RESTART at the predecessor's own prefix. - - ## ANNOUNCE_FLOOR {#announce-floor} A publisher sends an ANNOUNCE_FLOOR message to close a batch of a route's claims (see [Floors](#floors)). @@ -2156,10 +2117,10 @@ The `Message Length` describes the payload size on the wire. - Replaced ANNOUNCE_OK Active Count with an ordered, empty SYNCED event (0x4), marking the initial set without retaining claims that end before delivery. - Added `Instance` and `Epoch` to ANNOUNCE_START and ANNOUNCE_RESTART, and `Instance` to TRACK, TRACKS, SUBSCRIBE, and FETCH. The Instance is non-empty UTF-8 text naming one continuous stream, and a path and an Instance name one Broadcast; the Epoch is a counter within one Instance. A relay MAY move a subscription between claims with the same Instance, continuing from the first frame the subscriber lacks instead of at a group boundary, and never between claims with different Instances. Within one Instance only its highest Epoch is eligible. A request's Instance is empty when it pins none. Replaces the first-hop identity. - Added ANNOUNCE_RESTART (0x3): a live advertisement now serves another Instance, or a higher Epoch of the same one. -- Added ANNOUNCE_TAKEOVER (0x5): a claim that continues what a live claim served beneath it, such as a job answering a request through a worker's prefix at its exact path, so subscriptions resolved through the prefix are not reset. +- An advertiser MAY announce a path it answers through a claim under the claim's Instance, so subscriptions resolved through the claim continue, and a claim still serving that Instance when the announcement ends restarts under a new one. - A receiver mints an Instance, starting with `~`, for each announcement a session of an earlier version delivers. -- Added announce compression: ANNOUNCE_START and ANNOUNCE_TAKEOVER gain `Path Base` and `Path Keep` to copy the head of a live claim's suffix. -- Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, with a seqno, an additive metric, and a down-only bit, and loops are prevented by Babel's feasibility condition. Every ANNOUNCE_START and ANNOUNCE_TAKEOVER names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin rather than an update per claim. Removed the Hop ID list, ANNOUNCE_OK's Hop ID, and the SETUP Hop parameter (0x5). +- Added announce compression: ANNOUNCE_START gains `Path Base` and `Path Keep` to copy the head of a live claim's suffix. +- Split routing into two layers on the Announce Stream. ROUTE_START (0x6), ROUTE_UPDATE (0x7), and ROUTE_END (0x8) advertise how the publisher reaches an origin node, named by a Node ID, with a seqno, an additive metric, and a down-only bit, and loops are prevented by Babel's feasibility condition. Every ANNOUNCE_START names its route's Route ID in place of the Hop ID list, so a link change sends one ROUTE_UPDATE per origin rather than an update per claim. Removed the Hop ID list, ANNOUNCE_OK's Hop ID, and the SETUP Hop parameter (0x5). - Added ROUTE_HOLD (0xB), which holds a route while it recovers; ROUTE_END lets it go. Neither withdraws the origin's claims, and the subscriber echoes every ROUTE_END back. - Added ROUTE_REQUEST (0xA), sent by the subscriber on the Announce Stream to ask for a route with a newer seqno when every route it has is infeasible. - Added ANNOUNCE_FLOOR (0x9), closing each batch of a route's claims with the origin Revision at which the stream's claims across the requested prefix are certified plus one, or 0 to certify nothing, so a stale copy never revives a claim whose withdrawal a newer one delivered. A floor certifies only the claims its own request shows. diff --git a/quest/m4/bench-epoch-pin.md b/quest/m4/bench-epoch-pin.md index c7ac12d3219..26c4c324cfc 100644 --- a/quest/m4/bench-epoch-pin.md +++ b/quest/m4/bench-epoch-pin.md @@ -2,15 +2,15 @@ ## Goal -moq-bench requests each pooled broadcast pinned to the epoch it saw -announced, so a takeover between gathering and requesting is not counted as -the same broadcast. +moq-bench requests each pooled broadcast pinned to the instance it saw +announced, so another instance taking the path between gathering and +requesting is not counted as the same broadcast. ## Plan Decided 2026-10-29 (maintainer), a follow-up of [moq-dev/moq#7192](https://github.com/moq-dev/moq/pull/7192). Pass the -announced epoch to `request`. A pinned request that finds another instance is -refused; count that as a takeover, not a failure. +announced instance to `request`. A pinned request that finds another instance is +refused; count that as an instance change, not a failure. Public API: none. Wire: none. diff --git a/quest/m4/hls-follow-once.md b/quest/m4/hls-follow-once.md index 88992d2b0f5..6f9952b3ae6 100644 --- a/quest/m4/hls-follow-once.md +++ b/quest/m4/hls-follow-once.md @@ -3,8 +3,8 @@ ## Goal The moq-hls server resolves a path once, with one `follow`, instead of a -`request` for the first answer and a separate follower for takeovers. It -still reports a path nothing serves once discovery has synced. +`request` for the first answer and a separate follower for each new instance +at the path. It still reports a path nothing serves once discovery has synced. ## Plan @@ -15,8 +15,8 @@ nothing serves this path". Add that report to `Following`, the synced absence `request` already answers with `Unroutable`, without making a follower give up. Then move the server onto it. After the cut, since the double resolve works. -Tests: an unserved path is reported once discovery syncs, a takeover is -followed without a second resolution, and the hand-written resolve timeout -stays only a limit on a handler that never answers. +Tests: an unserved path is reported once discovery syncs, a new instance at +the path is followed without a second resolution, and the hand-written +resolve timeout stays only a limit on a handler that never answers. Public API: additive in moq-net. Wire: none. From 7757ecce6015817946d0364291bfdfe25cd433b3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 15:44:13 -0800 Subject: [PATCH 33/61] feat(js)!: a job continues its claim's instance, drop ANNOUNCE_TAKEOVER Co-Authored-By: Claude Opus 5.5 --- js/net/src/integration.test.ts | 55 +++++++- js/net/src/lite/announce.test.ts | 41 ++---- js/net/src/lite/announce.ts | 69 +--------- js/net/src/lite/subscriber.test.ts | 86 +----------- js/net/src/lite/subscriber.ts | 24 +--- js/net/src/lite/version.ts | 6 - js/net/src/origin-request.test.ts | 214 ++++++++++++++++++++++++++++- js/net/src/origin.ts | 124 ++++++++++++++--- js/watch/src/broadcast.test.ts | 27 ++++ 9 files changed, 414 insertions(+), 232 deletions(-) diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index 3808ed63d56..1c64395aaf1 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -2665,7 +2665,8 @@ test.each([Lite.ALPN_06, Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18, Ietf.ALPN.DRAFT_2 }, ); -test.each([Lite.ALPN_06, Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18, Ietf.ALPN.DRAFT_22])( +// Without an instance on the wire nothing says the job's route continues the claim's stream. +test.each([Lite.ALPN_06, Ietf.ALPN.DRAFT_18, Ietf.ALPN.DRAFT_22])( "publishing a job still resets its first subscribe on %s", async (protocol) => { const pair = createMockTransportPair(protocol); @@ -2702,6 +2703,58 @@ test.each([Lite.ALPN_06, Lite.ALPN_07_WIP, Ietf.ALPN.DRAFT_18, Ietf.ALPN.DRAFT_2 }, ); +// On lite-07 a job's exact route continues its claim's minted instance, so a reader resolved +// through the claim keeps its broadcast and its subscription once that route lands. +test.each(["request", "follow"] as const)( + "a job's route continues a reader resolved through its claim (%s)", + async (method) => { + const pair = createMockTransportPair(Lite.ALPN_07_WIP); + const origin = new OriginProducer(); + const consumer = new OriginProducer(); + const path = Path.from("live/cam"); + const handle = origin.routePrefix(Path.empty()); + const serving = (async () => { + const requests = handle.requested(); + const request = (await requests.next()).value; + if (!request) throw new Error("missing job request"); + const job = request.route({ hidden: true }); + const chat = job.broadcast.createTrack("chat", { timescale: Timescale.MILLI }); + chat.writeFrame(textFrame("job")); + job.announce(); + return { job, chat }; + })(); + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client, consume: consumer }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + const listing = consumer.announced(Path.Pattern.literal(path)); + const requested = method === "request" ? consumer.request(path) : consumer.follow(path); + const remote = await waitFor(requested.active, (broadcast) => broadcast !== undefined); + if (!remote) throw new Error("request did not resolve"); + const track = remote.track("chat").subscribe().ordered(); + expect(await track.readString()).toBe("job"); + + // Written once the job's route is listed at the reader: the same subscription carries it. + const { job, chat } = await serving; + expect(job.instance).toBe(handle.instance); + expect(await listing.next()).toMatchObject({ kind: "present", prefix: path }); + chat.writeFrame(textFrame("later")); + expect(await track.readString()).toBe("later"); + expect(requested.active.peek()).toBe(remote); + expect(requested.closed.peek()).toBeUndefined(); + + track.close(); + listing.close(); + requested.close(); + handle.close(); + job.close(); + client.abort(); + server.abort(); + origin.close(); + consumer.close(); + }, +); + // The peer's close code a killed session carries. const DEATH = SessionCode(71); diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index 90256dbf0d8..82e45f79ab1 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -100,35 +100,6 @@ test("AnnounceBroadcast carries a restart only on draft-07", async () => { expect(await decodeAnnounceBroadcast(reader, Version.DRAFT_06)).toEqual({ status: "skipped" }); }); -test("AnnounceBroadcast carries a takeover only on draft-07", async () => { - const hops = [HopSchema.parse(7n)]; - const msg = { - status: "takeover" as const, - predecessor: 2n, - suffix: Path.from("p"), - pathBase: { distance: 1n, keep: 1 }, - instance: Instance.random(), - epoch: 3n, - hops, - hopBase: undefined, - cost: 12n, - }; - expect(await roundTrip(msg, Version.DRAFT_07)).toEqual(msg); - - // Older versions send a start instead. - await expect( - bytes( - (w) => - encodeAnnounceBroadcast( - w, - { status: "takeover", predecessor: 0n, suffix: Path.from("p"), hops: [] }, - Version.DRAFT_06, - ), - Version.DRAFT_06, - ), - ).rejects.toThrow(); -}); - test("AnnounceBroadcast skips an unknown type on draft-06", async () => { const encoded = await bytes(async (w) => { await w.u53(4); @@ -139,6 +110,18 @@ test("AnnounceBroadcast skips an unknown type on draft-06", async () => { expect(await decodeAnnounceBroadcast(reader, Version.DRAFT_06)).toEqual({ status: "skipped" }); }); +// Type 5 was a lite-07 message once; the version is unpublished, so it is now unknown there too. +test("AnnounceBroadcast skips type 5 as unknown on draft-07", async () => { + const encoded = await bytes(async (w) => { + await w.u53(5); + await w.u53(2); + await w.u8(0); + await w.u8(0); + }, Version.DRAFT_07); + const reader = new Reader(undefined, encoded, Version.DRAFT_07); + expect(await decodeAnnounceBroadcast(reader, Version.DRAFT_07)).toEqual({ status: "skipped" }); +}); + test("AnnounceBroadcast drops the route cost before draft-06", async () => { // Pre-lite-06 has no room for a cost on the wire, so one set locally is // simply not sent, keeping mixed-version meshes ranking on hop count. diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index c9fad00621c..516f63e2bb2 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -10,7 +10,6 @@ import { hasAnnounceId, hasAnnounceOk, hasAnnounceRestart, - hasAnnounceTakeover, hasExcludeHop, hasHidden, hasRouteCost, @@ -33,8 +32,6 @@ const ANNOUNCE_UPDATE = 2; // lite-07: another publisher instance replaces the advertisement. const ANNOUNCE_RESTART = 3; const SYNCED = 4; -// lite-07: a new advertisement continues what a live one served beneath its path. -const ANNOUNCE_TAKEOVER = 5; /** * Lite-07: copy `keep` path segments (from the head) or hops (from the tail) of the @@ -47,10 +44,9 @@ export type Base = { distance: bigint; keep: number }; * An announcement on the Announce Stream, advertising or retracting a broadcast. * * On lite-06+ these are independently-typed messages (`ANNOUNCE_START`, `ANNOUNCE_END`, - * `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART` and `ANNOUNCE_TAKEOVER`), each - * framed as `Type | Length | Body` like the subscribe stream's responses. Each `active` - * (ANNOUNCE_START) or `takeover` implicitly assigns the next announce id (a per-stream ordinal starting - * at 0); the others reference that id instead of repeating the path. Older versions send + * `ANNOUNCE_UPDATE`, and on lite-07 `ANNOUNCE_RESTART`), each framed as `Type | Length | Body` + * like the subscribe stream's responses. Each `active` (ANNOUNCE_START) implicitly assigns the + * next announce id (a per-stream ordinal starting at 0); the others reference that id instead of repeating the path. Older versions send * a single ANNOUNCE_BROADCAST message that retracts by path (`ended`). */ export type AnnounceBroadcast = @@ -89,20 +85,6 @@ export type AnnounceBroadcast = cost?: Cost; hopBase?: Base; } - /** Lite07+: an `active` that continues what the live announcement `predecessor` served - * beneath this one's path. It assigns the next id; one at the predecessor's own path - * retires the predecessor. */ - | { - status: "takeover"; - predecessor: bigint; - suffix: Path.Valid; - instance?: Instance.Valid; - epoch?: bigint; - hops: Hop[]; - cost?: Cost; - pathBase?: Base; - hopBase?: Base; - } /** An unknown lite-06+ announce type, skipped by length. Does not assign an id. */ | { status: "skipped" } /** Lite07+: the initial set has been delivered. */ @@ -264,13 +246,6 @@ async function encodeAnnounce06Body(w: Writer, msg: AnnounceBroadcast, version: await encodeHopsBlock(w, version, msg.hops, msg.hopBase); await encodeRouteCost(w, version, msg.cost); break; - case "takeover": - await w.u62(msg.predecessor); - await encodePath(w, version, msg.suffix, msg.pathBase); - await encodeRoute(w, version, msg.instance, msg.epoch); - await encodeHopsBlock(w, version, msg.hops, msg.hopBase); - await encodeRouteCost(w, version, msg.cost); - break; case "synced": break; case "ended": @@ -294,10 +269,6 @@ function announce06Type(msg: AnnounceBroadcast, version: Version): number { // Older versions send an end and a start instead. if (!hasAnnounceRestart(version)) throw new Error("announce restart not supported for this version"); return ANNOUNCE_RESTART; - case "takeover": - // Older versions send a start instead. - if (!hasAnnounceTakeover(version)) throw new Error("announce takeover not supported for this version"); - return ANNOUNCE_TAKEOVER; case "synced": if (!hasSynced(version)) throw new Error("synced not supported for this version"); return SYNCED; @@ -334,22 +305,6 @@ async function decodeAnnounce06Body(r: Reader, typ: number, version: Version): P const hops = await decodeHopsBlock(r, version); return { status: "restart", id, ...served, ...hops, cost: await decodeRouteCost(r, version) }; } - case ANNOUNCE_TAKEOVER: { - // Unknown before lite-07: skipped below like any other. - if (!hasAnnounceTakeover(version)) break; - const predecessor = await r.u62(); - const path = await decodePath(r, version); - const served = await decodeRoute(r, version); - const hops = await decodeHopsBlock(r, version); - return { - status: "takeover", - predecessor, - ...path, - ...served, - ...hops, - cost: await decodeRouteCost(r, version), - }; - } } // Skip the length-prefixed body so an earlier Lite06 build negotiating // the same ALPN does not kill the announce stream. @@ -375,7 +330,6 @@ async function encodeLegacyBody(w: Writer, msg: AnnounceBroadcast, version: Vers case "endedId": case "update": case "restart": - case "takeover": case "skipped": // The id-referencing forms only exist on lite-06+. throw new Error("announce ids not supported for this version"); @@ -476,23 +430,6 @@ export class AnnounceHistory { return { suffix, hops }; } - /** An ANNOUNCE_TAKEOVER: resolve it like a start against the live `predecessor`, - * returning the predecessor's suffix too. One at the predecessor's own suffix retires the - * predecessor, since a suffix holds one advertisement. */ - takeover(msg: { - predecessor: bigint; - suffix: Path.Valid; - hops: Hop[]; - pathBase?: Base; - hopBase?: Base; - }): Advertised & { previous: Path.Valid } { - const live = this.#live.get(msg.predecessor); - if (!live) throw new ProtocolViolation(`unknown announce id: ${msg.predecessor}`); - const resolved = this.start(msg); - if (resolved.suffix === live.suffix) this.#live.delete(msg.predecessor); - return { ...resolved, previous: live.suffix }; - } - /** An ANNOUNCE_UPDATE: resolve its chain before replacing the old one, which it may be based on. */ update(msg: { id: bigint; hops: Hop[]; hopBase?: Base }): Advertised { const hops = this.#hops(msg.hops, msg.hopBase); diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 5f99860d3a7..770bd46db7b 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -386,90 +386,6 @@ test("a lite-07 restart replaces the instance, and the next consume subscribes f subscriber.close(); }); -test("a lite-07 takeover surfaces as a start beneath its predecessor, or a restart of it", async () => { - const { subscriber, send, settle } = announceHarness(Version.DRAFT_07); - const announced = subscriber.routes(); - await settle(); - const pool = Path.from("pool"); - const job = Path.from("pool/job"); - - await send((w) => new AnnounceOk(PEER, 0).encode(w, Version.DRAFT_07)); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: pool, instance: Instance.random(), hops: [PUBLISHER_A] }, - Version.DRAFT_07, - ), - ); - expect(await announced.next()).toMatchObject({ prefix: pool, kind: "start" }); - - // Beneath the claim: a start with the next id (1), and the claim stays live. - const instance = Instance.random(); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "takeover", predecessor: 0n, suffix: job, instance, hops: [PUBLISHER_A] }, - Version.DRAFT_07, - ), - ); - expect(await announced.next()).toMatchObject({ prefix: job, kind: "start", route: { instance } }); - - // At the job's own path: a restart that retires id 1 and assigns id 2. - const next = Instance.random(); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "takeover", predecessor: 1n, suffix: job, instance: next, hops: [PUBLISHER_A] }, - Version.DRAFT_07, - ), - ); - expect(await announced.next()).toMatchObject({ prefix: job, kind: "restart", route: { instance: next } }); - - await send((w) => encodeAnnounceBroadcast(w, { status: "endedId", id: 2n }, Version.DRAFT_07)); - expect(await announced.next()).toMatchObject({ prefix: job, kind: "end" }); - await send((w) => encodeAnnounceBroadcast(w, { status: "endedId", id: 0n }, Version.DRAFT_07)); - expect(await announced.next()).toMatchObject({ prefix: pool, kind: "end" }); - - announced.close(); - subscriber.close(); -}); - -// A takeover must name a live predecessor whose path covers its own. -test.each([ - ["a retired predecessor", 1n, "pool/job"], - ["a predecessor not covering it", 0n, "other"], -] as const)("a lite-07 takeover of %s ends the stream", async (_, predecessor, suffix) => { - const { subscriber, send, settle } = announceHarness(Version.DRAFT_07); - const announced = subscriber.routes(); - await settle(); - - await send((w) => new AnnounceOk(PEER, 0).encode(w, Version.DRAFT_07)); - await send((w) => - encodeAnnounceBroadcast( - w, - { status: "active", suffix: Path.from("pool"), instance: Instance.random(), hops: [PUBLISHER_A] }, - Version.DRAFT_07, - ), - ); - expect(await announced.next()).toMatchObject({ kind: "start" }); - await send((w) => - encodeAnnounceBroadcast( - w, - { - status: "takeover", - predecessor, - suffix: Path.from(suffix), - instance: Instance.random(), - hops: [PUBLISHER_A], - }, - Version.DRAFT_07, - ), - ); - await expect(announced.next()).rejects.toThrow(); - - subscriber.close(); -}); - // A replaced prefix served everything consumed beneath it, so those paths subscribe fresh too, // whether the replacement arrives as a lite-07 restart or as an end then a start. test.each([ @@ -1890,7 +1806,7 @@ test.each([ ["answered on lite-06 stays", Version.DRAFT_06, "answered"], ["unanswered on lite-05 finishes", Version.DRAFT_05, "withdrawn"], ["unanswered on lite-06 finishes", Version.DRAFT_06, "withdrawn"], -] as const)("an unpinned SUBSCRIBE %s at a takeover or withdrawal", async (_, version, outcome) => { +] as const)("an unpinned SUBSCRIBE %s at a replacement or withdrawal", async (_, version, outcome) => { const warn = spyOn(console, "warn").mockImplementation(() => {}); const { quic, streams, opened } = fakeSession(); const subscriber = new Subscriber(quic, version, HopSchema.parse(1n)); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 5a2eb95690b..7f7b22882e7 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -531,8 +531,8 @@ export class Subscriber { stopDrain = this.#goaway?.changed(() => drainAdvertised()); // Lite06+: announce ids. Each received `active` implicitly assigns the next - // per-stream ordinal, as does each `takeover`; `endedId`/`update`/`restart` reference - // it, and lite-07 bases copy from it. Tracked even for announces we skip as reflected, since the sender + // per-stream ordinal; `endedId`/`update`/`restart` reference it, and lite-07 bases + // copy from it. Tracked even for announces we skip as reflected, since the sender // doesn't know we skipped. const history = new AnnounceHistory(); @@ -619,23 +619,6 @@ export class Subscriber { restart = true; break; } - case "takeover": { - // Continuity across the takeover is not kept here: a takeover at a new path - // surfaces as a start, and one at its predecessor's own path as a restart, - // as an older wire carries it. - const resolved = history.takeover(announce); - if (!Path.hasPrefix(resolved.previous, resolved.suffix)) { - throw new ProtocolViolation(`takeover of ${resolved.previous} at ${resolved.suffix}`); - } - path = Path.join(prefix, resolved.suffix); - active = true; - hops = resolved.hops; - cost = announce.cost; - instance = announce.instance; - epoch = announce.epoch; - restart = resolved.suffix === resolved.previous; - break; - } case "skipped": continue; } @@ -657,8 +640,7 @@ export class Subscriber { // a duplicate means the same thing on both sides of it. Mirrors the branch the // Rust announce loop takes before `start_announce`. const duplicateIsUpdate = updateSupported(this.version) && !hasAnnounceId(this.version); - const starts = announce.status === "active" || (announce.status === "takeover" && !restart); - if (starts && !duplicateIsUpdate && advertised.has(path)) { + if (announce.status === "active" && !duplicateIsUpdate && advertised.has(path)) { throw new ProtocolViolation(`duplicate announce for ${path}`); } diff --git a/js/net/src/lite/version.ts b/js/net/src/lite/version.ts index 228e7266344..2713dc3bab8 100644 --- a/js/net/src/lite/version.ts +++ b/js/net/src/lite/version.ts @@ -435,12 +435,6 @@ export function hasAnnounceRestart(version: Version): boolean { } } -/** Whether the announce stream has ANNOUNCE_TAKEOVER: a new advertisement continuing what a - * live one served beneath its path. Added in lite-07. Older versions send an ANNOUNCE_START. */ -export function hasAnnounceTakeover(version: Version): boolean { - return hasAnnounceRestart(version); -} - /** Whether announcements carry their instance and epoch, and TRACK, TRACKS, SUBSCRIBE, and FETCH * the instance they pin. Added in lite-07. Older versions carry nothing, so the receiving session * mints an instance for each announcement. */ diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index aabf3d94108..feeb76627c0 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -345,16 +345,19 @@ test("a visible job route answers its waiting requesters at once", async () => { const pending = (await requests.next()).value; if (!pending) throw new Error("expected a request"); const shared = origin.request(path); + const minted = claim.instance; const job = pending.route(); expect(job.announced).toBe(true); - expect(job.instance).not.toBe(claim.instance); + expect(job.instance).toBe(minted); await settle(); for (const request of [first, shared]) expect(request.active.peek()?.instance).toBe(job.instance); // The request is answered: routing it again is refused. expect(() => pending.route()).toThrow("request is closed"); - // Ending the job withdraws only its path: the claim keeps answering later requests. + // Ending the job withdraws only its path: the claim keeps answering later requests, under a + // fresh instance. job.close(); + expect(claim.instance).not.toBe(minted); await settle(); const later = origin.request(path); const next = (await requests.next()).value; @@ -898,3 +901,210 @@ test("a peer's pin is served by a local broadcast that loses the path", async () publication.broadcast.close(); origin.close(); }); + +// A job naming no instance continues its claim's minted one: a reader resolved through the claim +// is handed the job's broadcast under the instance it resolved, once, and keeps reading it. +test.each(["request", "follow"] as const)("a %s through a claim continues on its job's route", async (method) => { + const origin = new Producer(); + const path = Path.from("room/alice"); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const reader = method === "request" ? origin.request(path) : origin.follow(path); + const handed: unknown[] = []; + const stop = reader.active.subscribe((active) => { + if (active) handed.push(active); + }); + const pending = (await requests.next()).value; + if (!pending) throw new Error("expected a request"); + const job = pending.route({ hidden: true }); + const data = job.broadcast.createTrack("data"); + data.writeFrame({ payload: new TextEncoder().encode("first") }); + expect(job.instance).toBe(minted); + job.announce(); + await settle(); + expect(job.instance).toBe(minted); + expect(claim.instance).toBe(minted); + const active = reader.active.peek(); + expect(active?.instance).toBe(minted); + const track = active?.track("data").subscribe().ordered(); + expect(await track?.readString()).toBe("first"); + data.writeFrame({ payload: new TextEncoder().encode("second") }); + expect(await track?.readString()).toBe("second"); + expect(handed).toEqual([active]); + expect(reader.closed.peek()).toBeUndefined(); + stop(); + track?.close(); + reader.close(); + job.close(); + claim.close(); + origin.close(); +}); + +// Ending a job moves its claim to a fresh instance at once: nothing answers the ended stream's +// instance again, and the restarted job is a new stream. +test("a restarted job gets a fresh instance and its old one is refused", async () => { + const origin = new Producer(); + const wire = wireOf(origin.consume()); + const path = Path.from("room/alice"); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const first = origin.request(path); + const pending = (await requests.next()).value; + if (!pending) throw new Error("expected a request"); + const job = pending.route(); + await settle(); + const pinned = origin.request(path, { instance: minted }); + expect(pinned.active.peek()?.instance).toBe(minted); + + job.close(); + expect(claim.instance).not.toBe(minted); + for (const request of [first, pinned]) { + expect((request.closed.peek() as StreamError).code).toBe(StreamCode.Unroutable); + } + // A peer naming the ended instance is refused rather than handed the next job. + await expect(wire.demand(path, minted)).rejects.toMatchObject({ code: StreamCode.Unroutable }); + + const later = origin.request(path); + const next = (await requests.next()).value; + if (!next) throw new Error("expected another request"); + const restarted = next.route(); + await settle(); + expect(restarted.instance).toBe(claim.instance); + expect(restarted.instance).not.toBe(minted); + expect(later.active.peek()?.instance).toBe(restarted.instance); + later.close(); + restarted.close(); + claim.close(); + origin.close(); +}); + +// A job binds its claim's instance when it announces, so one still hidden when an unrelated job +// ends keeps its request and answers it under the claim's new instance. +test("a hidden job announces under the instance its claim moved to", async () => { + const origin = new Producer(); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const alice = origin.request(Path.from("room/alice")); + const first = (await requests.next()).value; + if (!first) throw new Error("expected a request"); + const ended = first.route(); + const bob = origin.request(Path.from("room/bob")); + const second = (await requests.next()).value; + if (!second) throw new Error("expected a request"); + const hidden = second.route({ hidden: true }); + expect(hidden.instance).toBe(minted); + + ended.close(); + await settle(); + const moved = claim.instance; + expect(moved).not.toBe(minted); + expect(hidden.instance).toBe(moved); + expect(bob.closed.peek()).toBeUndefined(); + hidden.announce(); + await settle(); + expect(hidden.instance).toBe(moved); + expect(bob.active.peek()?.instance).toBe(moved); + for (const request of [alice, bob]) request.close(); + hidden.close(); + claim.close(); + origin.close(); +}); + +// Each live job keeps the instance it bound; only one carrying the claim's current instance moves +// the claim when it ends. +test("one job ending leaves another job's instance alone", async () => { + const origin = new Producer(); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const jobs = []; + for (const name of ["alice", "bob"]) { + const request = origin.request(Path.from(`room/${name}`)); + const pending = (await requests.next()).value; + if (!pending) throw new Error("expected a request"); + jobs.push({ request, job: pending.route() }); + } + const [alice, bob] = jobs; + await settle(); + expect(bob.job.instance).toBe(minted); + + alice.job.close(); + const moved = claim.instance; + expect(moved).not.toBe(minted); + await settle(); + expect(bob.job.instance).toBe(minted); + expect(bob.request.closed.peek()).toBeUndefined(); + expect(bob.request.active.peek()?.instance).toBe(minted); + const again = origin.request(Path.from("room/bob")); + expect(again.active.peek()?.instance).toBe(minted); + + bob.job.close(); + expect(claim.instance).toBe(moved); + for (const { request } of jobs) request.close(); + again.close(); + claim.close(); + origin.close(); +}); + +// A claim naming a stable instance keeps it for answers it serves; its jobs mint their own. +test("a claim naming its instance gives each job a fresh one", async () => { + const origin = new Producer(); + const stable = Instance.parse("stable"); + const claim = origin.routePrefix(Path.from("room"), { instance: stable, epoch: 1n }); + const requests = claim.requested(); + const request = origin.request(Path.from("room/alice")); + const pending = (await requests.next()).value; + if (!pending) throw new Error("expected a request"); + const job = pending.route(); + await settle(); + expect(job.instance).not.toBe(stable); + expect(request.active.peek()?.instance).toBe(job.instance); + job.close(); + expect(claim.instance).toBe(stable); + request.close(); + claim.close(); + origin.close(); +}); + +// A claim moving on after a job ends keeps the answers it served from held broadcasts: a later +// request for a held path is answered again without asking the handler, under the new instance. +test("a claim moving on keeps its held answers", async () => { + const origin = new Producer(); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const held = new BroadcastProducer(); + held.createTrack("data").writeFrame({ payload: new TextEncoder().encode("held") }); + const reader = origin.request(Path.from("room/held")); + const asked = (await requests.next()).value; + if (!asked) throw new Error("expected a request"); + asked.serve(held.consume()); + const alice = origin.request(Path.from("room/alice")); + const pending = (await requests.next()).value; + if (!pending) throw new Error("expected a request"); + const job = pending.route(); + await settle(); + expect(reader.active.peek()?.instance).toBe(minted); + + job.close(); + expect(claim.instance).not.toBe(minted); + await settle(); + // The reader resolved under the old instance keeps its broadcast. + expect(reader.closed.peek()).toBeUndefined(); + const track = reader.active.peek()?.track("data").subscribe().ordered(); + expect(await track?.readString()).toBe("held"); + const later = origin.request(Path.from("room/held")); + expect(later.active.peek()?.instance).toBe(claim.instance); + const again = later.active.peek()?.track("data").subscribe().ordered(); + expect(await again?.readString()).toBe("held"); + + track?.close(); + again?.close(); + for (const request of [reader, alice, later]) request.close(); + claim.close(); + held.close(); + origin.close(); +}); diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 7fc4a48be2b..d7241320467 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -379,6 +379,8 @@ class ServeState { settled = new Signal(0); /** Counts {@link reset}s, so a waiter can tell its request was released rather than answered. */ resets = 0; + /** Set on a claim serving an instance it minted, which it hands to its jobs' exact routes. */ + handoff?: Handoff; onChange: (path: Path.Valid) => void = () => {}; onReject: (path: Path.Valid, err: Error) => void = () => {}; @@ -479,6 +481,19 @@ class ServeState { } } +/** + * What a claim that minted its instance shares with its jobs. A job naming no instance + * continues the claim's, so a reader resolved through the claim continues on the job's exact + * route; once that job stops being advertised the claim moves to a fresh instance, so the next + * job for the path is a new stream and nothing answers the ended one's instance again. + */ +interface Handoff { + /** The instance and epoch the claim serves now, which a job binds when it first announces. */ + current(): { instance: Instance.Valid; epoch: bigint }; + /** A job serving `instance` stopped being advertised; the claim moves on if it still serves it. */ + advance(instance: Instance.Valid): void; +} + interface Presented extends Advertised { readonly captures: Path.Pattern[] | undefined; } @@ -500,10 +515,12 @@ function publishPath(path: Path.Valid): void { } } -// A broadcast created through the origin: its producer, and the handle the origin routes. +// A broadcast created through the origin: its producer, the handle the origin routes, and +// how its route stops being advertised. interface Created { producer: broadcast.Producer; front: broadcast.Consumer; + retract: () => void; } // End a created broadcast the origin no longer routes. Consumer handles never end a produced @@ -1250,7 +1267,8 @@ export interface Table extends Reader { * credential, and, rarely, a stable instance. * * The default announces at once, at cost 0, under the session's scope, serving a fresh random - * instance minted when the route is created. + * instance minted when the route is created. A job's route ({@link Request.route}) instead + * continues its prefix route's instance when that one was minted; see {@link Route.instance}. */ export interface RouteOptions { /** @@ -1442,14 +1460,26 @@ export class Producer implements Table { auth: options.auth, }); + // A job naming no instance continues its claim's, bound when it first announces rather + // than now, so a job still being prepared is never invalidated by the claim moving on. + const claim = request && options.instance === undefined ? requestHandoff(request) : undefined; + let bound = false; + const producer = new broadcast.Producer(); hooks.stampPath(producer, path); const front = producer.consume(); - const entry = { producer, front }; + const announced = () => this.#state.local.peek()?.get(path) === front; + // The claim moves on before the retraction reroutes anything, so no request resolving the + // claim in this step is answered under the ended stream. + const retract = () => { + if (bound && announced()) claim?.advance(info.instance as Instance.Valid); + this.#retractExact(path, front); + }; + const entry = { producer, front, retract }; const previous = created.get(path); created.set(path, entry); if (previous) { - this.#retractExact(path, previous.front); + previous.retract(); retire(previous); } @@ -1460,7 +1490,7 @@ export class Producer implements Table { // a later announce. A job's unanswered request can never be answered now. const release = this.#hold(() => { stop(); - this.#retractExact(path, front); + retract(); if (created.get(path) === entry) created.delete(path); retire(entry); const job = pending; @@ -1470,7 +1500,6 @@ export class Producer implements Table { // Synchronous, so a request made right after the broadcast closes finds it gone. const stop = hooks.onClosed(producer, release); - const announced = () => this.#state.local.peek()?.get(path) === front; const announce = () => { if (producer.closed.peek() !== undefined) throw new Error("broadcast is closed"); const job = pending; @@ -1482,6 +1511,10 @@ export class Producer implements Table { try { // A request released meanwhile can never be answered, so this route never announces. if (!jobPending(job)) throw unroutable(); + if (claim) { + info = { ...info, ...claim.current() }; + bound = true; + } this.#advertiseExact(path, front, info); } catch (err) { release(); @@ -1499,9 +1532,9 @@ export class Producer implements Table { const route = makeRoute({ broadcast: producer, - instance: () => info.instance as Instance.Valid, + instance: () => (claim && !bound ? claim.current().instance : (info.instance as Instance.Valid)), announce, - withdraw: () => this.#retractExact(path, front), + withdraw: retract, announced, change: (update) => change(RouteInfo.normalize({ ...info, ...update })), requested: async function* () {}, @@ -1555,9 +1588,12 @@ export class Producer implements Table { * A consumer resolving a path under it that no announced local broadcast wins is handed to * {@link Route.requested} as a {@link Request}: answer it by serving a broadcast already held * ({@link Request.serve}), under this route's instance, or by creating the path's own exact - * route ({@link Request.route}), under a fresh instance, so restarting one job never splices - * into a copy of the last. A publisher that knows its broadcasts creates each one's exact - * {@link route} instead, so subscribers can enumerate them. + * route ({@link Request.route}), a job. Without {@link RouteOptions.instance} this route + * serves a minted instance that its jobs continue, so a reader resolved through it keeps + * reading when the job's route lands; once a job stops being advertised, this route moves to + * a fresh instance, so restarting the job never splices into a copy of the last. With a named + * instance, each job mints its own. A publisher that knows its broadcasts creates each one's + * exact {@link route} instead, so subscribers can enumerate them. */ routePrefix(prefix: Path.Valid, options: RouteOptions = {}): Route { const info = RouteInfo.normalize({ @@ -1576,8 +1612,9 @@ export class Producer implements Table { #serving(prefix: Path.Valid, route: RouteInfo, originated: boolean, publishes: boolean, visible: boolean): Route { prefix = this.#scope.prefix(prefix); - // A route that names no instance serves one of its own. - if (route.instance === undefined) route = { ...route, instance: Instance.random(), epoch: 0n }; + // A route that names no instance serves one of its own, and a claim hands that one to its jobs. + const minted = route.instance === undefined; + if (minted) route = { ...route, instance: Instance.random(), epoch: 0n }; publishPath(prefix); const server = new ServeState(this.#scope.root, prefix); server.onChange = (path) => this.#state.refresh(path); @@ -1645,19 +1682,39 @@ export class Producer implements Table { if (visible) insert(); } - const update = (next: RouteInfo) => { + // An `advance` to a fresh instance of the claim's own keeps the requests its handler holds, + // which a job binding at announce answers under the new instance, and the held answers it + // served; only readers restart. + const update = (next: RouteInfo, advance = false) => { open(); const previous = entry.route.peek().instance; const restart = inserted && previous !== next.instance; if (restart) this.#state.restart(entry); entry.route.set(next); - if (previous !== next.instance) server.reset(); + if (previous !== next.instance) { + // Wakes a peer's demand pinned to the old instance, which then refuses it. + if (advance) server.settled.update((n) => n + 1); + else server.reset(); + } if (!inserted) return; this.#state.rebuildOriginated(); this.#state.refreshPrefix(prefix); this.#state.routes.mutate(() => {}); }; + if (publishes && minted) { + server.handoff = { + current: () => { + const { instance, epoch } = entry.route.peek(); + return { instance: instance as Instance.Valid, epoch: epoch ?? 0n }; + }, + advance: (instance) => { + if (ended || !this.#state.routes.peek() || entry.route.peek().instance !== instance) return; + update({ ...entry.route.peek(), instance: Instance.random(), epoch: 0n }, true); + }, + }; + } + const publisher = publishes ? this : undefined; return makeRoute({ instance: () => entry.route.peek().instance as Instance.Valid, @@ -1890,6 +1947,7 @@ let setRequestPublisher: (request: Request, publisher: Producer | undefined) => let jobPending: (request: Request) => boolean; let answerJob: (request: Request) => void; let rejectJob: (request: Request, err: Error) => void; +let requestHandoff: (request: Request) => Handoff | undefined; /** * An owned request for one publisher instance at a broadcast path; see {@link Consumer.request}. @@ -2616,7 +2674,15 @@ export class Consumer implements Reader { } // Released by an instance change, which also dropped this pass's count: ask again. if (server.resets !== resets) break; - if (!server.pending.has(path)) return undefined; + if (!server.pending.has(path)) { + // Answered by a job's exact route. A peer naming no instance cannot tell + // that route's stream from the claim's, so it is refused and subscribes + // again once the route's announcement reaches it. + if (instance === undefined) return undefined; + const job = this.#local(path, instance); + if (job) return job; + throw unroutable(); + } await race([Signal.race(server.settled, server.closed), cancelled]); } } finally { @@ -2718,7 +2784,16 @@ export class Route { return producer; } - /** The stream this route serves, minted when it was created unless {@link RouteOptions.instance} named one. */ + /** + * The stream this route serves, minted when it was created unless {@link RouteOptions.instance} + * named one. + * + * A prefix route that minted its instance moves to a fresh one, as a restart, the moment a + * job's route carrying its instance stops being advertised. A job's route + * ({@link Request.route}) naming no instance takes its prefix route's current one when it + * first announces, and reports that until then; a prefix route naming its instance gives each + * job a fresh one instead. + */ get instance(): Instance.Valid { return this.#control.instance(); } @@ -2793,8 +2868,8 @@ export class Route { * * Yielded by {@link Route.requested}. Answer it one of three ways: {@link serve} a broadcast * already held, under the prefix route's instance; create the path's own exact route with - * {@link route}, under a fresh instance; or {@link reject} it with an error. Advancing the - * iterator or closing it without answering rejects the request. + * {@link route}, a job; or {@link reject} it with an error. Advancing the iterator or closing it + * without answering rejects the request. * * @public */ @@ -2825,6 +2900,7 @@ export class Request { request.#server.settled.update((n) => n + 1); }; rejectJob = (request, err) => request.#server.reject(request, err); + requestHandoff = (request) => request.#server.handoff; finishRequest = (request, err) => { request.#done = true; request.#withdrawn = true; @@ -2835,9 +2911,13 @@ export class Request { /** * Answer with the path's own exact route, owning a new broadcast. * - * The route serves a fresh instance unless `options` names one, so a job that ends and - * restarts beneath the same prefix is a new stream for its restarted sequences. Closing the - * route withdraws only this path: the prefix route keeps answering later requests. Created + * Unless `options` names an instance, the route continues the prefix route's minted + * instance, bound when it first announces, so readers that resolved the path through the + * prefix route keep reading the same stream. Once this route stops being advertised the + * prefix route moves to a fresh instance, so a job that restarts beneath it is a new stream + * for its restarted sequences, and a request naming the ended instance is refused. Under a + * prefix route naming its instance, the route mints a fresh one. Closing the route withdraws + * only this path: the prefix route keeps answering later requests. Created * visible, the route announces now and answers the requesters waiting on its path. Created * {@link RouteOptions.hidden | hidden}, they wait until {@link Route.announce}, so the job can * fill its catalog first; closing the route or its broadcast before then refuses them. diff --git a/js/watch/src/broadcast.test.ts b/js/watch/src/broadcast.test.ts index 32e8ea4676a..11d27d4d592 100644 --- a/js/watch/src/broadcast.test.ts +++ b/js/watch/src/broadcast.test.ts @@ -628,6 +628,33 @@ describe("restart", () => { await settle(); }); + it("plays a claim's job on the instance it resolved", async () => { + const owner = new Origin.Producer(); + const name = Path.from("pool/job.hang"); + const pool = owner.routePrefix(Path.from("pool")); + const answers = serve(pool); + const source = new Broadcast({ origin: owner, name, enabled: true, follow: true, catalogFormat: "manual" }); + const handed: unknown[] = []; + const stop = source.out.active.subscribe((active) => { + if (active) handed.push(active); + }); + await settle(); + const active = source.out.active.peek(); + expect(active?.instance).toBe(pool.instance); + + // The job's exact route carries the pool's instance, so the player never restarts. + for (let i = 0; i < 5; i++) await settle(); + expect(handed).toEqual([active]); + expect(source.out.error.peek()).toBeUndefined(); + + stop(); + for (const answer of answers) answer.close(); + pool.close(); + source.close(); + owner.close(); + await settle(); + }); + it("resumes on a covering prefix after a gap", async () => { const owner = new Origin.Producer(); const name = Path.from("pool/job.hang"); From 8cc983e94947f0cb0bb3bfa2bc026f2d241a912c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:07:44 -0800 Subject: [PATCH 34/61] fix(net): build the route node without the fuzz feature, retarget a drill mutation The route node minted lite-06 hops through Hop::new, which only test and fuzz builds compile, so the plain library build failed. The relay-withdraws-lost-publisher mutation patched the removed takeover branch and now also unwraps the route where the routed subscriber consumes it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/origin.rs | 4 +- rs/moq-net/src/route/node.rs | 2 +- .../relay-withdraws-lost-publisher.patch | 37 +++++++++++-------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index f0566b926e4..9fba0b4331d 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -11415,9 +11415,7 @@ mod tests { // The mark survives narrowing the handle. let scoped = peer.scope("room", &Patterns::from(Pattern::all())).unwrap(); - let _nested = scoped - .forward("x", RouteInfo::default().with_via(origin(8))) - .unwrap(); + let _nested = scoped.forward("x", RouteInfo::default().with_via(origin(8))).unwrap(); assert_eq!(announced.expect_next_active("room/x").await.source(), Source::Peer(8)); } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 472cb643476..9f770abc16b 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -629,7 +629,7 @@ pub(crate) fn hop(id: Id) -> crate::Hop { hash = hash.wrapping_mul(0x0100_0000_01b3); } let id = (hash & ((1 << 53) - 1)).max(1); - crate::Hop::new(id).expect("a non-zero hop id") + crate::Hop::from(std::num::NonZeroU64::new(id).expect("a non-zero hop id")) } #[cfg(test)] diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b325d14d5e7..66e1b0ba07d 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,7 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -8818,7 +8818,7 @@ +@@ -8817,7 +8817,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +18,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -8902,7 +8902,7 @@ +@@ -8901,7 +8901,7 @@ fn mint_sources(subscriber: &Subscriber, entry let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +27,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -8930,7 +8930,8 @@ +@@ -8929,7 +8929,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -8950,7 +8951,7 @@ +@@ -8949,7 +8950,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -49,14 +49,21 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -470,8 +470,8 @@ - .map(|(taken, entry)| match *taken == key.1 { - // In place, its sources serve what the new one continues: they live on with - // it rather than closing with the old route. -- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -- false => (&entry.route.dynamic, None), -+ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -+ false => (&*entry.route.dynamic, None), - }); - // A new instance or epoch replaces the old route, attached before it goes. - tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); +@@ -354,7 +354,7 @@ impl Routed { + self.previous = previous; + self.outputs(subscriber); + for (_, served) in std::mem::take(&mut self.previous) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + Ok(()) + } +@@ -398,7 +398,7 @@ impl Routed { + } => self.serve(subscriber, id, named, claim, served), + node::Output::Unserve { id, claim } => { + if let Some(served) = self.served.remove(&(id, claim)) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + } + // The ROUTE_END echo goes out as the peer's ROUTE_END is read, and a stream From eea486e0e7fd07ddfeb5632e5dfcc436e22b8d1a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:40:14 -0800 Subject: [PATCH 35/61] test(bindings): a job route continues its prefix route's instance Co-Authored-By: Claude Opus 5.5 --- dart/moq/test/moq_test.dart | 5 +++-- dart/moq_ffi/lib/src/moq.dart | 4 ++-- go/wrapper/moq_test.go | 4 ++-- go/wrapper/origin.go | 7 +++++-- kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt | 3 +-- swift/Sources/Moq/Origin.swift | 6 ++++-- swift/Tests/MoqTests/SmokeTests.swift | 2 +- 7 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dart/moq/test/moq_test.dart b/dart/moq/test/moq_test.dart index 0e6d525cf5b..5623422e6a5 100644 --- a/dart/moq/test/moq_test.dart +++ b/dart/moq/test/moq_test.dart @@ -691,9 +691,10 @@ void main() { final pending = origin.consume().request(path: 'live/cam'); final request = (await dynamic.requested().timeout(timeout))!; expect(request.path(), 'live/cam'); - // A hidden job route answers nobody until it announces. + // A hidden job route answers nobody until it announces, under the instance its + // prefix route lends. final served = request.route(options: RouteOptions(hidden: true)); - expect(served.instanceId(), isNot(dynamic.instanceId())); + expect(served.instanceId(), dynamic.instanceId()); served.announce(); await pending.timeout(timeout); // Closing the job's broadcast ends its route. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 42518706383..308143e9c9d 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -24602,10 +24602,10 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_reject() != 9727) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 16046) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 2402) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_serve() != 2580) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_serve() != 10832) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqfollowing_available() != 26337) { diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 268a5cf20e1..88eba6e456f 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -98,8 +98,8 @@ func TestDynamicBroadcastRequest(t *testing.T) { t.Fatal(err) } defer runtime.KeepAlive(served) - if served.Instance() == dynamic.Instance() { - t.Fatal("a job's route serves a fresh instance") + if served.Instance() != dynamic.Instance() { + t.Fatal("a job's route continues the instance its prefix route lends") } broadcast, err := served.Broadcast() if err != nil { diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index c3d65e0954b..63ecfacbbc0 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -162,8 +162,11 @@ func (r *BroadcastRequest) Path() (string, error) { // Route answers with the path's own exact route, owning a new broadcast. // -// It serves a fresh instance unless options names one. Created visible, it -// announces now and answers the waiting requesters; created hidden, they wait +// Unless options names an instance, it takes the one its prefix route lends +// when it first announces, so readers that resolved the path through the prefix +// route continue on it; its end moves the prefix route to a fresh instance, so a +// restarted job is a new stream. Created visible, it announces now and answers +// the waiting requesters; created hidden, they wait // until [Route.Announce], so the job can fill its catalog first. func (r *BroadcastRequest) Route(options RouteOptions) (*Route, error) { inner, err := r.inner.Route(options) diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index bdb48cef8c3..e4a7597e87e 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -30,7 +30,6 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertIs -import kotlin.test.assertNotEquals import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue @@ -921,7 +920,7 @@ class SmokeTest { val request = assertNotNull(dynamic.requested()) assertEquals("live/cam", request.path()) request.route(RouteOptions(hidden = true)).use { job -> - assertNotEquals(dynamic.instance(), job.instance()) + assertEquals(dynamic.instance(), job.instance()) job.broadcast().use { fill -> // Only announcing answers the requester. job.announce() diff --git a/swift/Sources/Moq/Origin.swift b/swift/Sources/Moq/Origin.swift index 86f5f77e46f..25cf15e48bb 100644 --- a/swift/Sources/Moq/Origin.swift +++ b/swift/Sources/Moq/Origin.swift @@ -68,8 +68,10 @@ public final class BroadcastRequest: Sendable { /// Answer with the path's own exact route, owning a new broadcast. /// - /// It serves a fresh instance unless `instance` names one. Created visible, it announces - /// now and answers the waiting requesters; created `hidden`, they wait until + /// Unless `instance` names one, it takes the instance its prefix route lends when it first + /// announces, so readers that resolved the path through the prefix route continue on it; + /// its end moves the prefix route to a fresh instance, so a restarted job is a new stream. + /// Created visible, it announces now and answers the waiting requesters; created `hidden`, they wait until /// `Route.announce()`, so the job can fill its catalog first. public func route( hidden: Bool = false, cost: UInt64 = 0, instance: String? = nil, epoch: UInt64 = 0 diff --git a/swift/Tests/MoqTests/SmokeTests.swift b/swift/Tests/MoqTests/SmokeTests.swift index 82a5b7f49e4..d079d95456b 100644 --- a/swift/Tests/MoqTests/SmokeTests.swift +++ b/swift/Tests/MoqTests/SmokeTests.swift @@ -407,7 +407,7 @@ final class SmokeTests: XCTestCase { let request = try XCTUnwrap(requested) XCTAssertEqual(try request.path, "live/cam") let served = try request.route(hidden: true) - XCTAssertNotEqual(served.instance, dynamic.instance) + XCTAssertEqual(served.instance, dynamic.instance) let broadcast = try served.broadcast XCTAssertTrue(broadcast === (try served.broadcast)) // Only announcing answers the requester. From 5bb0240b189a5c839582e4e78fd463a5d2e3f0bc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 17:11:19 -0800 Subject: [PATCH 36/61] fix(js): a peer pinned to a moved claim still reaches its job A peer demand pinned to the claim's instance refused as soon as a sibling job ending moved the claim on, even though its own job still announced that instance. It now resolves the job's route first. Co-Authored-By: Claude Opus 5.5 --- js/net/src/origin-request.test.ts | 31 +++++++++++++++++++++++++++++++ js/net/src/origin.ts | 10 ++++++++-- rs/moq-net/src/model/origin.rs | 24 ++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index ea635c8c030..39195a58d71 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -902,6 +902,37 @@ test("a peer's pin is served by a local broadcast that loses the path", async () origin.close(); }); +// A peer pinned to the claim's instance is served by its job's route, even when a sibling job +// ending moved the claim on before the peer's demand looked again. +test("a peer's pin reaches its job after a sibling moved the claim on", async () => { + const origin = new Producer(); + const wire = wireOf(origin.consume()); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const reader = origin.request(Path.from("room/alice")); + const asked = (await requests.next()).value; + if (!asked) throw new Error("expected a request"); + const alice = asked.route(); + await settle(); + + const bob = Path.from("room/bob"); + const demanded = wire.demand(bob, minted); + const next = (await requests.next()).value; + if (!next) throw new Error("expected a request"); + // Bob answers under the claim's instance, then Alice ends before the demand resumes. + const job = next.route(); + alice.close(); + expect(claim.instance).not.toBe(minted); + expect(job.instance).toBe(minted); + expect(await demanded).toBeDefined(); + + reader.close(); + job.close(); + claim.close(); + origin.close(); +}); + // A job naming no instance continues its claim's minted one: a reader resolved through the claim // is handed the job's broadcast under the instance it resolved, once, and keeps reading it. test.each(["request", "follow"] as const)("a %s through a claim continues on its job's route", async (method) => { diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index e5d8a1d872f..3d00fea0a2a 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -2658,8 +2658,14 @@ export class Consumer implements Reader { try { for (;;) { signal?.throwIfAborted(); - // The route may name another instance by the time its handler answers. - if (instance !== undefined && entry.route.peek().instance !== instance) throw unroutable(); + // The route may name another instance by the time its handler answers, as when a + // sibling job ending moves a claim on. A job's route still carrying the requested + // instance serves it. + if (instance !== undefined && entry.route.peek().instance !== instance) { + const job = this.#local(path, instance); + if (job) return job; + throw unroutable(); + } const served = server.served.get(path); if (served && served.closed.peek() === undefined) return served; const rejected = server.rejected.get(path); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 9fba0b4331d..7e5d9d0c591 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -12173,6 +12173,30 @@ mod tests { assert_eq!(alice.await.unwrap().info().instance, Some(second)); } + /// A request pinned to the claim's instance resolves through its job's route even when a + /// sibling job ending moved the claim on before the request looked again. + #[kio::test] + #[cfg(feature = "async")] + async fn a_pinned_request_reaches_its_job_after_a_sibling_moves_the_claim() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); + let lent = claim.instance(); + + let reader = consumer.request("room/alice", None); + let alice = queued(&claim).await.route(RouteOptions::default()).unwrap(); + let held = reader.await.unwrap(); + + let pinned = consumer.request("room/bob", lent.clone()); + let bob = queued(&claim).await.route(RouteOptions::default()).unwrap(); + drop(alice); + assert_ne!(claim.instance(), lent); + assert_eq!(bob.instance(), lent); + let resolved = pinned.await.expect("the job still serves the pinned instance"); + assert_eq!(resolved.info().instance, Some(lent)); + drop(held); + } + /// Once a job ends, a request pinned to its instance is refused without asking the /// claim, which serves another stream now, while an unpinned one starts the next job. #[kio::test] From e067fe5ccf3bf08338c5feafb17ffc0bdd2369fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 16:07:44 -0800 Subject: [PATCH 37/61] fix(net): build the route node without the fuzz feature, retarget a drill mutation The route node minted lite-06 hops through Hop::new, which only test and fuzz builds compile, so the plain library build failed. The relay-withdraws-lost-publisher mutation patched the removed takeover branch and now also unwraps the route where the routed subscriber consumes it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 2 +- .../relay-withdraws-lost-publisher.patch | 37 +++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index fd0e3e130e0..d7db156c0a8 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -643,7 +643,7 @@ pub(crate) fn hop(id: Id) -> crate::Hop { hash = hash.wrapping_mul(0x0100_0000_01b3); } let id = (hash & ((1 << 53) - 1)).max(1); - crate::Hop::new(id).expect("a non-zero hop id") + crate::Hop::from(std::num::NonZeroU64::new(id).expect("a non-zero hop id")) } #[cfg(test)] diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index b325d14d5e7..66e1b0ba07d 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -9,7 +9,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs -@@ -8818,7 +8818,7 @@ +@@ -8817,7 +8817,7 @@ impl Announced { }; self.held.pop(); if let Held::Attached(entry) = held { @@ -18,7 +18,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. } } -@@ -8902,7 +8902,7 @@ +@@ -8901,7 +8901,7 @@ fn mint_sources(subscriber: &Subscriber, entry let validity = if subscriber.version.has_instance() { Default::default() } else { @@ -27,7 +27,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. }; let source = subscriber.origin.create_source(path.clone(), validity); // The handler exists before the requester sees the source, so a track it -@@ -8930,7 +8930,8 @@ +@@ -8929,7 +8929,8 @@ struct AnnouncedRoute { /// without recomputing the chain. route: crate::origin::RouteInfo, /// Dropping it retracts the route and rejects its queued requests. @@ -37,7 +37,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. /// Closes as the route goes, which closes every source it minted. live: kio::Producer<()>, /// Whether the GOAWAY drain already re-priced this route. -@@ -8950,7 +8951,7 @@ +@@ -8949,7 +8950,7 @@ impl AnnouncedRoute { ) -> Self { Self { route, @@ -49,14 +49,21 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -470,8 +470,8 @@ - .map(|(taken, entry)| match *taken == key.1 { - // In place, its sources serve what the new one continues: they live on with - // it rather than closing with the old route. -- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -- false => (&entry.route.dynamic, None), -+ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), -+ false => (&*entry.route.dynamic, None), - }); - // A new instance or epoch replaces the old route, attached before it goes. - tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); +@@ -354,7 +354,7 @@ impl Routed { + self.previous = previous; + self.outputs(subscriber); + for (_, served) in std::mem::take(&mut self.previous) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + Ok(()) + } +@@ -398,7 +398,7 @@ impl Routed { + } => self.serve(subscriber, id, named, claim, served), + node::Output::Unserve { id, claim } => { + if let Some(served) = self.served.remove(&(id, claim)) { +- served.route.dynamic.withdrawn(); ++ std::mem::ManuallyDrop::into_inner(served.route.dynamic).withdrawn(); + } + } + // The ROUTE_END echo goes out as the peer's ROUTE_END is read, and a stream From 140af504807d36a630cb9a320e797b68582d9c39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:19:57 -0800 Subject: [PATCH 38/61] fix(net): lite-07 receive streams honor their limit, hold echo slots, coalesce serves A permit with a new limit files the stream again with the new regions before replaying, so its floors certify against the scope it now has. A claim is served only through a session whose limit admits it, even when its regions reach it through another session proving the same key. An ended route keeps its announce slot until its ROUTE_END echo is flushed. The node coalesces serving changes per claim, and a claim left with no usable route keeps the name its peer gave the identity. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/auth.rs | 6 + rs/moq-net/src/lite/subscriber/routed.rs | 135 ++++++++++++++++++--- rs/moq-net/src/route/node.rs | 147 +++++++++++++++++++++-- 3 files changed, 260 insertions(+), 28 deletions(-) diff --git a/rs/moq-net/src/auth.rs b/rs/moq-net/src/auth.rs index 1a412ad0dd8..8623a8acc55 100644 --- a/rs/moq-net/src/auth.rs +++ b/rs/moq-net/src/auth.rs @@ -588,6 +588,12 @@ impl Permit { Some(crate::model::interest_prefixes(limit)) } + /// Whether both set the same limit on the peer, whatever either was granted. + #[cfg(feature = "async")] + pub(crate) fn same_limit(&self, other: &Self) -> bool { + self.limit == other.limit + } + /// Whether this allows everything `other` does, judged part by part, so a `false` /// may be spurious but a `true` never is. pub(crate) fn covers(&self, other: &Self) -> bool { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index b804c10cd84..8c3740ca4db 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -82,6 +82,9 @@ pub(super) struct Routed { /// The session's announce cap: one slot per live route and claim. slots: crate::session::Slots, held: Vec, + /// The slots of routes the peer ended, kept until their ROUTE_END echo is flushed, + /// so a peer leaving its replies unread holds its routes' slots. + echoing: Vec, served: HashMap<(route::Id, Path), ServedClaim>, /// What the stream served before it filed again, while the new one takes it up. previous: HashMap<(route::Id, Path), ServedClaim>, @@ -138,6 +141,7 @@ impl Routed { claims: HashMap::new(), slots: subscriber.announces.clone(), held: Vec::new(), + echoing: Vec::new(), served: HashMap::new(), previous: HashMap::new(), ready: Default::default(), @@ -184,7 +188,7 @@ impl Routed { self.claims.remove(&claim); self.held.pop(); } - self.held.pop(); + self.echoing.extend(self.held.pop()); self.stream.route(route.node, None); self.replies.push(AnnounceReply::RouteEnd { id }); } @@ -332,24 +336,21 @@ impl Routed { self.outputs(subscriber); } - /// The session's permit changed: a scope the stream no longer matches, or a - /// limit admitting a different set of the peer's claims, files it again, the - /// peer's routes and the claims the limit admits replayed before the old stream - /// lets go. The peer still holds each claim either way. + /// The session's permit changed. A new limit files the stream again, the peer's + /// routes and the claims the limit admits replayed before the old stream lets go: + /// it may change the scope, the claims admitted, the claims the node may serve + /// here, and what the floors certify. The peer still holds each claim either way. pub(super) fn limit(&mut self, subscriber: &Subscriber, permit: &crate::auth::Permit) -> Result<(), Error> { - let regions = regions(&self.root, permit); - let admitted = (self.routes.values()) - .filter_map(|route| route.committed.as_ref()) - .flat_map(|(claims, _)| claims.keys()) - .any(|path| admits(&self.root, &self.permit, path) != admits(&self.root, permit, path)); + let same = permit.same_limit(&self.permit); self.permit = permit.clone(); - if regions == self.regions && !admitted { + if same { return Ok(()); } + self.regions = regions(&self.root, permit); let key = self.peer_setup.route_key().unwrap_or(self.session_key); let stream = self .node - .receive(scoped(&self.link, ®ions, key), self.prefix.clone()) + .receive(scoped(&self.link, &self.regions, key), self.prefix.clone()) .map_err(refused)?; // What the old stream served stays in the origin until the new one serves it // again, or lets go after. @@ -373,7 +374,6 @@ impl Routed { if self.drained { stream.goaway(); } - self.regions = regions; drop(std::mem::replace(&mut self.stream, stream)); self.previous = previous; self.outputs(subscriber); @@ -437,6 +437,13 @@ impl Routed { // Outside the session's root, so nothing a request could name here. return; }; + // A route reaches the peer's origin node, but the scope's regions can be wider + // than its limit: a claim another session of the same peer carries is served + // here only if this session may carry it too. Left unconfirmed, the node keeps + // it where the limit admits it. + if !self.permit.within_limit(path.as_str()) { + return; + } let cost = match self.drained || !served.available { true => crate::origin::Cost::DRAIN, false => crate::origin::Cost::new(served.claim.cost.saturating_add(served.metric)), @@ -507,8 +514,10 @@ impl Routed { for reply in self.replies.drain(..) { writer.buffer(&reply)?; } - if let Poll::Ready(Err(err)) = writer.poll_flush(waiter) { - return Err(err); + match writer.poll_flush(waiter) { + Poll::Ready(Ok(())) => self.echoing.clear(), + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => {} } while let Poll::Ready(Ok(key)) = self.ready.poll_pop(waiter) { let Some(entry) = self.served.get_mut(&key) else { @@ -631,6 +640,102 @@ mod tests { assert_eq!(served(&routed), ["live/a/public/x"]); } + /// An unscoped stream narrowed to a limit that leaves a claim out certifies nothing, + /// so widening it again brings that claim back at the same revision. + #[kio::test] + async fn a_new_scope_certifies_against_its_own_regions() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + routed.outputs(&subscriber); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + + routed.limit(&subscriber, &limited("live/*/public/**")).unwrap(); + assert_eq!(served(&routed), ["live/a/public/x"]); + + routed.limit(&subscriber, &limited("live/**")).unwrap(); + assert_eq!(served(&routed), ["live/a/private/y", "live/a/public/x"]); + } + + /// Two sessions proving one route key share the peer's routes, but a session serves + /// only the claims its own limit admits, even where its regions reach the rest. + #[kio::test] + async fn a_claim_is_served_only_where_the_limit_admits_it() { + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let everything = limited("live/**"); + let public = limited("live/*/public/**"); + let mut wide = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); + let mut narrow = super::Routed::new(&subscriber, &prefix, 1, &everything).unwrap(); + narrow.session_key = wide.session_key; + narrow.limit(&subscriber, &public).unwrap(); + + for routed in [&mut wide, &mut narrow] { + let mut decoder = AnnounceDecoder::default(); + let floor = Msg::Floor { + id: 0, + revision: Some(0), + }; + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + for msg in [route, start("live/a/public/x"), start("live/a/private/y"), floor] { + routed.receive(&prefix, msg, &mut decoder).unwrap(); + } + } + wide.outputs(&subscriber); + narrow.outputs(&subscriber); + assert_eq!(served(&wide), ["live/a/private/y", "live/a/public/x"]); + assert_eq!(served(&narrow), ["live/a/public/x"]); + } + + /// A route the peer ended keeps its announce slot until its ROUTE_END echo is + /// flushed, so a peer leaving its replies unread cannot churn routes past the cap. + #[kio::test] + async fn an_ended_route_holds_its_slot_until_its_echo_is_flushed() { + let (mut subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + subscriber.announces = crate::session::Slots::new(2); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let budget = kio::Producer::new(0); + let log = crate::lite::test_transport::Log::default(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::stepped(log, budget.consume()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + + for (id, node) in [(0, 7), (1, 8)] { + let route = Msg::RouteStart { node, advert: ADVERT }; + routed.receive(&prefix, route, &mut decoder).unwrap(); + routed.receive(&prefix, Msg::RouteEnd { id }, &mut decoder).unwrap(); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + } + let route = Msg::RouteStart { + node: 9, + advert: ADVERT, + }; + assert!(routed.receive(&prefix, route, &mut decoder).is_err(), "the unflushed echoes hold the cap"); + + *budget.write().ok().expect("the budget is open") = usize::MAX; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + assert_eq!(subscriber.announces.available(), 2); + } + /// What the stream keeps between floors is the net change to its committed claims, /// however many messages the peer sends without one. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index d7db156c0a8..46ef403cb6c 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -100,9 +100,10 @@ type Queues = HashMap>; /// What one stream has yet to read. Route changes and claim batches coalesce per /// identity, and what the reader was never sent leaves nothing when it ends: a route -/// started and ended unread is acked to the router at once, as the reader would. A -/// stream whose reader stalls holds one entry per route it holds or will start, -/// bounded by its claims, however much the node changes meanwhile. +/// started and ended unread is acked to the router at once, as the reader would. +/// Serving changes coalesce per claim. A stream whose reader stalls holds one entry +/// per route it holds or will start and per claim it serves, however much the node +/// changes meanwhile. #[derive(Default)] struct Outbox { /// Identities with route changes or claims pending, by when they first queued. @@ -111,6 +112,12 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, + /// Each claim's last serving change, read after the rest: its identity as the peer + /// names it and how it is served, or `None` to stop serving it. Keyed by when it + /// last changed, so a takeover's successor still reads before its predecessor goes. + serves: BTreeMap)>, + /// Where each claim's last change sits in `serves`. + serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, } @@ -133,7 +140,7 @@ type Changes = BTreeMap>; impl Outbox { fn is_empty(&self) -> bool { - self.ready.is_empty() && self.order.is_empty() + self.ready.is_empty() && self.order.is_empty() && self.serves.is_empty() } /// The pending entry for `id`, queued behind the others when new. @@ -200,6 +207,21 @@ impl Outbox { *last = floor; None } + // Each serving change replaces the last: the session acts on where the claim + // ends up, in place when it keeps its instance. + Output::Serve { + id, + named, + claim, + served, + } => { + self.serve(id, claim, Some((named, served))); + None + } + Output::Unserve { id, claim } => { + self.serve(id, claim, None); + None + } other => { self.ready.push_back(other); None @@ -209,13 +231,39 @@ impl Outbox { fn pop(&mut self) -> Option { if self.ready.is_empty() { - let (_, id) = self.order.pop_first()?; - let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); - self.take(id, pending); + match self.order.pop_first() { + Some((_, id)) => { + let (_, pending) = self.pending.remove(&id).expect("every queued identity is pending"); + self.take(id, pending); + } + None => { + let (_, (id, claim, serve)) = self.serves.pop_first()?; + self.serving.remove(&(id, claim.clone())); + return Some(match serve { + Some((named, served)) => Output::Serve { + id, + named, + claim, + served, + }, + None => Output::Unserve { id, claim }, + }); + } + } } self.ready.pop_front() } + /// Replace the claim's queued serving change with `serve`, queued last. + fn serve(&mut self, id: Id, claim: Path, serve: Option<(Id, Served)>) { + let seq = self.next; + self.next += 1; + if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + self.serves.remove(&old); + } + self.serves.insert(seq, (id, claim, serve)); + } + /// Queue one identity's pending entry to read, as the router would have sent it. fn take(&mut self, id: Id, pending: Pending) { if pending.ended { @@ -266,6 +314,8 @@ struct Home { struct Slot { served: Served, + /// The identity as the stream's peer names it, kept while no route to it is left. + named: Id, confirmed: bool, /// No longer a route to the claim's identity: let go once a wanted stream confirms. leaving: bool, @@ -565,40 +615,45 @@ impl State { gone.remove(&claim); let routes = self.router.routes(id, &claim); let home = self.homes.entry((id, claim.clone())).or_default(); - let wanted: Vec<(Key, Metric, bool)> = match routes.is_empty() { + // With no route left, the router no longer knows the name a stream's peer + // gave the identity, so it keeps the one it was served under. + let wanted: Vec<(Key, Metric, bool, Id)> = match routes.is_empty() { true => (home.keys.iter()) - .map(|(key, slot)| (*key, slot.served.metric, false)) + .map(|(key, slot)| (*key, slot.served.metric, false, slot.named)) + .collect(), + false => (routes.iter()) + .map(|&(key, metric)| (key, metric, available, self.router.name(key, id))) .collect(), - false => routes.iter().map(|&(key, metric)| (key, metric, available)).collect(), }; for slot in home.keys.values_mut() { slot.leaving = true; } - for (key, metric, available) in wanted { + for (key, metric, available, named) in wanted { let served = Served { claim: item.clone(), metric, available, }; match home.keys.get_mut(&key) { - Some(slot) if slot.served == served => { + Some(slot) if slot.served == served && slot.named == named => { slot.leaving = false; continue; } Some(slot) => { slot.served = served.clone(); + slot.named = named; slot.leaving = false; } None => { let slot = Slot { served: served.clone(), + named, confirmed: false, leaving: false, }; home.keys.insert(key, slot); } } - let named = self.router.name(key, id); let serve = Output::Serve { id, named, @@ -794,6 +849,72 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A claim of a scoped peer left with no usable route stays where it is served, + /// under the name that peer gave its identity, so the requests it serves meanwhile + /// still name it as the peer does. + #[test] + fn a_claim_without_a_route_keeps_the_peers_name() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let scoped = || Link { + scope: Some(super::super::Scope { + regions: vec![Path::empty()], + qualify: super::super::qualifier([7; 32]), + }), + ..Link::default() + }; + let first = relay.receive(scoped(), Path::empty()).unwrap(); + let second = relay.receive(scoped(), Path::empty()).unwrap(); + let first_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let second_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&first_answer, &first); + pump(&second_answer, &second); + second.route(1, Some(Reach::Held)); + let names = |outputs: Vec| -> Vec<(Id, bool)> { + (served(outputs).into_iter()) + .filter_map(|output| match output { + Output::Serve { named, served, .. } => Some((named, served.available)), + _ => None, + }) + .collect() + }; + assert_eq!(names(outputs(&first)), [(1, true)]); + + // The second still holds a route it may recover through. + first.route(1, None); + assert_eq!(names(outputs(&first)), [(1, false)]); + } + + /// A stream whose reader stalls holds one serving change per claim, however often + /// the claim changes through another stream meanwhile. + #[test] + fn a_stalled_stream_holds_one_serve_per_claim() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live/x"), Some(claim(&instance, 0)), None); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + for cost in 1..1000 { + origin.own(p("live/x"), Some(claim(&instance, cost)), None); + pump(&fast_answer, &fast); + outputs(&fast); + } + pump(&stalled_answer, &stalled); + let read = served(outputs(&stalled)); + assert_eq!(read.len(), 1, "{read:?}"); + assert!( + matches!(&read[0], Output::Serve { served, .. } if served.claim.cost == 999), + "{read:?}" + ); + } + /// A reader that stops reading holds one entry for the route, however often its /// claims change: the last change to each, and none for a claim it was never sent. #[test] From f2e03117c2adc659cf0313f22bc06f381d2ed7c9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:28:51 -0800 Subject: [PATCH 39/61] fix(net): keep upstream-learned claims from lite-07 upstreams, answer nothing past the root A relay's own claims include routes it learned from an upstream over an older version; an upstream answer now leaves those out and certifies no floor for them, as the lite-06 horizon does. A lite-07 prefix too long to lie beneath the handle's root answers nothing instead of the root. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/cluster.md | 6 + rs/moq-net-tests/tests/integration/restart.rs | 9 +- rs/moq-net/src/lite/publisher.rs | 31 +++++ rs/moq-net/src/lite/publisher/routed.rs | 52 ++++---- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/model/origin.rs | 117 ++++++++++++++---- rs/moq-net/src/route/node.rs | 56 +++------ rs/moq-net/tests/integration/mesh_withdraw.rs | 37 ++++++ 8 files changed, 223 insertions(+), 86 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 11beeb67d9f..a08f00c2984 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -179,6 +179,12 @@ the same one for a given path. A publisher connected straight to the relay is not hashed at its own path: its newest session wins, so a publisher that restarts takes over from its lingering old session at once. +moq-lite-07-wip carries no hop list, so both hop-list steps drop out: the first +relay ranks the origin nodes announcing a path the same way otherwise, and +remaining ties hash the path with each node's routing identity. Later hops +follow their own route to the node it picked. Routes learned over an older +version rank as above. + ```toml [cluster] connect = ["https://sibling.same-dc/?cost=1", "https://us-east.example.com/?cost=10"] diff --git a/rs/moq-net-tests/tests/integration/restart.rs b/rs/moq-net-tests/tests/integration/restart.rs index 3a5a97e1e5f..3dcd2cfffd3 100644 --- a/rs/moq-net-tests/tests/integration/restart.rs +++ b/rs/moq-net-tests/tests/integration/restart.rs @@ -679,11 +679,10 @@ impl Pool { } } -/// Wake the worker for `path` through the claim and subscribe. Every wire carries the -/// answer as an ordinary announcement, which resets the first subscription through the -/// claim, so wake the worker first and subscribe through the exact route. Lite-07's -/// takeover is gone until quest/m2/announce-takeover.md carries the claim's instance onto -/// the exact route. +/// Wake the worker for `path` through the claim and subscribe. Wake the worker first and +/// subscribe through the exact route, so the test reads the same on every wire, whether +/// it carries the answer as a takeover or as an ordinary announcement that resets the +/// first subscription through the claim. async fn claimed(origin: &origin::Producer, path: &str) -> (broadcast::Consumer, track::Subscriber) { let cold = origin.consume().request(path, None).await.unwrap(); drop(cold.track("video").unwrap().subscribe(None).await); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 91551958e83..c3399d98c2e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3636,6 +3636,37 @@ mod announce_test { ); } + /// A lite-07 prefix too long to lie beneath the handle's root names nothing there, + /// so the answer announces nothing rather than what the root holds. + #[test] + fn a_lite07_prefix_past_the_root_announces_nothing() { + let origin = Hop::new(1).unwrap().produce(); + let root = "r".repeat(100); + let _route = origin.announce(format!("{root}/x"), origin::RouteInfo::default()).unwrap(); + let consumer = origin + .consume() + .scope(root.as_str(), &crate::Patterns::from(crate::Pattern::all())) + .unwrap(); + let prefix = crate::Path::new(&"p".repeat(crate::Path::MAX_BYTES - 50)).unwrap(); + let mut answer = super::routed::Answer::new(consumer, prefix, true, crate::auth::Link::new(false)); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut stream = Stream:: { + writer: Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ), + reader: Reader::new(crate::transport::boxed::RecvStream::new(PendingRecv), Version::Lite07), + }; + assert!(answer.poll(&mut stream, false, &kio::Waiter::noop()).is_pending()); + wire.take_ok(); + assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced]); + } + #[test] fn synced_survives_a_removed_initial_route_and_live_churn() { let origin = Hop::new(1).unwrap().produce(); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index cde695a2ae3..58dbe852dbc 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -2,9 +2,9 @@ //! //! The node sends the peer its routes and, closed by an ANNOUNCE_FLOOR, each route's //! claims under the requested prefix. The stream filters what it puts on the wire: -//! hidden claims unless the peer asked for them, and anything this handle's scope or -//! the session's grant leaves out, in which case a floor certifies nothing, since the -//! peer cannot tell what was left out from what is gone. +//! hidden claims unless the peer asked for them, and anything this handle's scope, the +//! session's grant, or the upstream horizon leaves out, in which case a floor certifies +//! nothing, since the peer cannot tell what was left out from what is gone. //! //! A handle's mounts beneath the prefix are read through a node stream each, at the //! target, and what they present is named under the mount point on the same routes. @@ -66,8 +66,9 @@ enum Phase { /// Answers one lite-07 announce stream. pub(super) struct Answer { origin: origin::Consumer, - /// The requested prefix, absolute. - prefix: Path, + /// The requested prefix, absolute. `None` when it is too long to lie beneath the + /// handle's root: nothing is there, so the stream announces nothing. + prefix: Option, hidden: bool, link: route::Link, /// The prefix's own node stream, then one per mount beneath it. Empty until the @@ -100,7 +101,7 @@ impl Answer { proxy: origin.proxy(), scope: None, }; - let absolute = origin.root().join(&prefix).unwrap_or_else(|_| origin.root().clone()); + let absolute = origin.root().join(&prefix).ok(); Self { gate: crate::auth::Gate::new(auth, prefix, crate::auth::Direction::Publish), prefix: absolute, @@ -139,8 +140,9 @@ impl Answer { active: 0, })?; let node = self.origin.route_node(); - let mounts = self.origin.mounted(&self.prefix); - let interests = std::iter::once((self.prefix.clone(), None)) + let mounts = (self.prefix.as_ref()).map_or_else(Vec::new, |prefix| self.origin.mounted(prefix)); + let interests = (self.prefix.iter()) + .map(|prefix| (prefix.clone(), None)) .chain(mounts.into_iter().map(|mount| (mount.interest.clone(), Some(mount)))); for (interest, mount) in interests { let stream = node @@ -394,9 +396,13 @@ impl Answer { /// Send the ANNOUNCE_FLOOR closing a batch for the route to `id`. fn floor(&mut self, stream: &mut Stream, id: route::Id) -> Result<(), Error> { let certified = self.certified(); + let upstream = self.link.upstream; let sent = self.routes.get_mut(&id).expect("only a held route has a batch"); sent.floored = true; - let revision = sent.floor.filter(|_| certified); + // Like the grant, the upstream horizon leaves claims out that a floor would + // certify absent. + let hides = upstream && sent.claims.values().any(|claim| claim.upstream); + let revision = sent.floor.filter(|_| certified && !hides); stream.writer.buffer(&Msg::Floor { id: sent.wire, revision, @@ -407,13 +413,21 @@ impl Answer { /// scope or the session's grant leaves out lies beneath it. A hidden claim left out /// does not count, since the peer did not ask to see it. fn certified(&self) -> bool { - let relative = self.prefix.strip_prefix(self.origin.root()).unwrap_or_default(); - self.origin.covers(&self.prefix) && self.permit.covers_subtree(relative.as_str()) + let Some(prefix) = &self.prefix else { + return false; + }; + let relative = prefix.strip_prefix(self.origin.root()).unwrap_or_default(); + self.origin.covers(prefix) && self.permit.covers_subtree(relative.as_str()) + } + + /// The absolute `path` as the peer names it, under its requested prefix. + fn suffix(&self, path: &Path) -> Option { + path.strip_prefix(self.prefix.as_ref()?) } /// Whether the peer is shown the claim at the absolute `path`. fn shows(&self, path: &Path) -> bool { - let Some(suffix) = path.strip_prefix(&self.prefix) else { + let Some(suffix) = self.suffix(path) else { return false; }; let Some(relative) = path.strip_prefix(self.origin.root()) else { @@ -434,7 +448,9 @@ impl Answer { started: &mut BTreeSet, ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); - let wanted = sent.claims.get(&path).filter(|_| self.shows(&path)).cloned(); + let wanted = (sent.claims.get(&path)) + .filter(|claim| self.shows(&path) && !(self.link.upstream && claim.upstream)) + .cloned(); let wire = sent.wire; let taken = (wanted.as_ref()) .and_then(|claim| self.continued(id, &path, claim)) @@ -442,10 +458,7 @@ impl Answer { let key = (id, path); match (self.live.get_mut(&key), wanted) { (None, Some(claim)) => { - let suffix = key - .1 - .strip_prefix(&self.prefix) - .expect("shown claims are under the prefix"); + let suffix = self.suffix(&key.1).expect("shown claims are under the prefix"); let announce = match taken { Some((taken, predecessor)) => { tracing::debug!(route = %key.1, node = id, predecessor = %taken, "takeover"); @@ -476,10 +489,7 @@ impl Answer { // In place: a new Announce ID replaces the one it continues. Some((taken, predecessor)) if taken == key.1 => { tracing::debug!(route = %key.1, node = id, "takeover"); - let suffix = key - .1 - .strip_prefix(&self.prefix) - .expect("held claims are under the prefix"); + let suffix = self.suffix(&key.1).expect("held claims are under the prefix"); self.takeover(stream, wire, predecessor, suffix, &claim)? } _ if restart => { diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 8c3740ca4db..28db4a8d62f 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -541,6 +541,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos epoch, cost: cost.value(), continues: None, + upstream: false, }) } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 414d0c3f11b..fd702db6140 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3594,6 +3594,12 @@ impl Drop for AnnounceProducer { /// for as long as the origin lives. struct OwnClaims { routes: Routes, + /// The same routes, less those learned on an upstream link. + downstream: Routes, + /// The claim fed at each prefix. + best: HashMap, + /// The instance at each prefix that a route not learned upstream serves. + local: HashMap, node: crate::route::Node, } @@ -3601,40 +3607,99 @@ impl kio::Task for OwnClaims { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { - while let Some(event) = ready!(self.routes.poll_next(waiter)) { - self.apply(event); + loop { + if let Poll::Ready(event) = self.downstream.poll_next(waiter) { + match event { + Some(event) => self.local(event), + None => return Poll::Ready(()), + } + continue; + } + match ready!(self.routes.poll_next(waiter)) { + Some(event) => self.apply(event), + None => return Poll::Ready(()), + } } - Poll::Ready(()) } } impl OwnClaims { /// Feed the node the routes the table already holds, so a stream answered before /// the driver next runs sees them. - fn new(routes: Routes, node: crate::route::Node) -> Self { - let mut own = Self { routes, node }; + fn new(routes: Routes, downstream: Routes, node: crate::route::Node) -> Self { + let mut own = Self { + routes, + downstream, + best: HashMap::new(), + local: HashMap::new(), + node, + }; + while let Some(event) = own.downstream.try_next() { + own.local(event); + } while let Some(event) = own.routes.try_next() { own.apply(event); } own } - fn apply(&self, event: RouteEvent) { + fn apply(&mut self, event: RouteEvent) { let (announce, predecessor) = match event { RouteEvent::Start(announce) | RouteEvent::Update(announce) | RouteEvent::Restart(announce) => { (announce, None) } RouteEvent::Takeover { announce, predecessor } => (announce, Some(predecessor)), - RouteEvent::End(announce) => return self.node.own(announce.prefix, None, None), + RouteEvent::End(announce) => { + self.best.remove(&announce.prefix); + return self.node.own(announce.prefix, None, None); + } }; let route = announce.route; - let claim = route.instance.map(|instance| crate::route::node::Claim { + let Some(instance) = route.instance else { + return; + }; + let claim = crate::route::node::Claim { instance, epoch: route.epoch, cost: route.cost.value(), continues: None, - }); - self.node.own(announce.prefix, claim, predecessor.as_ref()); + upstream: false, + }; + self.best.insert(announce.prefix.clone(), claim); + self.feed(announce.prefix, predecessor.as_ref()); + } + + /// The route not learned upstream changed at a prefix: whether the claim there may + /// go to an upstream peer follows it. + fn local(&mut self, event: RouteEvent) { + match event { + RouteEvent::End(announce) => { + self.local.remove(&announce.prefix); + self.feed(announce.prefix, None); + } + RouteEvent::Start(announce) + | RouteEvent::Update(announce) + | RouteEvent::Restart(announce) + | RouteEvent::Takeover { announce, .. } => { + if let Some(instance) = announce.route.instance { + self.local.insert(announce.prefix.clone(), instance); + } + self.feed(announce.prefix, None); + } + } + } + + /// Feed the node the claim at `prefix`, marked as learned upstream unless a route + /// not learned upstream serves the same instance there. + fn feed(&self, prefix: Path, predecessor: Option<&Path>) { + let Some(claim) = self.best.get(&prefix) else { + return; + }; + let claim = crate::route::node::Claim { + upstream: self.local.get(&prefix) != Some(&claim.instance), + ..claim.clone() + }; + self.node.own(prefix, Some(claim), predecessor); } } @@ -7147,20 +7212,24 @@ impl Consumer { shared.route_node = Some(node.clone()); drop(shared); // The whole table, whatever this handle's scope. - let own = Self { - scope: OriginScope::default(), - root: Path::default(), - horizon: Horizon { - node: NodeFilter::Own, - ..Horizon::default() - }, - covering: false, - ..self.clone() - } - .with_hidden(true) - // Feeding the node is no reason to ask any peer about anything. - .serving(); - self.tasks.push(OwnClaims::new(own.routes(), node.clone())); + let own = |upstream| { + Self { + scope: OriginScope::default(), + root: Path::default(), + horizon: Horizon { + node: NodeFilter::Own, + upstream, + ..Horizon::default() + }, + covering: false, + ..self.clone() + } + .with_hidden(true) + // Feeding the node is no reason to ask any peer about anything. + .serving() + }; + let claims = OwnClaims::new(own(false).routes(), own(true).routes(), node.clone()); + self.tasks.push(claims); node } diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 46ef403cb6c..4a4a288d0f3 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -41,6 +41,10 @@ pub(crate) struct Claim { /// this claim's broadcast. Kept while the instance stays, and sent as /// ANNOUNCE_TAKEOVER to a peer still holding that claim. pub continues: Option<(Path, Instance)>, + /// This node's own claim, served through a route learned from an upstream peer: it + /// is never sent to another upstream, so this node never carries traffic between + /// two. Always unset on a claim off the wire. + pub upstream: bool, } /// A shown claim as the origin serves it through the selected route. @@ -112,10 +116,9 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// Each claim's last serving change, read after the rest: its identity as the peer - /// names it and how it is served, or `None` to stop serving it. Keyed by when it - /// last changed, so a takeover's successor still reads before its predecessor goes. - serves: BTreeMap)>, + /// Each claim's last Serve or Unserve, read after the rest. Keyed by when it last + /// changed, so a takeover's successor still reads before its predecessor goes. + serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. @@ -209,17 +212,13 @@ impl Outbox { } // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. - Output::Serve { - id, - named, - claim, - served, - } => { - self.serve(id, claim, Some((named, served))); - None - } - Output::Unserve { id, claim } => { - self.serve(id, claim, None); + Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + let seq = self.next; + self.next += 1; + if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + self.serves.remove(&old); + } + self.serves.insert(seq, output); None } other => { @@ -237,33 +236,17 @@ impl Outbox { self.take(id, pending); } None => { - let (_, (id, claim, serve)) = self.serves.pop_first()?; - self.serving.remove(&(id, claim.clone())); - return Some(match serve { - Some((named, served)) => Output::Serve { - id, - named, - claim, - served, - }, - None => Output::Unserve { id, claim }, - }); + let (_, output) = self.serves.pop_first()?; + if let Output::Serve { id, claim, .. } | Output::Unserve { id, claim } = &output { + self.serving.remove(&(*id, claim.clone())); + } + return Some(output); } } } self.ready.pop_front() } - /// Replace the claim's queued serving change with `serve`, queued last. - fn serve(&mut self, id: Id, claim: Path, serve: Option<(Id, Served)>) { - let seq = self.next; - self.next += 1; - if let Some(old) = self.serving.insert((id, claim.clone()), seq) { - self.serves.remove(&old); - } - self.serves.insert(seq, (id, claim, serve)); - } - /// Queue one identity's pending entry to read, as the router would have sent it. fn take(&mut self, id: Id, pending: Pending) { if pending.ended { @@ -716,6 +699,7 @@ mod tests { epoch: 0, cost, continues: None, + upstream: false, } } diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index 035d0e4959f..f02e8d392a3 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -679,3 +679,40 @@ async fn mixed_version_ring_withdraw_retracts() { ); } } + +/// A relay never carries traffic between two upstreams, across versions too: a +/// broadcast it learned from a moq-lite-06 upstream enters the route layer as one of +/// its own claims, yet never reaches a moq-lite-07 upstream, while one published at +/// the relay does. +#[kio::test] +async fn an_upstream_learned_claim_never_reaches_another_upstream() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let first = produce_origin(1); + let relay = produce_origin(2); + let second = produce_origin(3); + let upstream = relay.clone().with_link(origin::Link::Upstream); + let mut pairs = Vec::new(); + for (version, core) in [(lite06, &first), (lite07, &second)] { + let core = core.clone().with_link(origin::Link::Peer); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(upstream.consume().with_hidden(true)); + options.client_subscribe = Some(upstream.clone()); + options.server_publish = Some(core.consume().with_hidden(true)); + options.server_subscribe = Some(core); + pairs.push(connect_mock(options).await); + } + + let mut at_relay = watch(relay.consume().routes()); + let mut at_second = watch(second.consume().routes()); + let learned = first.route("room/learned", origin::RouteOptions::hidden()).unwrap(); + learned.announce().unwrap(); + let local = relay.route("room/local", origin::RouteOptions::hidden()).unwrap(); + local.announce().unwrap(); + kio::clock::sim::idle().await; + let relayed = drain(&mut at_relay); + assert!(relayed.contains_key("room/learned"), "{relayed:?}"); + let seen = drain(&mut at_second); + assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); + assert!(!seen.contains_key("room/learned"), "{seen:?}"); +} From 7e0897d2495c9ad46acea5bc2dfa8dbe6b9532a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:33:52 -0800 Subject: [PATCH 40/61] style(net): format the new lite-07 tests Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 4 +++- rs/moq-net/src/lite/subscriber/routed.rs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index c3399d98c2e..78892904425 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3642,7 +3642,9 @@ mod announce_test { fn a_lite07_prefix_past_the_root_announces_nothing() { let origin = Hop::new(1).unwrap().produce(); let root = "r".repeat(100); - let _route = origin.announce(format!("{root}/x"), origin::RouteInfo::default()).unwrap(); + let _route = origin + .announce(format!("{root}/x"), origin::RouteInfo::default()) + .unwrap(); let consumer = origin .consume() .scope(root.as_str(), &crate::Patterns::from(crate::Pattern::all())) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 28db4a8d62f..e31ac3fd245 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -730,7 +730,10 @@ mod tests { node: 9, advert: ADVERT, }; - assert!(routed.receive(&prefix, route, &mut decoder).is_err(), "the unflushed echoes hold the cap"); + assert!( + routed.receive(&prefix, route, &mut decoder).is_err(), + "the unflushed echoes hold the cap" + ); *budget.write().ok().expect("the budget is open") = usize::MAX; routed.poll(&subscriber, &mut writer, &waiter).unwrap(); From d5ed89fd6045c6e0bcf726519083ccb831a10d2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:41:13 -0800 Subject: [PATCH 41/61] test(drill): the lost-publisher mutation compiles against the routed subscriber The ported patch dropped the deref where the routed subscriber passes the predecessor's route to the origin, so the mutated tree failed to build. Co-Authored-By: Claude Opus 5.5 --- .../relay-withdraws-lost-publisher.patch | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/drill/mutations/relay-withdraws-lost-publisher.patch b/test/drill/mutations/relay-withdraws-lost-publisher.patch index 66e1b0ba07d..109e8096040 100644 --- a/test/drill/mutations/relay-withdraws-lost-publisher.patch +++ b/test/drill/mutations/relay-withdraws-lost-publisher.patch @@ -49,7 +49,7 @@ diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber. diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs -@@ -354,7 +354,7 @@ impl Routed { +@@ -378,7 +378,7 @@ impl Routed { self.previous = previous; self.outputs(subscriber); for (_, served) in std::mem::take(&mut self.previous) { @@ -58,7 +58,7 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs } Ok(()) } -@@ -398,7 +398,7 @@ impl Routed { +@@ -422,7 +422,7 @@ impl Routed { } => self.serve(subscriber, id, named, claim, served), node::Output::Unserve { id, claim } => { if let Some(served) = self.served.remove(&(id, claim)) { @@ -67,3 +67,14 @@ diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subs } } // The ROUTE_END echo goes out as the peer's ROUTE_END is read, and a stream +@@ -477,8 +477,8 @@ impl Routed { + .map(|(taken, entry)| match *taken == key.1 { + // In place, its sources serve what the new one continues: they live on with + // it rather than closing with the old route. +- true => (&entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), +- false => (&entry.route.dynamic, None), ++ true => (&*entry.route.dynamic, Some(std::mem::take(&mut entry.route.live))), ++ false => (&*entry.route.dynamic, None), + }); + // A new instance or epoch replaces the old route, attached before it goes. + tracing::debug!(route = %subscriber.log_path(&path), node = id, "announce"); From f51f5292bec7f044cfb7d437acfb79b512080f20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 18:57:03 -0800 Subject: [PATCH 42/61] fix(net): a coalesced Serve keeps its place behind a takeover Replacing a queued Serve moved it to the back, behind its predecessor's Unserve, so a takeover repriced before a stalled reader drained lost the predecessor it continues. A Serve replacing a Serve now keeps its place; an Unserve, or a Serve after one, still goes last. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 4a4a288d0f3..c9e1756fe59 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -116,8 +116,10 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// Each claim's last Serve or Unserve, read after the rest. Keyed by when it last - /// changed, so a takeover's successor still reads before its predecessor goes. + /// Each claim's last Serve or Unserve, read after the rest, in queue order. A + /// takeover queues its successor's Serve before its predecessor's Unserve, and the + /// session needs the predecessor still served to carry it on: so a Serve replacing + /// a queued Serve keeps its place, while an Unserve, or a Serve after one, goes last. serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, @@ -213,9 +215,17 @@ impl Outbox { // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + let key = (id, claim.clone()); + if let Some(&seq) = self.serving.get(&key) + && matches!(output, Output::Serve { .. }) + && matches!(self.serves.get(&seq), Some(Output::Serve { .. })) + { + self.serves.insert(seq, output); + return None; + } let seq = self.next; self.next += 1; - if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + if let Some(old) = self.serving.insert(key, seq) { self.serves.remove(&old); } self.serves.insert(seq, output); @@ -833,6 +843,39 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A stalled stream reads a takeover's successor before its predecessor goes, even + /// once the successor changed again meanwhile, so the session still finds the + /// predecessor to carry on. + #[test] + fn a_stalled_takeover_reads_its_successor_first() { + let (old, new) = (Instance::random(), Instance::random()); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&old, 0)), None); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + outputs(&fast); + outputs(&stalled); + + origin.own(p("live/x"), Some(claim(&new, 0)), Some(&p("live"))); + origin.own(p("live"), None, None); + pump(&fast_answer, &fast); + origin.own(p("live/x"), Some(claim(&new, 5)), None); + pump(&fast_answer, &fast); + let read: Vec<(Path, Option)> = (served(outputs(&stalled)).into_iter()) + .map(|output| match output { + Output::Serve { claim, served, .. } => (claim, Some(served.claim.cost)), + Output::Unserve { claim, .. } => (claim, None), + other => panic!("not a serving change: {other:?}"), + }) + .collect(); + assert_eq!(read, [(p("live/x"), Some(5)), (p("live"), None)]); + } + /// A claim of a scoped peer left with no usable route stays where it is served, /// under the name that peer gave its identity, so the requests it serves meanwhile /// still name it as the peer does. From 1f5d0533683b6eed9212299e3d6cc86bf4998d12 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:09:03 -0800 Subject: [PATCH 43/61] fix(relay): a removed cluster ID names its removal at startup --cluster-id, MOQ_CLUSTER_ID, and [cluster] id stay as hidden settings reported through deprecated(), so a config that still sets one stops saying the Hop ID is random rather than refusing an unknown setting. The moq-net guide marks the proxying claim as lite-07-wip and opt-in. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-net.md | 2 +- rs/moq-relay/src/cluster.rs | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index e655a8d6508..1449c8678ca 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -21,7 +21,7 @@ What you use it for, beyond what the concept page already describes: - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. -- **Relay by opting into proxying.** On moq-lite-07 a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. +- **Relay by opting into proxying.** On moq-lite-07 (`moq-lite-07-wip`, opt-in) a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)`, or one consumer with `consumer.with_proxy(true)`, to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` announces a hidden endpoint at `/.socket/`, used only to route dials. A session speaks one of two protocols, each with its own dial and accept. `accept_tcp()` and `accept_track()` yield an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_tcp(peer)` and an accepted `tcp` dial yield a `socket::TcpStream`, a `group::Producer` and `group::Consumer` pair carrying one reliable, ordered byte pipe each way (finish is FIN, abort is RST, and `ready` paces the writer to the reader); `dial_track(peer, info)` and an accepted `track` dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). - **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 12594c1198f..d47179d427c 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -794,6 +794,12 @@ pub struct Config { #[usage(skip)] #[serde(rename = "connect_api_tls_root", skip_serializing)] pub(crate) removed_api_root: Option, + #[usage(long = "cluster-id", env = "MOQ_CLUSTER_ID", hide = true)] + #[serde(skip)] + pub(crate) removed_id: Option, + #[usage(skip)] + #[serde(rename = "id", skip_serializing)] + pub(crate) removed_id_toml: Option, /// This relay's own externally-reachable URL (identity). Sent to /// `--cluster-api-url` as a `?node=` query param so the endpoint can return @@ -862,6 +868,13 @@ impl Config { if self.removed_api_root.is_some() { found.toml("[cluster] connect_api_tls_root", "[cluster.api.tls] root", None); } + const ID: &str = "the Hop ID is random on every start and nothing pins it"; + if self.removed_id.is_some() { + found.removed("--cluster-id", Some("MOQ_CLUSTER_ID"), ID); + } + if self.removed_id_toml.is_some() { + found.toml("[cluster] id", "removed", Some(ID)); + } if self.connect.iter().any(|peer| is_legacy_peer(peer.url())) { found.changed( @@ -3765,6 +3778,31 @@ mod tests { dialed.release("never-dialed", DialSource::Mdns, &mut no_dial); } + /// A removed cluster ID still parses, from the flag, the variable, or the file, so + /// the process can say why it stops instead of refusing an unknown setting. + #[test] + fn a_removed_cluster_id_is_reported() { + let flagged = Config { + removed_id: Some("7".into()), + ..Default::default() + }; + let reported = flagged.deprecated().to_string(); + assert!( + reported.contains("--cluster-id / MOQ_CLUSTER_ID -> removed"), + "{reported}" + ); + assert!(new_cluster(flagged).is_err()); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cluster-id.toml"); + std::fs::write(&path, "[cluster]\nid = 7\n").unwrap(); + let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; + let Err(err) = RelayConfig::parse_and_merge(args) else { + panic!("a removed cluster ID loaded"); + }; + assert!(format!("{err:#}").contains("[cluster] id -> removed"), "{err:#}"); + } + #[test] fn cluster_connect_api_survives_toml_merge() { let toml = "[cluster.api]\nurl = \"https://api.example.com/cluster/connect\"\ntls.root = [\"api.pem\"]\n"; From 70771b3cd64cbef909cd4f150b3a353bfaf117b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 20:43:35 -0800 Subject: [PATCH 44/61] fix(net): a node that does not proxy keeps lite-06-learned claims to itself Routes learned over an older version enter the route layer as this node's own claims. An own claim whose route crossed another node is now marked learned, and an answer to a peer this node does not proxy for leaves it out and certifies no floor for it, so an application is never transit between a lite-06 and a lite-07 peer. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 23 ++++++++---- rs/moq-net/src/lite/subscriber/routed.rs | 1 + rs/moq-net/src/model/origin.rs | 2 ++ rs/moq-net/src/route/node.rs | 5 +++ rs/moq-net/tests/integration/mesh_withdraw.rs | 35 +++++++++++++++++++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 58dbe852dbc..fe02bec9cb9 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -3,8 +3,9 @@ //! The node sends the peer its routes and, closed by an ANNOUNCE_FLOOR, each route's //! claims under the requested prefix. The stream filters what it puts on the wire: //! hidden claims unless the peer asked for them, and anything this handle's scope, the -//! session's grant, or the upstream horizon leaves out, in which case a floor certifies -//! nothing, since the peer cannot tell what was left out from what is gone. +//! session's grant, the upstream horizon, or the proxy preference leaves out, in which +//! case a floor certifies nothing, since the peer cannot tell what was left out from +//! what is gone. //! //! A handle's mounts beneath the prefix are read through a node stream each, at the //! target, and what they present is named under the mount point on the same routes. @@ -63,6 +64,13 @@ enum Phase { Closing, } +/// Whether a stream on `link` carries `claim` at all: never an upstream-learned claim +/// to an upstream, nor one learned from another node to a peer this node does not +/// proxy for. +fn sends(link: &route::Link, claim: &node::Claim) -> bool { + !(link.upstream && claim.upstream) && (link.proxy || !claim.learned) +} + /// Answers one lite-07 announce stream. pub(super) struct Answer { origin: origin::Consumer, @@ -396,12 +404,13 @@ impl Answer { /// Send the ANNOUNCE_FLOOR closing a batch for the route to `id`. fn floor(&mut self, stream: &mut Stream, id: route::Id) -> Result<(), Error> { let certified = self.certified(); - let upstream = self.link.upstream; + // Like the grant, the upstream horizon and the proxy preference leave claims out + // that a floor would certify absent. + let hides = (self.routes.get(&id).into_iter()) + .flat_map(|sent| sent.claims.values()) + .any(|claim| !sends(&self.link, claim)); let sent = self.routes.get_mut(&id).expect("only a held route has a batch"); sent.floored = true; - // Like the grant, the upstream horizon leaves claims out that a floor would - // certify absent. - let hides = upstream && sent.claims.values().any(|claim| claim.upstream); let revision = sent.floor.filter(|_| certified && !hides); stream.writer.buffer(&Msg::Floor { id: sent.wire, @@ -449,7 +458,7 @@ impl Answer { ) -> Result<(), Error> { let sent = self.routes.get(&id).expect("only a held route syncs"); let wanted = (sent.claims.get(&path)) - .filter(|claim| self.shows(&path) && !(self.link.upstream && claim.upstream)) + .filter(|claim| self.shows(&path) && sends(&self.link, claim)) .cloned(); let wire = sent.wire; let taken = (wanted.as_ref()) diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index e31ac3fd245..31732dc7ddd 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -542,6 +542,7 @@ fn claim(instance: Option, epoch: u64, cost: crate::origin::Cos cost: cost.value(), continues: None, upstream: false, + learned: false, }) } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index fd702db6140..7c9749d5f10 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3664,6 +3664,8 @@ impl OwnClaims { cost: route.cost.value(), continues: None, upstream: false, + // Only a route published here has crossed no node. + learned: !route.hops.is_empty(), }; self.best.insert(announce.prefix.clone(), claim); self.feed(announce.prefix, predecessor.as_ref()); diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index c9e1756fe59..66f62a45940 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -45,6 +45,10 @@ pub(crate) struct Claim { /// is never sent to another upstream, so this node never carries traffic between /// two. Always unset on a claim off the wire. pub upstream: bool, + /// This node's own claim, served through a route another node announced over an + /// older version: sent only to a peer this node proxies for, so a node that does + /// not proxy is never transit. Always unset on a claim off the wire. + pub learned: bool, } /// A shown claim as the origin serves it through the selected route. @@ -710,6 +714,7 @@ mod tests { cost, continues: None, upstream: false, + learned: false, } } diff --git a/rs/moq-net/tests/integration/mesh_withdraw.rs b/rs/moq-net/tests/integration/mesh_withdraw.rs index f02e8d392a3..ffdbb0cb14b 100644 --- a/rs/moq-net/tests/integration/mesh_withdraw.rs +++ b/rs/moq-net/tests/integration/mesh_withdraw.rs @@ -716,3 +716,38 @@ async fn an_upstream_learned_claim_never_reaches_another_upstream() { assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); assert!(!seen.contains_key("room/learned"), "{seen:?}"); } + +/// A node that does not proxy is never transit, across versions too: a broadcast it +/// learned over moq-lite-06 enters the route layer as one of its own claims, yet never +/// reaches a moq-lite-07 peer, while one published there does. +#[kio::test] +async fn a_node_that_does_not_proxy_keeps_what_it_learned_over_lite06() { + let lite07: Version = "moq-lite-07-wip".parse().unwrap(); + let lite06: Version = "moq-lite-06".parse().unwrap(); + let first = produce_origin(1); + let app = produce_origin(2); + let second = produce_origin(3); + let mut pairs = Vec::new(); + for (version, relay) in [(lite06, &first), (lite07, &second)] { + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(app.consume()); + options.client_subscribe = Some(app.clone()); + options.server_publish = Some(relay.consume().with_hidden(true)); + options.server_subscribe = Some(relay.clone().with_link(origin::Link::Peer)); + options.proxy = false; + pairs.push(connect_mock(options).await); + } + + let mut at_app = watch(app.consume().routes()); + let mut at_second = watch(second.consume().routes()); + let learned = first.route("room/learned", origin::RouteOptions::hidden()).unwrap(); + learned.announce().unwrap(); + let local = app.route("room/local", origin::RouteOptions::hidden()).unwrap(); + local.announce().unwrap(); + kio::clock::sim::idle().await; + let seen_by_app = drain(&mut at_app); + assert!(seen_by_app.contains_key("room/learned"), "{seen_by_app:?}"); + let seen = drain(&mut at_second); + assert_eq!(seen.get("room/local"), Some(&vec![Kind::Start]), "{seen:?}"); + assert!(!seen.contains_key("room/learned"), "{seen:?}"); +} From 9f9c45d5290c9e2f8b7056b674ed80886dec580f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:11:47 -0800 Subject: [PATCH 45/61] fix(net): a job's Serve keeps its place ahead of its claim's Unserve The takeover that motivated the queue-order rule is gone, but a job continuing its claim's instance needs its route in place before the claim's goes, so the rule and its test stay, retargeted at a job. --- rs/moq-net/src/route/node.rs | 50 +++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index e8cc2ff57bc..bd3b20ef44c 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -115,8 +115,11 @@ struct Outbox { pending: HashMap, /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, - /// Each claim's last Serve or Unserve, read after the rest, keyed by when it last - /// changed. + /// Each claim's last Serve or Unserve, read after the rest, in queue order. A job + /// queues its Serve before the Unserve of the claim it continues, and the session + /// needs the job's route in place before the claim's goes, so a reader resolved + /// through the claim carries on: so a Serve replacing a queued Serve keeps its + /// place, while an Unserve, or a Serve after one, goes last. serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, @@ -212,9 +215,17 @@ impl Outbox { // Each serving change replaces the last: the session acts on where the claim // ends up, in place when it keeps its instance. Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { + let key = (id, claim.clone()); + if let Some(&seq) = self.serving.get(&key) + && matches!(output, Output::Serve { .. }) + && matches!(self.serves.get(&seq), Some(Output::Serve { .. })) + { + self.serves.insert(seq, output); + return None; + } let seq = self.next; self.next += 1; - if let Some(old) = self.serving.insert((id, claim.clone()), seq) { + if let Some(old) = self.serving.insert(key, seq) { self.serves.remove(&old); } self.serves.insert(seq, output); @@ -823,6 +834,39 @@ mod tests { assert_eq!(claims(outputs(&b)), [p("live/b/y")]); } + /// A stalled stream reads a job's Serve before the Unserve of the claim it + /// continues, even once the job changed again meanwhile, so a reader resolved + /// through the claim finds the job's route before the claim's goes. + #[test] + fn a_stalled_stream_reads_a_job_before_its_claim_goes() { + let instance = Instance::random(); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&instance, 0))); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + outputs(&fast); + outputs(&stalled); + + origin.own(p("live/x"), Some(claim(&instance, 0))); + origin.own(p("live"), None); + pump(&fast_answer, &fast); + origin.own(p("live/x"), Some(claim(&instance, 5))); + pump(&fast_answer, &fast); + let read: Vec<(Path, Option)> = (served(outputs(&stalled)).into_iter()) + .map(|output| match output { + Output::Serve { claim, served, .. } => (claim, Some(served.claim.cost)), + Output::Unserve { claim, .. } => (claim, None), + other => panic!("not a serving change: {other:?}"), + }) + .collect(); + assert_eq!(read, [(p("live/x"), Some(5)), (p("live"), None)]); + } + /// A claim of a scoped peer left with no usable route stays where it is served, /// under the name that peer gave its identity, so the requests it serves meanwhile /// still name it as the peer does. From 4365701e0d1a5ccea4f349b2c8f35e13dee32a2b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:23:31 -0800 Subject: [PATCH 46/61] fix(net): a route reports the instance it serves now, in Rust and the bindings An unannounced job reported the lent instance it was created under, even after a sibling's end moved the claim on, while JS reported the claim's current one. Rust now reads the claim's current instance, as JS does, and moq-c and moq-ffi read the route's instance on each call instead of caching the one minted at creation. The docs qualify lending and say how lite-07 ties hash the origin node rather than the instance. --- doc/bin/relay/cluster.md | 8 +++++--- doc/concept/identity.md | 7 ++++--- doc/lib/js/net.md | 2 +- doc/lib/rs/moq-net.md | 2 +- go/wrapper/origin.go | 11 +++++++---- py/moq-rs/README.md | 4 ++-- py/moq-rs/moq/origin.py | 12 ++++++++---- rs/moq-c/src/api.rs | 14 +++++++++----- rs/moq-c/src/origin.rs | 8 +++++--- rs/moq-c/src/test.rs | 6 ++++-- rs/moq-ffi/src/origin.rs | 23 ++++++++++++++++------- rs/moq-ffi/src/test.rs | 6 +++++- rs/moq-net/src/model/origin.rs | 21 ++++++++++++++++----- swift/Sources/Moq/Origin.swift | 7 +++++-- 14 files changed, 88 insertions(+), 43 deletions(-) diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index a08f00c2984..6810a5879fa 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -49,9 +49,11 @@ every frame once. A publisher whose groups restart, such as an encoder restarting from group 0, publishes under a new instance, which replaces the old broadcast instead of resuming it; the RTMP, SRT, and WHIP ingests mint one per connection. Among instances every relay ranks routes in [one -order](/concept/identity#choosing-a-route), ending in a hash of the instance -and the path, so the relays of a mesh agree on each path's stream and a pool's -paths spread over its hosts. Seamless failover needs moq-lite 07 on every link +order](/concept/identity#choosing-a-route), ending in a hash of the path with +the route's origin node on moq-lite 07, or with the instance on older wires, so +the relays of a mesh agree on each path's stream and a pool's paths spread over +its hosts. On moq-lite 07 a path therefore stays on its host when the instance +changes. Seamless failover needs moq-lite 07 on every link the route crosses. Over moq-lite 06 and older, or moq-transport, the receiving relay mints an instance per announcement, so failover onto such a route is announced as a restart. Viewers re-request on a restart, and a relay drops its diff --git a/doc/concept/identity.md b/doc/concept/identity.md index fdd27cc2607..4aad183b886 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -73,9 +73,10 @@ A route resolving a path is chosen in this order: because the sequence space continues. 3. Among different instances: the preferred link, then an identified chain over an anonymous one, then the lowest cost, then a broadcast published on - this origin, then the shortest hop chain, then a hash of the instance and - the full requested path. Every relay agrees on the hash, and it spreads a - pool's paths over its hosts. A route that entered a node for exactly the + this origin, then the shortest hop chain, then a hash of the full requested + path with the route's origin node, which moq-lite 07 carries, or else with + the instance. Every relay agrees on the hash, and it spreads a pool's paths + over its hosts. A route that entered a node for exactly the requested path is not hashed: the newest one wins, so a restarted publisher takes over from its lingering session at once. diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 55e802b19f0..6df104d1de9 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -41,7 +41,7 @@ route.announce(); - **The origin holds the routes, not the connection.** `origin.route(path)` creates a route that owns a new broadcast; fill its tracks through `route.broadcast`. Closing a session withdraws its announcements and leaves the routes for the next one. A route is visible at once unless created `{ hidden: true }`, which keeps it invisible, locally and remotely, until `announce()`. `withdraw()` hides it again and `announce()` brings the same stream back. `close()` retracts it and closes its broadcast, as closing the broadcast does. - **Answer claimed paths with a route.** `origin.routePrefix(prefix)` claims a prefix, and `route.requested()` yields each requested path. Answer with `request.route()`, a new exact route under the claim's instance (a fresh one if the claim names its own), so readers resolved through the claim continue, or `request.route({ hidden: true })` to fill its broadcast before `announce()` answers the requesters; closing it or its broadcast before then refuses them as unroutable, so `using job = request.route({ hidden: true })` refuses on a throw. Serve a broadcast you already hold with `request.serve(broadcast)`, or refuse with `request.reject(err)`. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, `finish()` each track, then `close()` it; closing a track without finishing aborts it. The broadcast stays announced and keeps serving them until you `close()` it. How long to keep it up for late readers is yours to decide. -- **Requests can pin an instance.** `consumer.request(path, { instance })` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances). The resolved broadcast's `instance` names the stream it came through, pinned or not. A route mints a random instance when it is created and keeps it across `withdraw()` and `announce()`; pass `origin.route(path, { instance, epoch })` only for a publisher that continues its group sequences across restarts and raises the epoch each time. +- **Requests can pin an instance.** `consumer.request(path, { instance })` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances). The resolved broadcast's `instance` names the stream it came through, pinned or not. `origin.route(path)` mints a random instance when it is created and keeps it across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Pass `origin.route(path, { instance, epoch })` only for a publisher that continues its group sequences across restarts and raises the epoch each time. - **Follow restarts.** `origin.follow(path)` follows whichever publisher instance serves the path: its `active` changes to a fresh broadcast on a restart, clears on an end, and resolves again on the next start, so read it reactively and close it when done. Instances are never stitched, so subscribe again to each new broadcast. Another route of the same instance taking over (instances travel only on `moq-lite-07-wip`; older wires mint one per announcement) keeps `active` as is, and each track subscription resumes there from the first group it lacks. A refusal ends it. Pass `{ instance }` to pin one stream, which it waits for again after an end. `request(path)` instead keeps its accepted instance and ends once that instance stops serving; while pending, it fails `Unroutable` once discovery of its path has synced without a route for it. To list what is live, iterate `announced(scope)`. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A `consume` origin asks the relay for announcements only under the prefixes its readers watch with `announced(scope)`, `request(path)`, or `follow(path)`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. Every request and follower watches its path from creation until closed, so it outlives the listing that found it. A relay that refuses an announce request downgrades that session: requests subscribe blind through it, and followers keep waiting for a session that can discover. - **Connections share by URL and versions.** `new Connection({ url })` pools and reconnects with backoff, which the elements use. An ordered `versions` list, spelled like `moq-lite-06` as in Rust, applies to both WebTransport and WebSocket; equivalent lists share, while a different order or an omitted list gets its own pool entry. Supplying your own transport options or origin selects a private loop. Receive estimates are on demand: observe `connection.probe` reactively or subscribe while you need them; `peek()` alone opens no PROBE stream. diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 1449c8678ca..8dc2f0246a6 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -17,7 +17,7 @@ What you use it for, beyond what the concept page already describes: - **An origin outlives the session.** `origin.route(path, options)` returns a `Route` that owns the broadcast to fill (`route.broadcast()`) and controls its announcement: `announce()`, `withdraw()`, `set_cost()`, and dropping the last clone retracts it. `origin.route_prefix(prefix, options)` serves requests beneath a prefix instead, each answered with `request.serve(held)`, `request.route(options)` (the path's own exact route) or `reject`. A standalone broadcast cannot announce; publish one only through a prefix route's `serve`. A reconnect announces existing routes again, and closing the session does not delete them. - **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()`, `cost()`, the hop ids it crossed (`hops()`, oldest first, 0 for an anonymous hop) and `is_anonymous()`; an application sets only `RouteOptions`. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. -- **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. A route serves a random instance minted when it is created and kept across `withdraw()` and `announce()`; set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. +- **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a prefix route moves to a fresh instance each time a job holding its current one ends, and a job takes the prefix route's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths only as present or absent. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-rtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index 63ecfacbbc0..9c0c5d42bef 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -95,8 +95,10 @@ func (r *Route) Broadcast() (*BroadcastProducer, error) { return r.broadcast, nil } -// Instance returns the stream this route serves, minted at creation unless -// [RouteOptions] named one. +// Instance returns the stream this route serves now. A named instance never +// changes. Unnamed, a prefix route moves to a fresh one each time a job holding +// its current one ends, and a job's route reports its prefix route's current one +// until it first announces. func (r *Route) Instance() string { return r.inner.InstanceId() } @@ -164,8 +166,9 @@ func (r *BroadcastRequest) Path() (string, error) { // // Unless options names an instance, it takes the one its prefix route lends // when it first announces, so readers that resolved the path through the prefix -// route continue on it; its end moves the prefix route to a fresh instance, so a -// restarted job is a new stream. Created visible, it announces now and answers +// route continue on it; its end then moves the prefix route to a fresh instance, +// so a restarted job is a new stream. A prefix route that names its instance +// lends none, and the route mints its own. Created visible, it announces now and answers // the waiting requesters; created hidden, they wait // until [Route.Announce], so the job can fill its catalog first. func (r *BroadcastRequest) Route(options RouteOptions) (*Route, error) { diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 78acb272b95..7d6074cb905 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -286,9 +286,9 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect - `.route_prefix(prefix, *, hidden=False, cost=0, instance=None, epoch=0) → Route`. A prefix route that yields the requests beneath it. - **`Route`**. A route on an origin. Hold it: dropping or cancelling it retracts it, and so does closing its broadcast. - `.broadcast → BroadcastProducer`. The broadcast an exact route owns; raises `Internal` on a prefix route. - - `.instance`. The stream it serves, kept across `withdraw()` and `announce()`. + - `.instance`. The stream it serves now, kept across `withdraw()` and `announce()`. Unless one is named, a prefix route moves to a fresh one each time a job holding it ends, and a job reports its prefix route's until it first announces. - `.announce()`, `.withdraw()`, `.set_cost(cost)`. Show, hide, or re-price it in place; `announce` and `set_cost` raise `Closed` once the broadcast closed. - - `await .requested() → BroadcastRequest | None`, or async iteration over a prefix route. Call `.serve(consumer)` to answer with held content under the prefix route's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the prefix route's instance until the job ends and then moves the prefix route to a fresh one, or `.reject(code)` to fail the requester. + - `await .requested() → BroadcastRequest | None`, or async iteration over a prefix route. Call `.serve(consumer)` to answer with held content under the prefix route's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the prefix route's instance until the job ends and then moves the prefix route to a fresh one (unless either names an instance: a named prefix route lends none, and a named job keeps its own), or `.reject(code)` to fail the requester. - **`OriginConsumer`**. Discover broadcasts. - `.announced(prefix, filter=None) → AnnounceConsumer` (async iterator of `AnnounceEvent`: `AnnounceEventPresent` or `AnnounceEventAbsent`, each carrying an `Announce` whose `.route.instance` names the stream; another instance taking a path over keeps it present, so `follow` the path to see that); `filter` is a pattern relative to the literal prefix, while each `Announce.prefix` stays origin-relative and `.captures` is a required list of wildcard matches, empty without wildcards - `.follow(path, instance=None) → Following` (awaitable, waits until something serves the path, then returns each stream that takes over, or `None` once the path goes offline; an `instance` follows only that stream) diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index d3843b23458..a1e0faa348b 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -121,8 +121,9 @@ def route(self, *, hidden: bool = False, cost: int = 0, instance: str | None = N Unless ``instance`` names one, the route takes the instance its prefix route lends when it first announces, so readers that resolved the path through the prefix route - continue on it; its end moves the prefix route to a fresh instance, so a job that - restarts is a new stream. Created visible, it announces now and answers the waiting + continue on it; its end then moves the prefix route to a fresh instance, so a job that + restarts is a new stream. A prefix route that names its instance lends none, and the + route mints its own. Created visible, it announces now and answers the waiting requesters; created ``hidden``, they wait until :meth:`Route.announce`, so the job can fill its catalog first, and cancelling the route before then refuses them. Ending the route withdraws only this path. @@ -165,8 +166,11 @@ def broadcast(self) -> BroadcastProducer: @property def instance(self) -> str: - """The stream this route serves, minted at creation unless one was named, and kept - across :meth:`withdraw` and :meth:`announce`.""" + """The stream this route serves now. + + A named instance never changes. Unnamed, a prefix route moves to a fresh one each time + a job holding its current one ends, and a job's route reports its prefix route's + current one until it first announces. Withdrawing and announcing keep it.""" return self._inner.instance_id() def announce(self) -> None: diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index a190e85d0ef..2c6fcddc2bf 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -2122,10 +2122,13 @@ pub extern "C" fn moq_route_broadcast(route: u32) -> i32 { }) } -/// The stream a route serves, minted at creation unless its options named one. +/// The stream a route serves now. /// -/// Kept across [moq_route_withdraw] and [moq_route_announce]. The destination borrows the -/// route's storage until [moq_route_free]. +/// Named options keep one instance. Unnamed, a prefix route moves to a fresh one each time a +/// job holding its current one ends, and a route from [moq_broadcast_request_route] reports +/// its prefix route's current one until it first announces, binding whichever that is then. +/// Withdrawing and announcing again keep it. The destination borrows the route's storage +/// until the next [moq_route_instance] on the route or [moq_route_free]. /// /// Returns zero on success, or a negative code on failure. /// @@ -2248,8 +2251,9 @@ pub extern "C" fn moq_broadcast_request_serve(request: u32, broadcast: u32) -> i /// Consumes the request handle and returns a positive route handle, or a negative error. /// Unless `options` names an instance, the route takes the one its prefix route lends when /// it first announces, so readers that resolved the path through the prefix route continue -/// on it; its end moves the prefix route to a fresh instance, so a job that restarts is a -/// new stream. Created visible, it announces now and answers the waiting requesters; +/// on it; its end then moves the prefix route to a fresh instance, so a job that restarts is +/// a new stream. A prefix route that names its instance lends none, and the route mints its +/// own. Created visible, it announces now and answers the waiting requesters; /// created hidden, they wait until [moq_route_announce], so the job can fill its catalog /// first, and freeing the route before then refuses them. Ending the route withdraws only /// this path. Unreadable `options` fail without consuming the request. diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index 481ca1d27f2..b0482e651cc 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -150,7 +150,7 @@ impl State { struct RouteEntry { inner: moq_net::origin::Route, driver: crate::ffi::Background, - /// Minted at creation and kept for the route's lifetime, borrowed by [Origin::route_instance]. + /// The instance last read, borrowed by [Origin::route_instance]. instance: String, /// An exact route's broadcast fill handle, or a prefix route's request delivery task. kind: RouteKind, @@ -519,8 +519,10 @@ impl Origin { } } - pub fn route_instance(&self, route: Id, dst: &mut moq_string) -> Result<(), Error> { - let entry = self.route_entry(route)?; + pub fn route_instance(&mut self, route: Id, dst: &mut moq_string) -> Result<(), Error> { + let entry = self.routes.get_mut(route).ok_or(Error::NotFound)?; + entry.driver.check()?; + entry.instance = entry.inner.instance().to_string(); *dst = moq_string { data: entry.instance.as_ptr().cast::(), len: entry.instance.len(), diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 3b5d9a8fa8c..294387f34c3 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -4130,13 +4130,15 @@ fn dynamic_serves_a_request_under_a_prefix() { unsafe { moq_broadcast_request_route(request, std::ptr::null()) } < 0, "request is consumed" ); - assert_eq!(instance_of(job), instance_of(dynamic)); + let lent = instance_of(dynamic); + assert_eq!(instance_of(job), lent); let served = id(moq_route_broadcast(job)); assert_eq!(moq_route_announce(job), 0); assert!(req_cb.recv() > 0); req_cb.recv_terminal(); - // The answer's route is the job's to withdraw. + // The answer's route is the job's to withdraw, which moves the prefix route on. assert_eq!(moq_route_withdraw(job), 0); + assert_ne!(instance_of(dynamic), lent); assert_eq!(moq_route_free(job), 0); assert_eq!(moq_route_free(dynamic), 0); diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index b6e6fbec248..28c5d7579cf 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -152,8 +152,8 @@ pub struct MoqRoute { reads: Polled, /// An exact route's broadcast, wrapped once so every caller shares its catalog. broadcast: Option>, - /// Minted at creation and kept for the route's lifetime. - instance: String, + /// The instance last read, reported once the route is cancelled. + instance: std::sync::Mutex, driver: Background, } @@ -441,7 +441,7 @@ impl MoqRoute { fn new(route: moq_net::origin::Route, broadcast: Option>, driver: Background) -> Self { Self { - instance: route.instance().to_string(), + instance: std::sync::Mutex::new(route.instance().to_string()), reads: Polled::new(route).supervised(driver.clone()), broadcast, driver, @@ -469,13 +469,21 @@ impl MoqRoute { }) } - /// The stream this route serves, minted at creation unless the options named one. + /// The stream this route serves now, or the last one once cancelled. + /// + /// Named options keep one instance. Unnamed, a prefix route moves to a fresh one each + /// time a job holding its current one ends, and a job's route reports its prefix route's + /// current one until it first announces, binding whichever that is then. /// /// Named `instance_id` rather than `instance` because the generated C and C++ /// objects keep their handle in a private field named `instance`; the wrappers /// expose it as `instance`. pub fn instance_id(&self) -> String { - self.instance.clone() + let mut last = self.instance.lock().unwrap(); + if let Some(current) = self.reads.peek(|route| route.instance().to_string()) { + *last = current; + } + last.clone() } /// Make the route visible, or keep it so. @@ -555,8 +563,9 @@ impl MoqBroadcastRequest { /// /// Unless `options` names an instance, the route takes the one its prefix route lends /// when it first announces, so readers that resolved the path through the prefix route - /// continue on it; its end moves the prefix route to a fresh instance, so a job that - /// restarts is a new stream. Created visible, it announces now and answers the waiting + /// continue on it; its end then moves the prefix route to a fresh instance, so a job that + /// restarts is a new stream. A prefix route that names its instance lends none, and the + /// route mints its own. Created visible, it announces now and answers the waiting /// requesters; created hidden, they wait until [`MoqRoute::announce`], so the job can /// fill its catalog first, and cancelling the route before then refuses them. Ending the /// route withdraws only this path. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 256dde8d82f..d2bb1f59198 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -5009,7 +5009,7 @@ async fn dynamic_broadcast_request() { let job = request.route(hidden()).unwrap(); let served = job.broadcast().unwrap(); let track = served.publish_track("status".into(), Some(timed_info())).unwrap(); - // Only announce answers the requester, under a fresh instance and after admission. + // Only announce answers the requester, under the claim's instance and after admission. let instance = job.instance_id(); job.announce().unwrap(); // A re-price keeps the instance the answer was announced under. @@ -5040,8 +5040,12 @@ async fn dynamic_broadcast_request() { assert_eq!(frame.payload, payload); assert_eq!(frame.timestamp.map(micros), Some(20_000)); + // The job carried the claim's instance, and its end moves the claim on. + assert_eq!(dynamic.instance_id(), instance); track.finish().unwrap(); served.close().unwrap(); + job.cancel(); + assert_ne!(dynamic.instance_id(), instance); } /// A prefix serves requests beneath it; cancelling the handle diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index bc8d60c5e19..a1d5accfb4c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2950,7 +2950,9 @@ struct RouteControl { impl RouteControl { fn new(options: RouteOptions, announcement: &AnnounceProducer, request: Option) -> Self { let held = options.instance.is_none(); - let instance = announcement.instance().expect("a route just inserted has its entries"); + let instance = announcement + .instance(None) + .expect("a route just inserted has its entries"); Self { options, instance, @@ -3008,11 +3010,13 @@ impl Route { /// /// Unnamed, it can change: a prefix route ([`Producer::route_prefix`]) moves to a fresh /// instance each time an answer holding its current one ends, and a job's route - /// ([`Request::route`]) takes its prefix route's when it first announces. + /// ([`Request::route`]) reports its prefix route's current one until it first announces, + /// binding whichever that is then. pub fn instance(&self) -> Instance { let mut control = self.state.control.lock(); + let lender = control.request.as_ref().map(|request| &request.home); if control.held - && let Some(current) = self.state.announcement.instance() + && let Some(current) = self.state.announcement.instance(lender) { control.instance = current; } @@ -3354,11 +3358,15 @@ impl AnnounceProducer { Ok(true) } - /// The instance and epoch the entries carry, or `None` once they are gone. - fn instance(&self) -> Option<(Instance, u64)> { + /// The instance and epoch the entries carry, or `None` once they are gone. A job's + /// route not yet announced reads its `lender`'s current one, which it would bind now. + fn instance(&self, lender: Option<&kio::Shared>) -> Option<(Instance, u64)> { let shared = self.shared.lock(); let (prefix, id) = self.entries.first()?; let entry = shared.routes.entry(prefix, *id)?; + if let Some(lent) = lender.and_then(|server| shared.lender(prefix, server)) { + return Some((lent.instance, lent.epoch)); + } Some((entry.instance.clone(), entry.epoch)) } @@ -12243,10 +12251,13 @@ mod tests { bob.await.unwrap(); let first = claim.instance(); assert_eq!(sibling.instance(), first); + assert_eq!(prepared.instance(), first); drop(sibling); let second = claim.instance(); assert_ne!(second, first); + // Unannounced, it reports the instance it would bind now. + assert_eq!(prepared.instance(), second); prepared.announce().unwrap(); assert_eq!(prepared.instance(), second); diff --git a/swift/Sources/Moq/Origin.swift b/swift/Sources/Moq/Origin.swift index 25cf15e48bb..0c7683bfbde 100644 --- a/swift/Sources/Moq/Origin.swift +++ b/swift/Sources/Moq/Origin.swift @@ -70,7 +70,8 @@ public final class BroadcastRequest: Sendable { /// /// Unless `instance` names one, it takes the instance its prefix route lends when it first /// announces, so readers that resolved the path through the prefix route continue on it; - /// its end moves the prefix route to a fresh instance, so a restarted job is a new stream. + /// its end then moves the prefix route to a fresh instance, so a restarted job is a new + /// stream. A prefix route that names its instance lends none, and the route mints its own. /// Created visible, it announces now and answers the waiting requesters; created `hidden`, they wait until /// `Route.announce()`, so the job can fill its catalog first. public func route( @@ -119,7 +120,9 @@ public final class Route: AsyncSequence, Sendable { } } - /// The stream this route serves, minted at creation unless `instance` named one. + /// The stream this route serves now. A named instance never changes. Unnamed, a prefix + /// route moves to a fresh one each time a job holding its current one ends, and a job's + /// route reports its prefix route's current one until it first announces. public var instance: String { ffi.instanceId() } /// Make the route visible, or keep it so; throws `Closed` once its broadcast closed. From 335ebe32809f7f7f58ea6ace1f54545e13a62a0d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Nov 2026 23:53:05 -0800 Subject: [PATCH 47/61] chore(dart): regenerate the UniFFI bindings for the instance_id doc --- dart/moq_ffi/lib/src/moq.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 308143e9c9d..04666f336ad 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -24602,7 +24602,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_reject() != 9727) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 2402) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 53132) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_serve() != 10832) { @@ -24641,7 +24641,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqroute_cancel() != 41411) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqroute_instance_id() != 18728) { + if (uniffi_moq_ffi_checksum_method_moqroute_instance_id() != 6842) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqroute_requested() != 49558) { From b02bdff9f86596376b9c85b379f9bf7dd84895f5 Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sat, 7 Nov 2026 01:02:28 -0800 Subject: [PATCH 48/61] fix(net): initialize routing node in base setup tests --- rs/moq-net/src/lite/subscriber.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 59ab1d3fc95..5a7b7041282 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -6462,6 +6462,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; @@ -6559,6 +6560,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; @@ -6621,6 +6623,7 @@ mod tests { subscriber, path: Path::from_static("room/host"), instance: None, + node: 0, name: "audio".to_string(), }; From 915074a36d7f545844fce6f731de10c2eed658d9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 06:08:11 -0800 Subject: [PATCH 49/61] doc: the C and Dart guides describe how a request's route takes its prefix route's instance Co-Authored-By: Claude Opus 5.5 --- doc/lib/c/index.md | 12 ++++++++---- doc/lib/dart/index.md | 2 +- rs/moq-c/MIGRATE.md | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index fc6690ca6c5..c9ca2d47cfe 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -82,9 +82,9 @@ if (moq_route_announce((uint32_t)route) < 0) The route keeps the broadcast announced: `moq_route_free()` retracts it, and `moq_publish_close()` ends both. A route carries only a cost and the stream's instance; -relays add the hops. Each route mints a fresh instance (`moq_route_instance()`) and keeps it -across `moq_route_withdraw()` and `moq_route_announce()`, so a restarted publisher, being a -new route, is a new stream that `moq_origin_follow` hands viewers instead of a stalled +relays add the hops. `moq_origin_route()` mints a fresh instance (`moq_route_instance()`) and +keeps it across `moq_route_withdraw()` and `moq_route_announce()`, so a restarted publisher, +being a new route, is a new stream that `moq_origin_follow` hands viewers instead of a stalled broadcast; the path stays `MOQ_ANNOUNCE_KIND_PRESENT` throughout. The follow callback reports each instance as `MOQ_FOLLOW_EVENT_ONLINE` with its handle, `MOQ_FOLLOW_EVENT_OFFLINE` when the path goes offline, and one `MOQ_FOLLOW_EVENT_END`. Set `instance` only for a publisher that continues its group sequences @@ -92,7 +92,11 @@ across restarts, and raise `epoch` on each: readers stitch every route of one in together, so restarting the sequences under an old instance splices unrelated groups into their tracks. To serve paths on demand, `moq_origin_route_prefix()` delivers each request to answer with `moq_broadcast_request_serve()`, `moq_broadcast_request_route()`, or -`moq_broadcast_request_reject()`. +`moq_broadcast_request_reject()`. Unless one is named, a prefix route moves to a fresh +instance each time a job holding its current one ends, and a job from +`moq_broadcast_request_route()` takes the prefix route's, so a reader resolved through the +prefix route does not restart. The string `moq_route_instance()` writes is borrowed until the +next `moq_route_instance()` on that route or `moq_route_free()`. Raw video encoders take an exact numerator/denominator frame rate, such as 30000/1001. Publication track names stay separate from codec settings. For simulcast, give each rendition the next one down as the `lower` handle in its `moq_video_encode_options`: it stops encoding, advertised as disabled, while its bandwidth grant cannot fund it, and resumes once the grant recovers. A rung only disables while the session probes for bandwidth, which on native transports takes moq-quic with BBR3. For pre-encoded video, publish from the lowest rendition up and set `moq_video_init.lower` to the next lower media handle. Reserve each track with `moq_allocator_reserve_media`, then bind its reservation with `moq_rate_control_create` and call `moq_rate_control_update` on every frame. Apply changed targets to your encoder and `moq_publish_media_set_enabled`, resuming video on a keyframe; close the control and reservation separately when done. diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index abfc1fe159c..f2d241bd2d2 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -149,7 +149,7 @@ one to one; the API reference has the names, and [Layers](/lib/layers) shows which module holds each. - **Names.** Types drop the `Moq` prefix (`Session`, `BroadcastProducer`, `RouteOptions`) as aliases, so the generated names still work. `MoqContainer`, `MoqRoute`, and the exceptions keep it, since Flutter owns those names. -- **Routes.** `route(path)` publishes a broadcast and `routePrefix(prefix)` answers requests beneath a prefix, each returning a `MoqRoute`. Hold it: cancelling or disposing it retracts the route, while a broadcast handle alone does not keep it announced. It keeps its instance across `withdraw()` and `announce()`, so a new run is a new route. +- **Routes.** `route(path)` publishes a broadcast and `routePrefix(prefix)` answers requests beneath a prefix, each returning a `MoqRoute`. Hold it: cancelling or disposing it retracts the route, while a broadcast handle alone does not keep it announced. A `route(path)` keeps its instance across `withdraw()` and `announce()`, so a new run is a new route. Unless one is named, a request's own route (`request.route(options: ...)`) takes its prefix route's instance, and the prefix route moves to a fresh one each time such a route holding its current one ends. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or less often to batch frames into fewer groups. - **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Media reads.** `media.TrackConsumer`, `audio.Consumer`, and `video.Consumer` return a `Read` in order: a `ReadFrame`, a `ReadGap` where media was lost, or a `ReadEnd` where the publisher stopped its encoder. A gap breaks continuity. An encoder end is not stream EOF: following frames before its endpoint are the old encoder's drain; the first at or past it starts fresh. A decoder reads everything the track still holds unless a live player sets `start: media.DecodeStart.latest` in its decoder output, which joins at the newest group and skips the cached backlog. diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 2f11ec28ba1..9cd2ef818c1 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -39,7 +39,7 @@ A route handle replaces the publication, the prepared answer, and the request ha - `moq_publication_announce(publication, route)` is `moq_route_announce(route)`, and re-pricing is `moq_route_set_cost(route, cost)`. `moq_publication_unannounce` is `moq_route_withdraw`, and `moq_publication_free` is `moq_route_free`. - `moq_origin_requests(origin, prefix, len, on_request, user_data)` is `moq_origin_route_prefix(origin, prefix, len, options, on_request, user_data)`, and `moq_origin_requests_cancel` is `moq_route_free`. - `moq_broadcast_request_prepare(request, &broadcast)` then `moq_broadcast_prepared_publish` is `moq_broadcast_request_route(request, options)`: create it hidden to fill it first, then `moq_route_announce`. `moq_broadcast_prepared_free` is `moq_route_free`. -- `moq_route_instance(route, &dst)` reads the stream's instance, minted at creation and kept across withdraw and announce. +- `moq_route_instance(route, &dst)` reads the instance the route serves now, borrowed until the next `moq_route_instance` on that route or `moq_route_free`. An exact route mints it at creation and keeps it across withdraw and announce; a job takes its prefix route's. `moq_announce_route` is `moq_route_options`, given once at creation, and gains a trailing `hidden`. A zeroed or NULL `moq_route_options` is visible at once, so set `hidden` and call `moq_route_announce` after filling the catalog to keep the old create-then-announce order. A stable `instance` is fixed for the route's lifetime: announcing another one is a new route. From 8906c97a04cd6b118f9da6e63a9d06495b3b649a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 06:25:41 -0800 Subject: [PATCH 50/61] quest: MIGRATE.md stays for the release cut; the C note lives in ## Impact Co-Authored-By: Claude Opus 5.5 --- rs/moq-c/MIGRATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 9cd2ef818c1..2f11ec28ba1 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -39,7 +39,7 @@ A route handle replaces the publication, the prepared answer, and the request ha - `moq_publication_announce(publication, route)` is `moq_route_announce(route)`, and re-pricing is `moq_route_set_cost(route, cost)`. `moq_publication_unannounce` is `moq_route_withdraw`, and `moq_publication_free` is `moq_route_free`. - `moq_origin_requests(origin, prefix, len, on_request, user_data)` is `moq_origin_route_prefix(origin, prefix, len, options, on_request, user_data)`, and `moq_origin_requests_cancel` is `moq_route_free`. - `moq_broadcast_request_prepare(request, &broadcast)` then `moq_broadcast_prepared_publish` is `moq_broadcast_request_route(request, options)`: create it hidden to fill it first, then `moq_route_announce`. `moq_broadcast_prepared_free` is `moq_route_free`. -- `moq_route_instance(route, &dst)` reads the instance the route serves now, borrowed until the next `moq_route_instance` on that route or `moq_route_free`. An exact route mints it at creation and keeps it across withdraw and announce; a job takes its prefix route's. +- `moq_route_instance(route, &dst)` reads the stream's instance, minted at creation and kept across withdraw and announce. `moq_announce_route` is `moq_route_options`, given once at creation, and gains a trailing `hidden`. A zeroed or NULL `moq_route_options` is visible at once, so set `hidden` and call `moq_route_announce` after filling the catalog to keep the old create-then-announce order. A stable `instance` is fixed for the route's lifetime: announcing another one is a new route. From 3bd73637d1c12958d85df2f3b4db3f461f1aba16 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 06:39:41 -0800 Subject: [PATCH 51/61] doc: route-hash agreement and job continuation hold on moq-lite 07 Co-Authored-By: Claude Opus 5.5 --- doc/concept/identity.md | 3 ++- doc/lib/c/index.md | 4 ++-- doc/lib/dart/index.md | 2 +- doc/lib/js/net.md | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 4aad183b886..d86bab5897a 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -75,7 +75,8 @@ A route resolving a path is chosen in this order: over an anonymous one, then the lowest cost, then a broadcast published on this origin, then the shortest hop chain, then a hash of the full requested path with the route's origin node, which moq-lite 07 carries, or else with - the instance. Every relay agrees on the hash, and it spreads a pool's paths + the instance. Relays agree on the hash only where routes carry a consistent + origin node, which takes moq-lite 07 on every hop, and it spreads a pool's paths over its hosts. A route that entered a node for exactly the requested path is not hashed: the newest one wins, so a restarted publisher takes over from its lingering session at once. diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index c9ca2d47cfe..af9df1869d8 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -94,8 +94,8 @@ their tracks. To serve paths on demand, `moq_origin_route_prefix()` delivers eac answer with `moq_broadcast_request_serve()`, `moq_broadcast_request_route()`, or `moq_broadcast_request_reject()`. Unless one is named, a prefix route moves to a fresh instance each time a job holding its current one ends, and a job from -`moq_broadcast_request_route()` takes the prefix route's, so a reader resolved through the -prefix route does not restart. The string `moq_route_instance()` writes is borrowed until the +`moq_broadcast_request_route()` takes the prefix route's, so on moq-lite 07 a reader resolved through the +prefix route does not restart; older wires mint another instance for the job's route, restarting it. The string `moq_route_instance()` writes is borrowed until the next `moq_route_instance()` on that route or `moq_route_free()`. Raw video encoders take an exact numerator/denominator frame rate, such as 30000/1001. Publication track names stay separate from codec settings. For simulcast, give each rendition the next one down as the `lower` handle in its `moq_video_encode_options`: it stops encoding, advertised as disabled, while its bandwidth grant cannot fund it, and resumes once the grant recovers. A rung only disables while the session probes for bandwidth, which on native transports takes moq-quic with BBR3. For pre-encoded video, publish from the lowest rendition up and set `moq_video_init.lower` to the next lower media handle. Reserve each track with `moq_allocator_reserve_media`, then bind its reservation with `moq_rate_control_create` and call `moq_rate_control_update` on every frame. Apply changed targets to your encoder and `moq_publish_media_set_enabled`, resuming video on a keyframe; close the control and reservation separately when done. diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index f2d241bd2d2..9a122a3a087 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -149,7 +149,7 @@ one to one; the API reference has the names, and [Layers](/lib/layers) shows which module holds each. - **Names.** Types drop the `Moq` prefix (`Session`, `BroadcastProducer`, `RouteOptions`) as aliases, so the generated names still work. `MoqContainer`, `MoqRoute`, and the exceptions keep it, since Flutter owns those names. -- **Routes.** `route(path)` publishes a broadcast and `routePrefix(prefix)` answers requests beneath a prefix, each returning a `MoqRoute`. Hold it: cancelling or disposing it retracts the route, while a broadcast handle alone does not keep it announced. A `route(path)` keeps its instance across `withdraw()` and `announce()`, so a new run is a new route. Unless one is named, a request's own route (`request.route(options: ...)`) takes its prefix route's instance, and the prefix route moves to a fresh one each time such a route holding its current one ends. +- **Routes.** `route(path)` publishes a broadcast and `routePrefix(prefix)` answers requests beneath a prefix, each returning a `MoqRoute`. Hold it: cancelling or disposing it retracts the route, while a broadcast handle alone does not keep it announced. A `route(path)` keeps its instance across `withdraw()` and `announce()`, so a new run is a new route. Unless one is named, a request's own route (`request.route(options: ...)`) takes its prefix route's instance (on moq-lite 07; older wires mint another, restarting readers resolved through the prefix route), and the prefix route moves to a fresh one each time such a route holding its current one ends. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `cut()`: after every frame, or less often to batch frames into fewer groups. - **Timeline breaks.** On a seek or pause, call `discontinuity()`, then keep timestamps moving forward and resume video on a keyframe. - **Media reads.** `media.TrackConsumer`, `audio.Consumer`, and `video.Consumer` return a `Read` in order: a `ReadFrame`, a `ReadGap` where media was lost, or a `ReadEnd` where the publisher stopped its encoder. A gap breaks continuity. An encoder end is not stream EOF: following frames before its endpoint are the old encoder's drain; the first at or past it starts fresh. A decoder reads everything the track still holds unless a live player sets `start: media.DecodeStart.latest` in its decoder output, which joins at the newest group and skips the cached backlog. diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 6df104d1de9..b467e2c1a06 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -39,7 +39,7 @@ route.announce(); ``` - **The origin holds the routes, not the connection.** `origin.route(path)` creates a route that owns a new broadcast; fill its tracks through `route.broadcast`. Closing a session withdraws its announcements and leaves the routes for the next one. A route is visible at once unless created `{ hidden: true }`, which keeps it invisible, locally and remotely, until `announce()`. `withdraw()` hides it again and `announce()` brings the same stream back. `close()` retracts it and closes its broadcast, as closing the broadcast does. -- **Answer claimed paths with a route.** `origin.routePrefix(prefix)` claims a prefix, and `route.requested()` yields each requested path. Answer with `request.route()`, a new exact route under the claim's instance (a fresh one if the claim names its own), so readers resolved through the claim continue, or `request.route({ hidden: true })` to fill its broadcast before `announce()` answers the requesters; closing it or its broadcast before then refuses them as unroutable, so `using job = request.route({ hidden: true })` refuses on a throw. Serve a broadcast you already hold with `request.serve(broadcast)`, or refuse with `request.reject(err)`. +- **Answer claimed paths with a route.** `origin.routePrefix(prefix)` claims a prefix, and `route.requested()` yields each requested path. Answer with `request.route()`, a new exact route under the claim's instance (a fresh one if the claim names its own), so on `moq-lite-07-wip` readers resolved through the claim continue (older wires mint another instance for it, restarting their subscription), or `request.route({ hidden: true })` to fill its broadcast before `announce()` answers the requesters; closing it or its broadcast before then refuses them as unroutable, so `using job = request.route({ hidden: true })` refuses on a throw. Serve a broadcast you already hold with `request.serve(broadcast)`, or refuse with `request.reject(err)`. - **End content by finishing tracks.** A broadcast has no finished state. When the input ends, `finish()` each track, then `close()` it; closing a track without finishing aborts it. The broadcast stays announced and keeps serving them until you `close()` it. How long to keep it up for late readers is yours to decide. - **Requests can pin an instance.** `consumer.request(path, { instance })` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances). The resolved broadcast's `instance` names the stream it came through, pinned or not. `origin.route(path)` mints a random instance when it is created and keeps it across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Pass `origin.route(path, { instance, epoch })` only for a publisher that continues its group sequences across restarts and raises the epoch each time. - **Follow restarts.** `origin.follow(path)` follows whichever publisher instance serves the path: its `active` changes to a fresh broadcast on a restart, clears on an end, and resolves again on the next start, so read it reactively and close it when done. Instances are never stitched, so subscribe again to each new broadcast. Another route of the same instance taking over (instances travel only on `moq-lite-07-wip`; older wires mint one per announcement) keeps `active` as is, and each track subscription resumes there from the first group it lacks. A refusal ends it. Pass `{ instance }` to pin one stream, which it waits for again after an end. `request(path)` instead keeps its accepted instance and ends once that instance stops serving; while pending, it fails `Unroutable` once discovery of its path has synced without a route for it. To list what is live, iterate `announced(scope)`. See [identity and recovery](/concept/identity) to choose between following and holding one instance. From b327b8eaa432c83dce01eb3f2aabc40366f67c5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 07:24:46 -0800 Subject: [PATCH 52/61] test(interop): skip the browser route key lane until JS speaks the routed lite-07 The lane main added in #8116 dials the relay from a browser on moq-lite-07-wip, which js/net still speaks from before the route layer. It records a skip naming routes/js.md, like the other JS lite-07 lanes. Co-Authored-By: Claude Opus 5.5 --- quest/m2/cluster-routing/routes/js.md | 4 ++-- test/interop/README.md | 9 ++++++--- test/interop/interop.sh | 5 +++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/quest/m2/cluster-routing/routes/js.md b/quest/m2/cluster-routing/routes/js.md index eb052a55830..2c5126b740e 100644 --- a/quest/m2/cluster-routing/routes/js.md +++ b/quest/m2/cluster-routing/routes/js.md @@ -25,8 +25,8 @@ the cheapest one; fix it with the route-selection rework here. so the nightly report stays meaningful. Rejected: leave them red; hold #8023 for this quest. Every skip names this quest (`grep -rn routes/js.md test rs`): `JS_LITE07_SKIP` in `test/interop/interop.sh` and `socket.sh` - (native JS grant printing, claims, track replacement, sockets, the lite-07 - compress and drop legs, the compressed catalog's lite-07 reader, and + (native JS grant printing, claims, track replacement, sockets, the browser + route key proof, the lite-07 compress and drop legs, the compressed catalog's lite-07 reader, and `--versions moq-lite-07-wip` with JS clients), `lite_tracks_interop` in `test/justfile`, and the lite-07 legs of `max_age_relay_javascript` and `relay_fill_javascript`. diff --git a/test/interop/README.md b/test/interop/README.md index 0382fe8437f..e91c5817136 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -23,8 +23,9 @@ request ends while its follower waits for the publisher to arrive. Until [`js/net` speaks lite-07's route layer](../../quest/m2/cluster-routing/routes/js.md), native JS dials its defaults in the matrix, leaving its grant unchecked, and every lane that would run it on lite-07 records a skip naming that quest: claims, track -replacement, sockets, and the lite-07 compress and drop legs. The compressed catalog -runs without its lite-07 reader, and `--versions moq-lite-07-wip` refuses JS clients. +replacement, sockets, the browser route key proof, and the lite-07 compress and +drop legs. The compressed catalog runs without its lite-07 reader, and +`--versions moq-lite-07-wip` refuses JS clients. Every publisher also sends an Opus tone, one sine per channel, and every subscriber decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and @@ -243,7 +244,9 @@ a varint rather than the draft's 64 bits, so `@moq/net` sends no proof from Firefox, and the tab must connect unproven. A build that cannot prove yet skips, or fails under `CI_STRICT`, which release PRs set, so the lite-07 cut waits for both browsers to prove. A build the table says cannot prove but that does fails, -so the table follows the browsers. +so the table follows the browsers. Until +[routes/js.md](../../quest/m2/cluster-routing/routes/js.md) ports `js/net` to the +route layer, the lane records a skip naming it. ## moq-transport diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 62d775e2d5f..520f3541633 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -2368,6 +2368,11 @@ run_matrix() { local engine status for engine in "${ROUTE_KEY_ENGINES[@]}"; do log="$HARNESS_RUN/route-key-$engine.log" + if [[ -n "$JS_LITE07_SKIP" ]]; then + echo " SKIP route key $engine" + record skip case="route key $engine" subscriber=js version=moq-lite-07-wip detail="$JS_LITE07_SKIP" + continue + fi if is_broken js; then echo " FAIL route key $engine (browser client unavailable)" record fail case="route key $engine" subscriber=js version=moq-lite-07-wip detail="browser client unavailable" From c1d875e4805c37f40f7fad5a12850d0c33bbf6a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 10:44:06 -0800 Subject: [PATCH 53/61] test(net): the lite-07 serve-loop helper names its node Main's SUBSCRIBE_SKIP test helper builds TrackServe without the node field the cluster-routing stack added. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/subscriber.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 4c5587fc625..6ef45e0c24a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -5270,6 +5270,7 @@ mod tests { subscriber, path: Path::from_static("room"), instance: None, + node: 0, name: "video".to_string(), }; let broadcast = crate::broadcast::Info::new().produce(); From 0b70a604ff3f0ee1e5f14940e4ead01decc15bba Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:31:45 -0800 Subject: [PATCH 54/61] fix(net): bound the lite-07 ROUTE_REQUEST backlog by identity A downstream peer raising its ROUTE_REQUEST seqno for a route through a stream whose peer leaves its replies unread queued one request per raise, and the session buffered every reply into the writer while its flush was pending, so relay memory grew without bound. The node now keeps each stream's requests per identity, the highest seqno subsuming lower ones, and the session takes them, and its ROUTE_END echoes, only once the writer has flushed. A stalled writer holds one request per identity the peer advertised, which the announce cap bounds. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher/routed.rs | 7 +- rs/moq-net/src/lite/subscriber/routed.rs | 96 ++++++++++++++++++++---- rs/moq-net/src/route/node.rs | 25 +++++- 3 files changed, 107 insertions(+), 21 deletions(-) diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index fe02bec9cb9..6ed94319d1e 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -273,11 +273,8 @@ impl Answer { match output { node::Output::Route { id, reach } => self.route(stream, source, id, reach), node::Output::Batch { id, changes, floor } => self.batch(stream, source, id, changes, floor), - // Only a stream this node asked for requests, serves, or echoes. - node::Output::Request { .. } - | node::Output::Echo { .. } - | node::Output::Serve { .. } - | node::Output::Unserve { .. } => Ok(()), + // Only a stream this node asked for serves or echoes. + node::Output::Echo { .. } | node::Output::Serve { .. } | node::Output::Unserve { .. } => Ok(()), } } diff --git a/rs/moq-net/src/lite/subscriber/routed.rs b/rs/moq-net/src/lite/subscriber/routed.rs index 31732dc7ddd..76f8d20358b 100644 --- a/rs/moq-net/src/lite/subscriber/routed.rs +++ b/rs/moq-net/src/lite/subscriber/routed.rs @@ -85,13 +85,13 @@ pub(super) struct Routed { /// The slots of routes the peer ended, kept until their ROUTE_END echo is flushed, /// so a peer leaving its replies unread holds its routes' slots. echoing: Vec, + /// The ROUTE_END echoes not yet written, each holding a slot in `echoing`. + ends: Vec, served: HashMap<(route::Id, Path), ServedClaim>, /// What the stream served before it filed again, while the new one takes it up. previous: HashMap<(route::Id, Path), ServedClaim>, /// Served claims whose request queue woke since the last pass. ready: kio::Queue<(route::Id, Path)>, - /// Messages for the peer, written by the stream's task. - replies: Vec, /// The peer sent a GOAWAY: every claim served here ranks last. drained: bool, } @@ -142,10 +142,10 @@ impl Routed { slots: subscriber.announces.clone(), held: Vec::new(), echoing: Vec::new(), + ends: Vec::new(), served: HashMap::new(), previous: HashMap::new(), ready: Default::default(), - replies: Vec::new(), drained: false, }) } @@ -190,7 +190,7 @@ impl Routed { } self.echoing.extend(self.held.pop()); self.stream.route(route.node, None); - self.replies.push(AnnounceReply::RouteEnd { id }); + self.ends.push(id); } Msg::Floor { id, revision } => { let certifies = self.certifies(); @@ -404,7 +404,7 @@ impl Routed { } /// Act on everything the node queued for the stream: serve the claims it hands - /// over, and queue the replies it asks for. + /// over. pub(super) fn outputs(&mut self, subscriber: &Subscriber) { while let Some(output) = self.stream.try_output() { self.output(subscriber, output); @@ -413,7 +413,6 @@ impl Routed { fn output(&mut self, subscriber: &Subscriber, output: node::Output) { match output { - node::Output::Request { id, seqno } => self.replies.push(AnnounceReply::RouteRequest { node: id, seqno }), node::Output::Serve { id, named, @@ -511,13 +510,29 @@ impl Routed { while let Poll::Ready(output) = self.stream.poll_output(waiter) { self.output(subscriber, output); } - for reply in self.replies.drain(..) { - writer.buffer(&reply)?; - } - match writer.poll_flush(waiter) { - Poll::Ready(Ok(())) => self.echoing.clear(), - Poll::Ready(Err(err)) => return Err(err), - Poll::Pending => {} + // Replies are buffered only once the last ones flushed, so a peer leaving them + // unread holds its echoes' slots and one request per identity, nothing more. + loop { + match writer.poll_flush(waiter) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Err(err), + Poll::Pending => break, + } + // The echoes buffered before went out: keep a slot per echo still queued. + self.echoing.truncate(self.ends.len()); + let requests = match self.stream.poll_requests(waiter) { + Poll::Ready(requests) => requests, + Poll::Pending => Default::default(), + }; + if self.ends.is_empty() && requests.is_empty() { + break; + } + for id in self.ends.drain(..) { + writer.buffer(&AnnounceReply::RouteEnd { id })?; + } + for (node, seqno) in requests { + writer.buffer(&AnnounceReply::RouteRequest { node, seqno })?; + } } while let Poll::Ready(Ok(key)) = self.ready.poll_pop(waiter) { let Some(entry) = self.served.get_mut(&key) else { @@ -741,6 +756,61 @@ mod tests { assert_eq!(subscriber.announces.available(), 2); } + /// A downstream peer raising its ROUTE_REQUEST seqno while this stream's peer + /// leaves its replies unread holds one request for the identity, sent once the + /// writer flushes and naming the highest seqno. + #[kio::test] + async fn a_stalled_writer_holds_one_request_per_identity() { + use crate::coding::Decode; + + let (subscriber, _consumer) = super::super::tests::restart_subscriber(SinkSession::new(Default::default())); + let prefix = Path::empty(); + let mut routed = super::Routed::new(&subscriber, &prefix, 1, &Default::default()).unwrap(); + let mut decoder = AnnounceDecoder::default(); + let budget = kio::Producer::new(0); + let log = crate::lite::test_transport::Log::default(); + let mut writer = crate::coding::Writer::new( + crate::lite::test_transport::SinkSend::stepped(log.clone(), budget.consume()), + crate::lite::Version::Lite07, + ); + let waiter = kio::Waiter::noop(); + let route = Msg::RouteStart { + node: 7, + advert: ADVERT, + }; + routed.receive(&prefix, route, &mut decoder).unwrap(); + let downstream = (subscriber.origin.route_node()) + .answer( + crate::route::Link { + proxy: true, + ..Default::default() + }, + Path::empty(), + ) + .unwrap(); + + for seqno in 1..=1000 { + downstream.request(7, seqno); + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + } + + *budget.write().ok().expect("the budget is open") = usize::MAX; + routed.poll(&subscriber, &mut writer, &waiter).unwrap(); + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let mut requests = Vec::new(); + while !buf.is_empty() { + let (reply, size) = crate::lite::AnnounceReply::decode_slice(buf, crate::lite::Version::Lite07).unwrap(); + buf = &buf[size..]; + requests.push(reply); + } + // The first went out before the writer stalled; the rest merged behind it. + assert_eq!( + requests, + [1, 1000].map(|seqno| crate::lite::AnnounceReply::RouteRequest { node: 7, seqno }) + ); + } + /// What the stream keeps between floors is the net change to its committed claims, /// however many messages the peer sends without one. #[kio::test] diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 66f62a45940..5cb25080859 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -74,8 +74,6 @@ pub(crate) enum Output { changes: Vec<(Path, Option)>, floor: Option, }, - /// Send ROUTE_REQUEST, naming the identity as the peer advertised it. - Request { id: Id, seqno: Seqno }, /// Echo the peer's ROUTE_END for the identity it named `id`. Echo { id: Id }, /// Serve `claim` of `id` into the origin through this stream, or update how it @@ -129,6 +127,9 @@ struct Outbox { serving: HashMap<(Id, Path), u64>, /// Each route the reader holds, and the claims it was sent under it. delivered: HashMap>, + /// ROUTE_REQUESTs to send, by the identity as the peer advertised it. A higher + /// seqno subsumes a lower one, so a writer that stalls holds one per identity. + requests: BTreeMap, } /// One identity's route changes and claim batches since the reader last took it. @@ -485,6 +486,17 @@ impl Stream { self.queue.lock().pop() } + /// The ROUTE_REQUESTs queued since the last call, each identity's highest seqno. + /// Take them only once the writer has room, so they keep coalescing meanwhile. + pub fn poll_requests(&self, waiter: &kio::Waiter) -> Poll> { + let ready = |outbox: &kio::Ref<'_, Outbox>| match outbox.requests.is_empty() { + true => Poll::Pending, + false => Poll::Ready(()), + }; + let mut outbox = ready!(self.queue.poll(waiter, ready)); + Poll::Ready(std::mem::take(&mut outbox.requests)) + } + fn with(&self, f: impl FnOnce(&mut Router, Key) -> R) -> R { let mut state = self.node.lock(); let out = f(&mut state.router, self.key); @@ -570,7 +582,14 @@ impl State { let (key, output) = match out { Out::Route { to, id, reach } => (to, Output::Route { id, reach }), Out::Batch { to, id, changes, floor } => (to, Output::Batch { id, changes, floor }), - Out::Request { to, id, seqno } => (to, Output::Request { id, seqno }), + Out::Request { to, id, seqno } => { + if let Some(queue) = self.queues.get(&to) { + let mut outbox = queue.lock(); + let want = outbox.requests.entry(id).or_insert(seqno); + *want = (*want).max(seqno); + } + continue; + } Out::Echo { to, id } => (to, Output::Echo { id }), }; if let Some(id) = push(&self.queues, key, output) { From 2a7e718e033d119b7693891c6878f2cbf60cad85 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 11:46:17 -0800 Subject: [PATCH 55/61] fix(net): a lite-07 answer gives route outputs a turn beside replies A peer sending ROUTE_REQUEST without pause kept the answer loop reading replies, so it never served the node's outputs and never yielded the task. Each pass now takes one reply and one output, and the loop yields on a cooperative budget like the publisher's other loops. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 66 +++++++++++++++++++++++++ rs/moq-net/src/lite/publisher/routed.rs | 14 ++++-- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5ef3e6c7b78..b4f12a640dc 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3671,6 +3671,72 @@ mod announce_test { assert_eq!(wire.take_announces(), [lite::AnnounceBroadcast::Synced]); } + /// A lite-07 peer that never stops sending ROUTE_REQUEST neither holds the task + /// nor keeps a new claim from being announced. + #[kio::test] + async fn a_lite07_reply_flood_yields_and_announces() { + /// A peer whose replies never run out. + struct Flood(bytes::Bytes); + impl crate::transport::poll::Chunks for Flood { + type Error = SinkError; + fn poll_read_chunk( + &mut self, + _: &kio::Waiter, + _: usize, + ) -> std::task::Poll, SinkError>> { + std::task::Poll::Ready(Ok(Some(self.0.clone()))) + } + fn stop(&mut self, _: u32) {} + } + + let origin = Hop::new(1).unwrap().produce(); + let consumer = origin.consume(); + let mut answer = + super::routed::Answer::new(consumer, crate::Path::empty(), true, crate::auth::Link::new(false)); + let request = lite::AnnounceReply::RouteRequest { node: 7, seqno: 1 }; + let flood = Flood(crate::coding::Encode::encode_bytes(&request, Version::Lite07).unwrap()); + let log = Log::default(); + let mut wire = Wire { + writes: log.writes.clone(), + cursor: 0, + version: Version::Lite07, + }; + let mut stream = Stream:: { + writer: Writer::new( + crate::transport::boxed::SendStream::new(SinkSend::new(log)), + Version::Lite07, + ), + reader: Reader::new( + crate::transport::boxed::RecvStream::new(crate::transport::poll::Buffered::new(flood)), + Version::Lite07, + ), + }; + let waiter = kio::Waiter::noop(); + assert!( + answer.poll(&mut stream, false, &waiter).is_pending(), + "the flood yields" + ); + wire.take_ok(); + let announces = wire.take_announces(); + assert_eq!( + announces.last(), + Some(&lite::AnnounceBroadcast::Synced), + "{announces:?}" + ); + + let _route = origin.announce("x", origin::RouteInfo::default()).unwrap(); + // The origin feeds its node from a task. + kio::clock::sim::yield_now().await; + assert!(answer.poll(&mut stream, false, &waiter).is_pending()); + let announces = wire.take_announces(); + assert!( + announces + .iter() + .any(|msg| matches!(msg, lite::AnnounceBroadcast::Active { .. })), + "{announces:?}" + ); + } + #[test] fn synced_survives_a_removed_initial_route_and_live_churn() { let origin = Hop::new(1).unwrap().produce(); diff --git a/rs/moq-net/src/lite/publisher/routed.rs b/rs/moq-net/src/lite/publisher/routed.rs index 6ed94319d1e..fb4310ec671 100644 --- a/rs/moq-net/src/lite/publisher/routed.rs +++ b/rs/moq-net/src/lite/publisher/routed.rs @@ -96,6 +96,8 @@ pub(super) struct Answer { epoch: u64, permit: crate::auth::Permit, phase: Phase, + /// Passes before the stream yields, so a peer flooding replies cannot hold the task. + budget: kio::coop::Budget, } impl Answer { @@ -126,6 +128,7 @@ impl Answer { epoch: 0, permit: Default::default(), phase: Phase::Init, + budget: kio::coop::Budget::new(32), } } @@ -180,6 +183,7 @@ impl Answer { } loop { + ready!(self.budget.poll_yield(waiter)); // Deliver what is buffered before taking more work. ready!(stream.writer.poll_flush(waiter))?; @@ -214,10 +218,12 @@ impl Answer { continue; } + // One reply and one output per pass, so neither direction starves the other. + let mut progress = false; match stream.reader.poll_decode_maybe::(waiter) { Poll::Ready(Ok(Some(reply))) => { self.reply(reply)?; - continue; + progress = true; } // The peer closed its side: it wants nothing more. Poll::Ready(Ok(None)) => return Poll::Ready(Ok(())), @@ -235,9 +241,11 @@ impl Answer { if let Some((index, output)) = ready { self.turn = index + 1; self.output(stream, index, output)?; - continue; + progress = true; + } + if !progress { + return Poll::Pending; } - return Poll::Pending; } } From 5d6e7fd9ac50ec597f18c64bacb5216f7fa8f947 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 15:46:52 -0800 Subject: [PATCH 56/61] fix(net): serve a job before its claim's restart A job continuing its claim's instance and the claim's restart arriving together were served claim first, so on a multithreaded relay a front resolved through the old instance could find no route between the two and end while the job stayed live. Serve claims most specific first, and send a restart behind every queued serve rather than in a queued serve's place. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/route/node.rs | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/rs/moq-net/src/route/node.rs b/rs/moq-net/src/route/node.rs index 4c3d01b759c..ff66f4e7f02 100644 --- a/rs/moq-net/src/route/node.rs +++ b/rs/moq-net/src/route/node.rs @@ -114,10 +114,11 @@ struct Outbox { /// Outputs to read in order: the other kinds, and a pending entry once taken. ready: VecDeque, /// Each claim's last Serve or Unserve, read after the rest, in queue order. A job - /// queues its Serve before the Unserve of the claim it continues, and the session - /// needs the job's route in place before the claim's goes, so a reader resolved - /// through the claim carries on: so a Serve replacing a queued Serve keeps its - /// place, while an Unserve, or a Serve after one, goes last. + /// queues its Serve before the Unserve or restart of the claim it continues, and the + /// session needs the job's route in place before the claim's goes, so a reader + /// resolved through the claim carries on: so a Serve replacing a queued Serve of the + /// same instance and epoch keeps its place, while an Unserve, a restart, or a Serve + /// after an Unserve goes last. serves: BTreeMap, /// Where each claim's last change sits in `serves`. serving: HashMap<(Id, Path), u64>, @@ -218,8 +219,10 @@ impl Outbox { Output::Serve { id, ref claim, .. } | Output::Unserve { id, ref claim } => { let key = (id, claim.clone()); if let Some(&seq) = self.serving.get(&key) - && matches!(output, Output::Serve { .. }) - && matches!(self.serves.get(&seq), Some(Output::Serve { .. })) + && let Output::Serve { served, .. } = &output + && let Some(Output::Serve { served: queued, .. }) = self.serves.get(&seq) + && queued.claim.instance == served.claim.instance + && queued.claim.epoch == served.claim.epoch { self.serves.insert(seq, output); return None; @@ -614,7 +617,9 @@ impl State { .take_while(|((home, _), _)| *home == id) .map(|((_, claim), _)| claim.clone()) .collect(); - for (claim, item, available) in claims { + // Most specific first: a job continuing its claim's instance is served before + // the claim moves on, so a reader resolved through the claim finds the job's route. + for (claim, item, available) in claims.into_iter().rev() { gone.remove(&claim); let routes = self.router.routes(id, &claim); let home = self.homes.entry((id, claim.clone())).or_default(); @@ -886,6 +891,45 @@ mod tests { assert_eq!(read, [(p("live/x"), Some(5)), (p("live"), None)]); } + /// A job and its claim's restart arriving together are read job first, even past a + /// change to the claim still queued, so a reader resolved through the claim's old + /// instance finds the job's route before the claim's moves on. + #[test] + fn a_job_is_read_before_its_claims_restart() { + let (old, new) = (Instance::random(), Instance::random()); + let origin = Node::new(1, 1); + let relay = Node::new(2, 2); + let fast = relay.receive(Link::default(), Path::empty()).unwrap(); + let stalled = relay.receive(Link::default(), Path::empty()).unwrap(); + let fast_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + let stalled_answer = origin.answer(Link::default(), Path::empty()).unwrap(); + origin.own(p("live"), Some(claim(&old, 0))); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + outputs(&fast); + outputs(&stalled); + + // A change to the claim the stalled stream leaves queued. + origin.own(p("live"), Some(claim(&old, 1))); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + outputs(&fast); + + origin.own(p("live/x"), Some(claim(&old, 0))); + origin.own(p("live"), Some(claim(&new, 0))); + pump(&fast_answer, &fast); + pump(&stalled_answer, &stalled); + for stream in [&fast, &stalled] { + let read: Vec<(Path, Instance)> = (served(outputs(stream)).into_iter()) + .map(|output| match output { + Output::Serve { claim, served, .. } => (claim, served.claim.instance), + other => panic!("not a serve: {other:?}"), + }) + .collect(); + assert_eq!(read, [(p("live/x"), old.clone()), (p("live"), new.clone())]); + } + } + /// A claim of a scoped peer left with no usable route stays where it is served, /// under the name that peer gave its identity, so the requests it serves meanwhile /// still name it as the peer does. From baecc27d4b1476953c2421867c2eb3e35e237260 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 16:57:58 -0800 Subject: [PATCH 57/61] quest: drop route-roles' requirement on the finished announce-takeover quest Co-Authored-By: Claude Opus 5.5 --- quest/m2/route-roles.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/quest/m2/route-roles.md b/quest/m2/route-roles.md index b856f5b4562..a8c67fa0e86 100644 --- a/quest/m2/route-roles.md +++ b/quest/m2/route-roles.md @@ -37,7 +37,3 @@ put the comparison in the PR under `## Waiting` with a recommendation. After the maintainer picks, land the winner in JS, moq-ffi, moq-c, and the wrappers in the same PR. The route instance accessor's five spellings across bindings stay with [the exporter](/quest/m3/ffi-exporter.md). - -## Required - -- [A job's exact route continues its claim's readers](/quest/m2/announce-takeover.md) - changes how `Request::route` lends a claim's instance, which both shapes must keep From 9e40dbae5183f5568437415ee8e4aaf2c0fee4a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 17:40:12 -0800 Subject: [PATCH 58/61] fix(net): reading a hidden job's instance binds it A signer built from `route.instance()` before a hidden job announces named the claim's instance at that moment. If another job under the claim ended before the announce, the job announced under the claim's new instance and subscribers rejected every signed frame. Reading an unannounced job's instance now binds it. `announce()` then refuses with Unroutable once the claim has moved past it, and its requesters ask the claim again: a front treats a refusal for an instance its route has moved off as not standing, and JS requeues the request. Jobs that never read their instance still bind at announce. moq-c and moq-ffi no longer read the instance at creation, which would have bound every job. Co-Authored-By: Claude Opus 5.5 --- doc/concept/identity.md | 6 ++- go/wrapper/origin.go | 8 +-- js/net/src/origin-request.test.ts | 47 +++++++++++++++-- js/net/src/origin.ts | 42 +++++++++++---- py/moq-rs/README.md | 2 +- py/moq-rs/moq/origin.py | 8 +-- rs/moq-c/src/api.rs | 8 +-- rs/moq-c/src/origin.rs | 3 +- rs/moq-ffi/src/origin.rs | 27 +++++++--- rs/moq-net/src/model/origin.rs | 86 +++++++++++++++++++++++++------ swift/Sources/Moq/Origin.swift | 8 +-- 11 files changed, 195 insertions(+), 50 deletions(-) diff --git a/doc/concept/identity.md b/doc/concept/identity.md index d86bab5897a..49df5551284 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -195,7 +195,11 @@ while let Some(request) = claim.requested().await { JavaScript mirrors this with `request.route({ hidden: true })` and `job.announce()`; see [`@moq/net`](/lib/js/net) and [`moq-net`](/lib/rs/moq-net). A hidden job route is unannounced until it announces, and dropping it before -then refuses the request. Ending a job's route withdraws its path, and the +then refuses the request. It takes the claim's instance when it announces, or +earlier when its instance is first read, as a signer built from it does. Read +first, the job announces under it or not at all: if another job under the claim +ends in between, moving the claim to a fresh instance, `announce` fails with +`Unroutable` and the request reaches the worker again. Ending a job's route withdraws its path, and the prefix route keeps answering later requests. A lazy job's lifecycle, as a viewer of `transcode/alice` sees it through a relay: diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index 9c0c5d42bef..55021fc10a0 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -97,8 +97,10 @@ func (r *Route) Broadcast() (*BroadcastProducer, error) { // Instance returns the stream this route serves now. A named instance never // changes. Unnamed, a prefix route moves to a fresh one each time a job holding -// its current one ends, and a job's route reports its prefix route's current one -// until it first announces. +// its current one ends. A job's route takes its prefix route's current one when +// it first announces or when this first reads it. Read first, the job announces +// under it, and [Route.Announce] fails with Unroutable once the prefix route has +// moved past it, so a signer built from it stays valid. func (r *Route) Instance() string { return r.inner.InstanceId() } @@ -165,7 +167,7 @@ func (r *BroadcastRequest) Path() (string, error) { // Route answers with the path's own exact route, owning a new broadcast. // // Unless options names an instance, it takes the one its prefix route lends -// when it first announces, so readers that resolved the path through the prefix +// when it first announces or [Route.Instance] first reads it, so readers that resolved the path through the prefix // route continue on it; its end then moves the prefix route to a fresh instance, // so a restarted job is a new stream. A prefix route that names its instance // lends none, and the route mints its own. Created visible, it announces now and answers diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index 39195a58d71..ae7c9f5d541 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -1011,8 +1011,8 @@ test("a restarted job gets a fresh instance and its old one is refused", async ( origin.close(); }); -// A job binds its claim's instance when it announces, so one still hidden when an unrelated job -// ends keeps its request and answers it under the claim's new instance. +// A job that never read its instance binds its claim's when it announces, so one still hidden +// when an unrelated job ends keeps its request and answers it under the claim's new instance. test("a hidden job announces under the instance its claim moved to", async () => { const origin = new Producer(); const claim = origin.routePrefix(Path.from("room")); @@ -1026,13 +1026,11 @@ test("a hidden job announces under the instance its claim moved to", async () => const second = (await requests.next()).value; if (!second) throw new Error("expected a request"); const hidden = second.route({ hidden: true }); - expect(hidden.instance).toBe(minted); ended.close(); await settle(); const moved = claim.instance; expect(moved).not.toBe(minted); - expect(hidden.instance).toBe(moved); expect(bob.closed.peek()).toBeUndefined(); hidden.announce(); await settle(); @@ -1044,6 +1042,47 @@ test("a hidden job announces under the instance its claim moved to", async () => origin.close(); }); +// A hidden job that read its instance, as a signer does, announces under it or not at all: once an +// unrelated job's end moves the claim past it, the announce is refused and its request is asked +// again, so the worker prepares the job under the claim's new instance. +test("a job that read its instance is refused once its claim moves", async () => { + const origin = new Producer(); + const claim = origin.routePrefix(Path.from("room")); + const minted = claim.instance; + const requests = claim.requested(); + const alice = origin.request(Path.from("room/alice")); + const first = (await requests.next()).value; + if (!first) throw new Error("expected a request"); + const ended = first.route(); + const bob = origin.request(Path.from("room/bob")); + const second = (await requests.next()).value; + if (!second) throw new Error("expected a request"); + const hidden = second.route({ hidden: true }); + expect(hidden.instance).toBe(minted); + + ended.close(); + await settle(); + const moved = claim.instance; + expect(moved).not.toBe(minted); + expect(hidden.instance).toBe(minted); + expect(() => hidden.announce()).toThrow(expect.objectContaining({ code: StreamCode.Unroutable })); + await settle(); + expect(bob.closed.peek()).toBeUndefined(); + + const third = (await requests.next()).value; + if (!third) throw new Error("expected the request again"); + expect(third.path).toBe(Path.from("room/bob")); + const again = third.route({ hidden: true }); + expect(again.instance).toBe(moved); + again.announce(); + await settle(); + expect(bob.active.peek()?.instance).toBe(moved); + for (const request of [alice, bob]) request.close(); + again.close(); + claim.close(); + origin.close(); +}); + // Each live job keeps the instance it bound; only one carrying the claim's current instance moves // the claim when it ends. test("one job ending leaves another job's instance alone", async () => { diff --git a/js/net/src/origin.ts b/js/net/src/origin.ts index 3d00fea0a2a..dfdfba82a1a 100644 --- a/js/net/src/origin.ts +++ b/js/net/src/origin.ts @@ -487,7 +487,7 @@ class ServeState { * job for the path is a new stream and nothing answers the ended one's instance again. */ interface Handoff { - /** The instance and epoch the claim serves now, which a job binds when it first announces. */ + /** The instance and epoch the claim serves now, which a job binds when it first announces or reads its instance. */ current(): { instance: Instance.Valid; epoch: bigint }; /** A job serving `instance` stopped being advertised; the claim moves on if it still serves it. */ advance(instance: Instance.Valid): void; @@ -1459,8 +1459,9 @@ export class Producer implements Table { auth: options.auth, }); - // A job naming no instance continues its claim's, bound when it first announces rather - // than now, so a job still being prepared is never invalidated by the claim moving on. + // A job naming no instance continues its claim's, bound when it first announces or when its + // instance is read, whichever comes first. One that never read it is not invalidated by the + // claim moving on while it is prepared; one that read it announces under it or not at all. const claim = request && options.instance === undefined ? requestHandoff(request) : undefined; let bound = false; @@ -1511,7 +1512,14 @@ export class Producer implements Table { // A request released meanwhile can never be answered, so this route never announces. if (!jobPending(job)) throw unroutable(); if (claim) { - info = { ...info, ...claim.current() }; + const current = claim.current(); + // A job that read its instance announces under it or not at all. Its requesters ask + // the claim again, so the worker prepares the job under the claim's new instance. + if (bound && current.instance !== info.instance) { + requeueJob(job); + throw unroutable(); + } + info = { ...info, ...current }; bound = true; } this.#advertiseExact(path, front, info); @@ -1531,7 +1539,13 @@ export class Producer implements Table { const route = makeRoute({ broadcast: producer, - instance: () => (claim && !bound ? claim.current().instance : (info.instance as Instance.Valid)), + instance: () => { + if (claim && !bound && pending) { + info = { ...info, ...claim.current() }; + bound = true; + } + return info.instance as Instance.Valid; + }, announce, withdraw: retract, announced, @@ -1946,6 +1960,7 @@ let setRequestPublisher: (request: Request, publisher: Producer | undefined) => let jobPending: (request: Request) => boolean; let answerJob: (request: Request) => void; let rejectJob: (request: Request, err: Error) => void; +let requeueJob: (request: Request) => void; let requestHandoff: (request: Request) => Handoff | undefined; /** @@ -2796,8 +2811,10 @@ export class Route { * A prefix route that minted its instance moves to a fresh one, as a restart, the moment a * job's route carrying its instance stops being advertised. A job's route * ({@link Request.route}) naming no instance takes its prefix route's current one when it - * first announces, and reports that until then; a prefix route naming its instance gives each - * job a fresh one instead. + * first announces or when this first reads it. Read first, the job announces under it, and + * {@link announce} throws {@link StreamCode | StreamCode.Unroutable} once the prefix route has + * moved past it, so a signer built from it stays valid. A prefix route naming its instance + * gives each job a fresh one instead. */ get instance(): Instance.Valid { return this.#control.instance(); @@ -2809,7 +2826,9 @@ export class Route { * Visible to local consumers before it returns. A job's route ({@link Request.route}) also * answers the requesters waiting on its path. Throws once the route or its broadcast closed, * and, announcing nothing, with {@link StreamCode | StreamCode.Unroutable} when a job's request - * was released or its prefix route retracted meanwhile: that route can never announce. + * was released or its prefix route retracted meanwhile, or when the job read its + * {@link instance} and its prefix route has moved past it since: that route can never + * announce. In that last case its requesters ask the prefix route again. */ announce(): void { this.#control.announce(); @@ -2905,6 +2924,11 @@ export class Request { request.#server.settled.update((n) => n + 1); }; rejectJob = (request, err) => request.#server.reject(request, err); + requeueJob = (request) => { + const path = Path.join(request.#server.root, request.path); + request.#server.forget(path, request); + request.#server.enqueue(path); + }; requestHandoff = (request) => request.#server.handoff; finishRequest = (request, err) => { request.#done = true; @@ -2917,7 +2941,7 @@ export class Request { * Answer with the path's own exact route, owning a new broadcast. * * Unless `options` names an instance, the route continues the prefix route's minted - * instance, bound when it first announces, so readers that resolved the path through the + * instance, bound when it first announces or {@link Route.instance} first reads it, so readers that resolved the path through the * prefix route keep reading the same stream. Once this route stops being advertised the * prefix route moves to a fresh instance, so a job that restarts beneath it is a new stream * for its restarted sequences, and a request naming the ended instance is refused. Under a diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index b0b249a19dc..40e0152cb4b 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -288,7 +288,7 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect - `.route_prefix(prefix, *, hidden=False, cost=0, instance=None, epoch=0) → Route`. A prefix route that yields the requests beneath it. - **`Route`**. A route on an origin. Hold it: dropping or cancelling it retracts it, and so does closing its broadcast. - `.broadcast → BroadcastProducer`. The broadcast an exact route owns; raises `Internal` on a prefix route. - - `.instance`. The stream it serves now, kept across `withdraw()` and `announce()`. Unless one is named, a prefix route moves to a fresh one each time a job holding it ends, and a job reports its prefix route's until it first announces. + - `.instance`. The stream it serves now, kept across `withdraw()` and `announce()`. Unless one is named, a prefix route moves to a fresh one each time a job holding it ends, and a job takes its prefix route's when it first announces or when `.instance` first reads it. Read first, the job announces under it, and `announce()` fails with `Unroutable` once the prefix route has moved past it. - `.announce()`, `.withdraw()`, `.set_cost(cost)`. Show, hide, or re-price it in place; `announce` and `set_cost` raise `Closed` once the broadcast closed. - `await .requested() → BroadcastRequest | None`, or async iteration over a prefix route. Call `.serve(consumer)` to answer with held content under the prefix route's continuous sequence space, `.route(hidden=..., ...)` for the path's own route, which continues the prefix route's instance until the job ends and then moves the prefix route to a fresh one (unless either names an instance: a named prefix route lends none, and a named job keeps its own), or `.reject(code)` to fail the requester. - **`OriginConsumer`**. Discover broadcasts. diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index a1e0faa348b..e99774ac0c5 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -120,7 +120,7 @@ def route(self, *, hidden: bool = False, cost: int = 0, instance: str | None = N """Answer with the path's own exact :class:`Route`, owning a new broadcast. Unless ``instance`` names one, the route takes the instance its prefix route lends - when it first announces, so readers that resolved the path through the prefix route + when it first announces or :attr:`Route.instance` first reads it, so readers that resolved the path through the prefix route continue on it; its end then moves the prefix route to a fresh instance, so a job that restarts is a new stream. A prefix route that names its instance lends none, and the route mints its own. Created visible, it announces now and answers the waiting @@ -169,8 +169,10 @@ def instance(self) -> str: """The stream this route serves now. A named instance never changes. Unnamed, a prefix route moves to a fresh one each time - a job holding its current one ends, and a job's route reports its prefix route's - current one until it first announces. Withdrawing and announcing keep it.""" + a job holding its current one ends. A job's route takes its prefix route's current one + when it first announces or when this first reads it. Read first, the job announces + under it, and :meth:`announce` fails with ``Unroutable`` once the prefix route has moved + past it, so a signer built from it stays valid. Withdrawing and announcing keep it.""" return self._inner.instance_id() def announce(self) -> None: diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 5901249e680..6b6c35a8cb0 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -2124,8 +2124,10 @@ pub extern "C" fn moq_route_broadcast(route: u32) -> i32 { /// The stream a route serves now. /// /// Named options keep one instance. Unnamed, a prefix route moves to a fresh one each time a -/// job holding its current one ends, and a route from [moq_broadcast_request_route] reports -/// its prefix route's current one until it first announces, binding whichever that is then. +/// job holding its current one ends. A route from [moq_broadcast_request_route] takes its +/// prefix route's current one when it first announces or when this first reads it. Read first, +/// the job announces under it, and [moq_route_announce] fails with an unroutable error once the +/// prefix route has moved past it, so a signer built from it stays valid. /// Withdrawing and announcing again keep it. The destination borrows the route's storage /// until the next [moq_route_instance] on the route or [moq_route_free]. /// @@ -2249,7 +2251,7 @@ pub extern "C" fn moq_broadcast_request_serve(request: u32, broadcast: u32) -> i /// /// Consumes the request handle and returns a positive route handle, or a negative error. /// Unless `options` names an instance, the route takes the one its prefix route lends when -/// it first announces, so readers that resolved the path through the prefix route continue +/// it first announces or [moq_route_instance] first reads it, so readers that resolved the path through the prefix route continue /// on it; its end then moves the prefix route to a fresh instance, so a job that restarts is /// a new stream. A prefix route that names its instance lends none, and the route mints its /// own. Created visible, it announces now and answers the waiting requesters; diff --git a/rs/moq-c/src/origin.rs b/rs/moq-c/src/origin.rs index b0482e651cc..4555090edd6 100644 --- a/rs/moq-c/src/origin.rs +++ b/rs/moq-c/src/origin.rs @@ -409,8 +409,9 @@ impl Origin { driver: crate::ffi::Background, broadcast: Id, ) -> Result { + // Read only on request: reading a hidden job's instance binds it. self.routes.insert(RouteEntry { - instance: route.instance().to_string(), + instance: String::new(), inner: route, driver, kind: RouteKind::Exact { broadcast }, diff --git a/rs/moq-ffi/src/origin.rs b/rs/moq-ffi/src/origin.rs index 28c5d7579cf..d1b252fc084 100644 --- a/rs/moq-ffi/src/origin.rs +++ b/rs/moq-ffi/src/origin.rs @@ -440,8 +440,15 @@ impl MoqRoute { } fn new(route: moq_net::origin::Route, broadcast: Option>, driver: Background) -> Self { + // Reading a hidden job's instance binds it, so one is cached only once it announces or + // is read. + let instance = if route.is_announced() || broadcast.is_none() { + route.instance().to_string() + } else { + String::new() + }; Self { - instance: std::sync::Mutex::new(route.instance().to_string()), + instance: std::sync::Mutex::new(instance), reads: Polled::new(route).supervised(driver.clone()), broadcast, driver, @@ -469,11 +476,14 @@ impl MoqRoute { }) } - /// The stream this route serves now, or the last one once cancelled. + /// The stream this route serves now, or once cancelled, the last one it reported or + /// announced under (empty for a hidden route that did neither). /// /// Named options keep one instance. Unnamed, a prefix route moves to a fresh one each - /// time a job holding its current one ends, and a job's route reports its prefix route's - /// current one until it first announces, binding whichever that is then. + /// time a job holding its current one ends. A job's route takes its prefix route's current + /// one when it first announces or when this first reads it. Read first, the job announces + /// under it, and [`MoqRoute::announce`] fails with `Unroutable` once the prefix route has + /// moved past it, so a signer built from it stays valid. /// /// Named `instance_id` rather than `instance` because the generated C and C++ /// objects keep their handle in a private field named `instance`; the wrappers @@ -493,7 +503,12 @@ impl MoqRoute { /// `Unroutable` when a request's route can never announce because the request was /// released or its prefix route retracted meanwhile. pub fn announce(&self) -> Result<(), MoqError> { - self.with(|route| route.announce()) + let instance = self.with(|route| { + route.announce()?; + Ok(route.instance().to_string()) + })?; + *self.instance.lock().unwrap() = instance; + Ok(()) } /// Hide the route from everyone, keeping its broadcast and instance: `announce` brings the @@ -562,7 +577,7 @@ impl MoqBroadcastRequest { /// Answer with the path's own exact route, owning a new broadcast. /// /// Unless `options` names an instance, the route takes the one its prefix route lends - /// when it first announces, so readers that resolved the path through the prefix route + /// when it first announces or [`MoqRoute::instance_id`] first reads it, so readers that resolved the path through the prefix route /// continue on it; its end then moves the prefix route to a fresh instance, so a job that /// restarts is a new stream. A prefix route that names its instance lends none, and the /// route mints its own. Created visible, it announces now and answers the waiting diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index cbddf1fd3ef..2b1ea7f5e46 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2943,6 +2943,10 @@ struct RouteControl { /// moves on as its jobs end (see [`OriginState::advance`]), or a job's route taking one /// from it. An update then names none, leaving the entry's as it is. held: bool, + /// Whether a job's route read its instance before it first announced: it then announces + /// under that one or not at all, so whatever was built from it, such as a signer, stays + /// valid. + bound: bool, /// The request a job's route answers when it first announces. request: Option, } @@ -2957,6 +2961,7 @@ impl RouteControl { options, instance, held, + bound: false, request, } } @@ -3008,17 +3013,20 @@ impl Route { /// The stream this route serves, minted when it was created unless /// [`RouteOptions::with_instance`] named one. /// - /// Unnamed, it can change: a prefix route ([`Producer::route_prefix`]) moves to a fresh - /// instance each time an answer holding its current one ends, and a job's route - /// ([`Request::route`]) reports its prefix route's current one until it first announces, - /// binding whichever that is then. + /// Unnamed, a prefix route's ([`Producer::route_prefix`]) can change: it moves to a fresh + /// instance each time an answer holding its current one ends. A job's route + /// ([`Request::route`]) takes its prefix route's current one when it first announces, + /// or when this reads it first. Read before announcing, it is the one the job announces + /// under, and [`Self::announce`] fails with [`Error::Unroutable`] once the prefix route + /// has moved past it. pub fn instance(&self) -> Instance { let mut control = self.state.control.lock(); - let lender = control.request.as_ref().map(|request| &request.home); - if control.held - && let Some(current) = self.state.announcement.instance(lender) - { - control.instance = current; + if control.held && !control.bound { + let lender = control.request.as_ref().map(|request| &request.home); + if let Some(current) = self.state.announcement.instance(lender) { + control.instance = current; + } + control.bound = control.request.is_some(); } control.instance.0.clone() } @@ -3030,14 +3038,16 @@ impl Route { /// /// Fails with [`Error::Closed`] once the route's broadcast closed or the origin's /// [`Driver`] was dropped, and with [`Error::Unroutable`], announcing nothing, when a - /// job's request was released or its prefix route retracted meanwhile: that route can - /// never announce. + /// job's request was released or its prefix route retracted meanwhile, or when the job + /// read its [`instance`](Self::instance) and its prefix route has moved past it since: + /// that route can never announce. Its requesters ask the prefix route again. pub fn announce(&self) -> Result<(), Error> { let mut control = self.state.control.lock(); let info = control.info(); let Some(request) = control.request.take() else { return self.state.announcement.update_if(info, None, |_| true).map(|_| ()); }; + let bound = std::mem::take(&mut control.bound).then(|| control.instance.0.clone()); let answer = self.broadcast().consume(); let mut resolved = None; // Answered under the table lock that every release takes first, so the route @@ -3058,6 +3068,15 @@ impl Route { if !home { return false; } + // A job that read its instance announces under it or not at all. A prefix route + // that lends nothing never moves, and one no longer serving released the request. + if let Some(bound) = &bound + && table + .lender(&request.path, &request.home) + .is_some_and(|lent| lent.instance != *bound) + { + return false; + } // The exact route owns this answer's availability. Caching it under the // prefix would keep serving it after this route is withdrawn. resolved = request @@ -4859,6 +4878,13 @@ impl kio::Task for FrontServe { standing: false, }), }, + // A refusal for an instance the route has since moved off, such as a + // job's announce refused because it read its claim's old instance, is + // not the answer for the new one: ask again. + Err(err) if self.stale(route, &asked) => Event::Resolved { + route, + result: Err(Refusal { err, standing: false }), + }, Ok(source) => { let id = self.next_source; self.next_source += 1; @@ -6485,7 +6511,7 @@ impl Request { /// Answer with the path's own exact route, owning a new broadcast. /// /// Unless `options` names an instance, the route takes the one its prefix route lends - /// when it first announces, so whatever resolved the path through the prefix route + /// when it first announces or [`Route::instance`] first reads it, so whatever resolved the path through the prefix route /// continues on this answer; a prefix route that lends nothing has the route mint its /// own. Ending the route withdraws only this path, and moves a prefix route that lent /// its instance to a fresh one, so a job that ends and restarts beneath the same prefix @@ -12236,7 +12262,8 @@ mod tests { } /// A job's route takes the instance its claim lends when it announces, not when it was - /// created: a hidden job outlives a sibling's end and joins the claim's next stream. + /// created: a hidden job that never read its instance outlives a sibling's end and joins + /// the claim's next stream. #[kio::test] #[cfg(feature = "async")] async fn a_hidden_job_takes_the_claims_instance_when_it_announces() { @@ -12251,19 +12278,46 @@ mod tests { bob.await.unwrap(); let first = claim.instance(); assert_eq!(sibling.instance(), first); - assert_eq!(prepared.instance(), first); drop(sibling); let second = claim.instance(); assert_ne!(second, first); - // Unannounced, it reports the instance it would bind now. - assert_eq!(prepared.instance(), second); prepared.announce().unwrap(); assert_eq!(prepared.instance(), second); assert_eq!(alice.await.unwrap().info().instance, Some(second)); } + /// A hidden job that read its instance, as a signer does, announces under it or not at + /// all: once a sibling's end moves the claim past it, the announce is refused and its + /// requesters ask the claim again, so the worker prepares the job under the new instance. + #[kio::test] + #[cfg(feature = "async")] + async fn a_job_that_read_its_instance_is_refused_once_the_claim_moves() { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let claim = producer.route_prefix("room", RouteOptions::default()).unwrap(); + + let alice = consumer.request("room/alice", None); + let prepared = queued(&claim).await.route(RouteOptions::hidden()).unwrap(); + let bob = consumer.request("room/bob", None); + let sibling = queued(&claim).await.route(RouteOptions::default()).unwrap(); + bob.await.unwrap(); + let first = prepared.instance(); + assert_eq!(first, claim.instance()); + + drop(sibling); + let second = claim.instance(); + assert_ne!(second, first); + assert_eq!(prepared.instance(), first, "a read instance moved"); + assert!(matches!(prepared.announce(), Err(Error::Unroutable))); + + let again = queued(&claim).await.route(RouteOptions::hidden()).unwrap(); + assert_eq!(again.instance(), second); + again.announce().unwrap(); + assert_eq!(alice.await.unwrap().info().instance, Some(second)); + } + /// A request pinned to the claim's instance resolves through its job's route even when a /// sibling job ending moved the claim on before the request looked again. #[kio::test] diff --git a/swift/Sources/Moq/Origin.swift b/swift/Sources/Moq/Origin.swift index 0c7683bfbde..44a196c274d 100644 --- a/swift/Sources/Moq/Origin.swift +++ b/swift/Sources/Moq/Origin.swift @@ -69,7 +69,7 @@ public final class BroadcastRequest: Sendable { /// Answer with the path's own exact route, owning a new broadcast. /// /// Unless `instance` names one, it takes the instance its prefix route lends when it first - /// announces, so readers that resolved the path through the prefix route continue on it; + /// announces or `Route.instance` first reads it, so readers that resolved the path through the prefix route continue on it; /// its end then moves the prefix route to a fresh instance, so a restarted job is a new /// stream. A prefix route that names its instance lends none, and the route mints its own. /// Created visible, it announces now and answers the waiting requesters; created `hidden`, they wait until @@ -121,8 +121,10 @@ public final class Route: AsyncSequence, Sendable { } /// The stream this route serves now. A named instance never changes. Unnamed, a prefix - /// route moves to a fresh one each time a job holding its current one ends, and a job's - /// route reports its prefix route's current one until it first announces. + /// route moves to a fresh one each time a job holding its current one ends. A job's route + /// takes its prefix route's current one when it first announces or when this first reads + /// it. Read first, the job announces under it, and `announce()` throws `Unroutable` once + /// the prefix route has moved past it, so a signer built from it stays valid. public var instance: String { ffi.instanceId() } /// Make the route visible, or keep it so; throws `Closed` once its broadcast closed. From 651b2d5e27dd2f0c464ea49acc59af8d6a25c843 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 17:53:03 -0800 Subject: [PATCH 59/61] fix(net): wrap the Python route docstring and regenerate Dart bindings Co-Authored-By: Claude Opus 5.5 --- dart/moq_ffi/lib/src/moq.dart | 4 ++-- py/moq-rs/moq/origin.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index daf38c9d6ee..ec2dcb307ca 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -24500,7 +24500,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_reject() != 9727) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 53132) { + if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_route() != 8265) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastrequest_serve() != 10832) { @@ -24539,7 +24539,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqroute_cancel() != 41411) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqroute_instance_id() != 6842) { + if (uniffi_moq_ffi_checksum_method_moqroute_instance_id() != 57871) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqroute_requested() != 49558) { diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index e99774ac0c5..b6f1c3d005b 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -120,9 +120,9 @@ def route(self, *, hidden: bool = False, cost: int = 0, instance: str | None = N """Answer with the path's own exact :class:`Route`, owning a new broadcast. Unless ``instance`` names one, the route takes the instance its prefix route lends - when it first announces or :attr:`Route.instance` first reads it, so readers that resolved the path through the prefix route - continue on it; its end then moves the prefix route to a fresh instance, so a job that - restarts is a new stream. A prefix route that names its instance lends none, and the + when it first announces or :attr:`Route.instance` first reads it, so readers that + resolved the path through the prefix route continue on it; its end then moves the + prefix route to a fresh instance, so a job that restarts is a new stream. A prefix route that names its instance lends none, and the route mints its own. Created visible, it announces now and answers the waiting requesters; created ``hidden``, they wait until :meth:`Route.announce`, so the job can fill its catalog first, and cancelling the route before then refuses them. Ending the From b7e7479720f87dd5a530dc490451c33588898927 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 18:04:26 -0800 Subject: [PATCH 60/61] fix(py): rewrap the request route docstring Co-Authored-By: Claude Opus 5.5 --- py/moq-rs/moq/origin.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/py/moq-rs/moq/origin.py b/py/moq-rs/moq/origin.py index b6f1c3d005b..1128922623f 100644 --- a/py/moq-rs/moq/origin.py +++ b/py/moq-rs/moq/origin.py @@ -119,14 +119,14 @@ def path(self) -> str: def route(self, *, hidden: bool = False, cost: int = 0, instance: str | None = None, epoch: int = 0) -> Route: """Answer with the path's own exact :class:`Route`, owning a new broadcast. - Unless ``instance`` names one, the route takes the instance its prefix route lends - when it first announces or :attr:`Route.instance` first reads it, so readers that - resolved the path through the prefix route continue on it; its end then moves the - prefix route to a fresh instance, so a job that restarts is a new stream. A prefix route that names its instance lends none, and the - route mints its own. Created visible, it announces now and answers the waiting - requesters; created ``hidden``, they wait until :meth:`Route.announce`, so the job can - fill its catalog first, and cancelling the route before then refuses them. Ending the - route withdraws only this path. + Unless ``instance`` names one, the route takes the instance its prefix route lends when + it first announces or :attr:`Route.instance` first reads it, so readers that resolved + the path through the prefix route continue on it; its end then moves the prefix route to + a fresh instance, so a job that restarts is a new stream. A prefix route that names its + instance lends none, and the route mints its own. Created visible, it announces now and + answers the waiting requesters; created ``hidden``, they wait until + :meth:`Route.announce`, so the job can fill its catalog first, and cancelling the route + before then refuses them. Ending the route withdraws only this path. """ return Route(self._inner.route(_options(hidden=hidden, cost=cost, instance=instance, epoch=epoch))) From c76e9872bffc285bc0f2aef22e9b182ae946472a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 18:14:47 -0800 Subject: [PATCH 61/61] fix(net): a refused bound job hands its request back to the worker A front that joined the shared request after the claim moved asks under the new instance, so treating the refusal as stale missed it and it failed with Unroutable. Requeue the request on the claim instead, as JS does, so every requester keeps waiting, and drop the front rule. Co-Authored-By: Claude Opus 5.5 --- js/net/src/origin-request.test.ts | 11 +++--- rs/moq-net/src/model/origin.rs | 48 +++++++++++++++++++++------ rs/moq-net/src/model/request_queue.rs | 16 +++++++++ 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index ae7c9f5d541..01b77ae4888 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -1043,8 +1043,8 @@ test("a hidden job announces under the instance its claim moved to", async () => }); // A hidden job that read its instance, as a signer does, announces under it or not at all: once an -// unrelated job's end moves the claim past it, the announce is refused and its request is asked -// again, so the worker prepares the job under the claim's new instance. +// unrelated job's end moves the claim past it, the announce is refused and its request goes back to +// the worker, which prepares the job under the claim's new instance. Every requester waits on. test("a job that read its instance is refused once its claim moves", async () => { const origin = new Producer(); const claim = origin.routePrefix(Path.from("room")); @@ -1065,9 +1065,10 @@ test("a job that read its instance is refused once its claim moves", async () => const moved = claim.instance; expect(moved).not.toBe(minted); expect(hidden.instance).toBe(minted); + const late = origin.request(Path.from("room/bob")); expect(() => hidden.announce()).toThrow(expect.objectContaining({ code: StreamCode.Unroutable })); await settle(); - expect(bob.closed.peek()).toBeUndefined(); + for (const request of [bob, late]) expect(request.closed.peek()).toBeUndefined(); const third = (await requests.next()).value; if (!third) throw new Error("expected the request again"); @@ -1076,8 +1077,8 @@ test("a job that read its instance is refused once its claim moves", async () => expect(again.instance).toBe(moved); again.announce(); await settle(); - expect(bob.active.peek()?.instance).toBe(moved); - for (const request of [alice, bob]) request.close(); + for (const request of [bob, late]) expect(request.active.peek()?.instance).toBe(moved); + for (const request of [alice, bob, late]) request.close(); again.close(); claim.close(); origin.close(); diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 2b1ea7f5e46..f119a770637 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -3040,14 +3040,16 @@ impl Route { /// [`Driver`] was dropped, and with [`Error::Unroutable`], announcing nothing, when a /// job's request was released or its prefix route retracted meanwhile, or when the job /// read its [`instance`](Self::instance) and its prefix route has moved past it since: - /// that route can never announce. Its requesters ask the prefix route again. + /// that route can never announce. In that last case the request goes back to the prefix + /// route's handler, its requesters still waiting. pub fn announce(&self) -> Result<(), Error> { let mut control = self.state.control.lock(); let info = control.info(); - let Some(request) = control.request.take() else { + let Some(mut request) = control.request.take() else { return self.state.announcement.update_if(info, None, |_| true).map(|_| ()); }; let bound = std::mem::take(&mut control.bound).then(|| control.instance.0.clone()); + let mut moved = false; let answer = self.broadcast().consume(); let mut resolved = None; // Answered under the table lock that every release takes first, so the route @@ -3075,6 +3077,7 @@ impl Route { .lender(&request.path, &request.home) .is_some_and(|lent| lent.instance != *bound) { + moved = true; return false; } // The exact route owns this answer's availability. Caching it under the @@ -3092,6 +3095,10 @@ impl Route { // advertising through the ordinary branch now that the request is taken. self.state.announcement.retract(false); drop(control); + // The prefix route moved past the instance the job read: its requesters wait on + // for the handler to prepare the job again under the new one. + request.requeue = moved; + drop(request); return Err(admitted.err().unwrap_or(Error::Unroutable)); }; drop(control); @@ -3184,6 +3191,7 @@ impl Route { producer, home: server.clone(), publisher: self.state.publisher.clone(), + requeue: false, })) } @@ -4878,13 +4886,6 @@ impl kio::Task for FrontServe { standing: false, }), }, - // A refusal for an instance the route has since moved off, such as a - // job's announce refused because it read its claim's old instance, is - // not the answer for the new one: ask again. - Err(err) if self.stale(route, &asked) => Event::Resolved { - route, - result: Err(Refusal { err, standing: false }), - }, Ok(source) => { let id = self.next_source; self.next_source += 1; @@ -6467,6 +6468,18 @@ impl ServeState { fn forget(shared: &kio::Shared, path: &Path, producer: &kio::Producer) { shared.lock().requests.remove_if(path, |p| p.same_channel(producer)); } + + /// Queue the still-pending entry again for the next handler, its requesters still waiting, + /// or drop it as [`Self::forget`] does when no handler is left to take it. + fn requeue(shared: &kio::Shared, path: &Path, producer: &kio::Producer) { + let mut state = shared.lock(); + if !state.requests.get(path).is_some_and(|p| p.same_channel(producer)) { + return; + } + if !state.requests.requeue(path.clone()) { + state.requests.remove_if(path, |p| p.same_channel(producer)); + } + } } /// A pending request for a broadcast to be served on demand. @@ -6500,6 +6513,10 @@ pub struct Request { // Applications can create routes in this origin; a session-forwarded request // can only serve its peer's already-routed broadcast. publisher: Option, + + // Dropped back onto its queue rather than refused: its job read an instance the prefix + // route has since moved past, so the handler answers it again. + requeue: bool, } impl Request { @@ -6551,6 +6568,12 @@ impl Request { impl Drop for Request { fn drop(&mut self) { + // Requeued in the same lock as the removal it replaces, so no handler pops it while + // this one still holds it. + if self.requeue { + ServeState::requeue(&self.home, &self.path, &self.producer); + return; + } // Handed off but neither accepted nor rejected: drop the still-pending entry so its // producer clone (plus this one) closes the channel, resolving coalesced requesters to // `Unroutable` rather than hanging. @@ -12290,7 +12313,8 @@ mod tests { /// A hidden job that read its instance, as a signer does, announces under it or not at /// all: once a sibling's end moves the claim past it, the announce is refused and its - /// requesters ask the claim again, so the worker prepares the job under the new instance. + /// request goes back to the worker, which prepares the job under the new instance. Every + /// requester waits on, including one that joined after the claim moved. #[kio::test] #[cfg(feature = "async")] async fn a_job_that_read_its_instance_is_refused_once_the_claim_moves() { @@ -12310,12 +12334,14 @@ mod tests { let second = claim.instance(); assert_ne!(second, first); assert_eq!(prepared.instance(), first, "a read instance moved"); + let late = consumer.request("room/alice", None); assert!(matches!(prepared.announce(), Err(Error::Unroutable))); let again = queued(&claim).await.route(RouteOptions::hidden()).unwrap(); assert_eq!(again.instance(), second); again.announce().unwrap(); - assert_eq!(alice.await.unwrap().info().instance, Some(second)); + assert_eq!(alice.await.unwrap().info().instance, Some(second.clone())); + assert_eq!(late.await.unwrap().info().instance, Some(second)); } /// A request pinned to the claim's instance resolves through its job's route even when a diff --git a/rs/moq-net/src/model/request_queue.rs b/rs/moq-net/src/model/request_queue.rs index ac27eefc763..8db9bf7453b 100644 --- a/rs/moq-net/src/model/request_queue.rs +++ b/rs/moq-net/src/model/request_queue.rs @@ -81,6 +81,22 @@ impl RequestQueue { Some(key) } + /// Queue a popped request again, still joinable, for the next handler to pop. Returns + /// `false`, leaving it as it is, when no handler is alive or nothing popped is pending. + pub fn requeue(&mut self, key: K) -> bool { + if self.handlers == 0 { + return false; + } + match self.pending.get_mut(&key) { + Some(entry) if !entry.queued => { + entry.queued = true; + self.order.push_back(key); + true + } + _ => false, + } + } + /// The pending request for `key`, if any. pub fn get(&self, key: &Q) -> Option<&V> where