From 22d0b0ef9dad70537cc980d2b9ecdfbbdd54c774 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 20:33:45 +0200 Subject: [PATCH 1/7] toyos-net-tcp: ready and orphans, what a caller that bounds [tcp] counts A listener's owner is woken for the connections that finished their handshake, and nothing told a caller how many wait short of accepting them: `Tcp::ready` does. A connection `close` leaves [tcp] to finish alone is still two buffers and its peer decides for how long, so a caller that bounds what it holds has to count them: `Tcp::orphans`, kept as the user changes hands and as the connection is freed. The shard forwards both, and `listener_port` and `close_listener`, which it had left out. None of them settles: no flow becomes eligible, and the resets a closed listener owes leave outside the round. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- toyos-net-shard/src/lib.rs | 19 +++++++++ toyos-net-shard/tcp/src/stack.rs | 18 +++++++++ toyos-net-shard/tcp/tests/held.rs | 66 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 toyos-net-shard/tcp/tests/held.rs diff --git a/toyos-net-shard/src/lib.rs b/toyos-net-shard/src/lib.rs index 5d2f804c14e..943fb772c32 100644 --- a/toyos-net-shard/src/lib.rs +++ b/toyos-net-shard/src/lib.rs @@ -466,6 +466,25 @@ impl Shard { self.tcp.accept(id) } + pub fn listener_port(&mut self, id: ListenerId) -> Result { + self.tcp.listener_port(id) + } + + /// [`Tcp::ready`]. + pub fn ready(&mut self, id: ListenerId) -> Result { + self.tcp.ready(id) + } + + /// [`Tcp::close_listener`]. The resets it owes leave outside the round. + pub fn close_listener(&mut self, now: Instant, id: ListenerId) -> Result<(), toyos_net_tcp::Error> { + self.tcp.close_listener(now, id) + } + + /// [`Tcp::orphans`]. + pub fn orphans(&self) -> usize { + self.tcp.orphans() + } + pub fn send(&mut self, now: Instant, id: ConnId, data: &[u8]) -> Result { let done = self.tcp.send(now, id, data); self.settle(now); diff --git a/toyos-net-shard/tcp/src/stack.rs b/toyos-net-shard/tcp/src/stack.rs index 3bdcbb9afde..4f71e2fb481 100644 --- a/toyos-net-shard/tcp/src/stack.rs +++ b/toyos-net-shard/tcp/src/stack.rs @@ -216,6 +216,8 @@ pub struct Tcp { time_waits: BTreeSet<(Instant, Tuple)>, deadlines: BTreeSet<(Instant, u32)>, /// Connections that became eligible for the caller's round since it last drained them. + /// Connections whose user let go and that have not ended. + orphans: usize, eligible: Vec, /// Connections freed while offered to the caller's round, since it last drained them. gone: Vec, @@ -375,6 +377,7 @@ impl Tcp { demux: BTreeMap::new(), time_waits: BTreeSet::new(), deadlines: BTreeSet::new(), + orphans: 0, eligible: Vec::new(), gone: Vec::new(), stubs: VecDeque::new(), @@ -401,6 +404,11 @@ impl Tcp { self.time_waits.len() } + /// Connections [`Self::close`] left to finish alone, each until it ends. + pub fn orphans(&self) -> usize { + self.orphans + } + fn local(&self, tuple: &Tuple) -> Local { let mtu = self.config.mtu; Local { @@ -441,6 +449,9 @@ impl Tcp { fn free(&mut self, index: u32) { let Some(generation) = self.conns.get(usize::try_from(index).unwrap_or(usize::MAX)).map(|s| s.generation) else { return }; let Some(conn) = release(&mut self.conns, &mut self.free_conns, index) else { return }; + if conn.user == User::Orphan { + self.orphans = self.orphans.saturating_sub(1); + } let remote = conn.tuple.remote.addr; let parked = self.parked.get(&remote).is_some_and(|p| p.conns.contains(&index)); // Its index may name another connection next. @@ -572,6 +583,11 @@ impl Tcp { slot(&mut self.listeners, id.index, id.generation).map(|l| l.options = options).ok_or(Error::NoSuchSocket) } + /// How many children completed their handshake and wait for [`Self::accept`]. + pub fn ready(&mut self, id: ListenerId) -> Result { + slot(&mut self.listeners, id.index, id.generation).map(|l| l.ready.len()).ok_or(Error::NoSuchSocket) + } + /// The oldest child that completed its handshake. pub fn accept(&mut self, id: ListenerId) -> Result, Error> { let listener = slot(&mut self.listeners, id.index, id.generation).ok_or(Error::NoSuchSocket)?; @@ -1056,6 +1072,7 @@ impl Tcp { Tcb::SynRcvd(rcvd) => { rcvd.shutdown_write(); conn.user = User::Orphan; + self.orphans = self.orphans.saturating_add(1); self.settle(id.index, now); } Tcb::Sync(sync) if sync.rx.unread() > 0 => { @@ -1068,6 +1085,7 @@ impl Tcp { sync.shutdown_write(now); sync.orphan(now); conn.user = User::Orphan; + self.orphans = self.orphans.saturating_add(1); self.settle(id.index, now); } } diff --git a/toyos-net-shard/tcp/tests/held.rs b/toyos-net-shard/tcp/tests/held.rs new file mode 100644 index 00000000000..08d9389e337 --- /dev/null +++ b/toyos-net-shard/tcp/tests/held.rs @@ -0,0 +1,66 @@ +//! What [tcp] says it holds, for a caller that bounds it: `orphans` and `ready`. No scenario ids: +//! the specifications hold neither call; the connections counted are `close.rs`'s and +//! `listen.rs`'s, by their scenarios' own segments. + +mod common; + +use common::*; +use toyos_net_tcp::{limits, State}; + +#[test] +fn an_orphan_is_counted_until_both_fins() { + let mut h = fixture_e(); + assert_eq!(h.tcp.orphans(), 0, "a held connection is its user's"); + h.close(0); + assert_eq!(h.tcp.orphans(), 1); + h.input(10, seg(5001).ack(1002).fin()); + assert_eq!((h.tcp.orphans(), h.tcp.time_wait_count()), (0, 1), "TIME-WAIT is no connection"); +} + +#[test] +fn an_orphan_is_counted_until_its_idle_reset() { + let mut h = fixture_e(); + h.close(0); + h.input(10, seg(5001).ack(1002)); + h.at(60_009); + assert_eq!(h.tcp.orphans(), 1); + h.at(60_010); + assert_eq!(h.tcp.orphans(), 0); +} + +#[test] +fn a_close_before_the_handshake_ends_leaves_an_orphan_until_it_gives_up() { + let mut h = H::new(65_535); + h.start(0); + h.conn = Some(h.tcp.connect(h.now(), A, Some(port(49152)), ep(B, 80)).unwrap()); + h.transmit(); + h.input(5, seg(5000).syn().mss(1460)); + assert_eq!(h.status().state, State::SynReceived); + h.close(6); + assert_eq!(h.tcp.orphans(), 1); + h.at(2 * i64::try_from(limits::SYN_GIVE_UP.as_millis()).unwrap()); + assert_eq!(h.tcp.orphans(), 0); +} + +#[test] +fn a_close_that_resets_leaves_no_orphan() { + let mut h = fixture_e(); + h.input(0, seg(5001).ack(1001).len(100)); + h.close(1); + assert_eq!(h.tcp.orphans(), 0, "unread text makes the close a reset"); +} + +#[test] +fn ready_counts_the_connections_accept_has_yet_to_return() { + let mut h = listening(); + let listener = h.listener.unwrap(); + h.input(0, seg(5000).syn().mss(1460)); + assert_eq!(h.tcp.ready(listener), Ok(0), "a handshake in progress is not ready"); + h.input(1, seg(5001).ack(1001)); + assert_eq!(h.tcp.ready(listener), Ok(1)); + assert!(h.tcp.accept(listener).unwrap().is_some()); + assert_eq!(h.tcp.ready(listener), Ok(0)); + let now = h.now(); + h.tcp.close_listener(now, listener).unwrap(); + assert_eq!(h.tcp.ready(listener), Err(toyos_net_tcp::Error::NoSuchSocket)); +} From 1e97631c0c1f2325777135cf0e045db2f56bff18 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 20:33:46 +0200 Subject: [PATCH 2/7] toyos-net-node: listeners on [tcp]'s accept queue, and one bound on what clients make the node hold Stage E of the cut of the track's stage 5. A listener is a passive open of [tcp]'s and the pipe its owner reads its wakes from; the queues stay [tcp]'s. The owner holds one unspent wake for each connection that waits and that there is a place for, written from what [tcp] and the bound say when a pass ends, so no wake depends on the pass that saw a connection arrive. An accept makes the oldest waiting connection a stream on its pipes. Places are the stream cap stage D left out, and the listeners' too: a stream, a listener and a connection [tcp] is finishing alone each hold one, and a connect, a listen and an accept past the number are refused with nothing made. The number is the shell's and a new node has none, so `connect` answers a refusal of the node's own beside the stack's. The two recorded listener failures of the stack this replaces are tests here that read the other way: a second peer is answered while a first handshake hangs, and a connect between two accepts is queued. Neither issue closes: what ships still has both until the move. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- issues/toyos-has-its-own-network-stack.md | 7 +- userland/netstack/node/Cargo.toml | 2 +- userland/netstack/node/src/lease/tcp.rs | 39 +- userland/netstack/node/src/lib.rs | 16 +- userland/netstack/node/src/listeners.rs | 177 ++++++ userland/netstack/node/src/places.rs | 34 + userland/netstack/node/src/streams.rs | 66 +- userland/netstack/node/tests/common/mod.rs | 6 +- userland/netstack/node/tests/listeners.rs | 702 +++++++++++++++++++++ userland/netstack/node/tests/streams.rs | 4 +- 10 files changed, 1023 insertions(+), 30 deletions(-) create mode 100644 userland/netstack/node/src/listeners.rs create mode 100644 userland/netstack/node/src/places.rs create mode 100644 userland/netstack/node/tests/listeners.rs diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index 7947e7f4ead..e45a1737521 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -16,7 +16,7 @@ netd runs smoltcp. Its replacement is ToyOS's own stack, built clean-room: reade - Stage 5: netd on one shard, pipe ABI unchanged; smoltcp leaves netd, `Cargo.toml` and `Cargo.lock` in the same PR. - Then multi-core, netring (blocked on the owner's ABI ruling), TCP and IP hardening, IPv6, offloads, soak. -Stage 5 lands in slices, the orchestrator's cut under the owner's words "i want smoltcp out as fast as possible": netd's decisions go into `toyos-net-node` (`userland/netstack/node`), pure and host-tested and shipped in nothing, and then one change moves netd onto it and deletes smoltcp. In the tree: the node, its DHCP lease and streams. Still to build on it: datagram sockets and mDNS, the resolver, listeners; then the move. +Stage 5 lands in slices, the orchestrator's cut under the owner's words "i want smoltcp out as fast as possible": netd's decisions go into `toyos-net-node` (`userland/netstack/node`), pure and host-tested and shipped in nothing, and then one change moves netd onto it and deletes smoltcp. In the tree: the node, its DHCP lease, streams, listeners and the bound on both (`userland/netstack/node/src/places.rs`). Still to build on it: datagram sockets and mDNS, the resolver; then the move. What the node does not yet meet: @@ -24,8 +24,11 @@ What the node does not yet meet: - It counts a DHCP message [udp] refused as `node.dhcp-unsent`, whatever the rule; the `dhcp.renew-unroutable` scenario owed above is not written. Exit: that scenario names the counter, or the node counts the renewal apart. - With the link down the DHCP client keeps its timers: every 4 to 64 s the node has a deadline, builds a DISCOVER that [udp] refuses, and counts it in `dhcp.tx.discover` and `node.dhcp-unsent`, from `Node::new` on. Exit: `toyos-dhcp`'s client is told the link went down and waits for it, and the node's first DISCOVER is the one that leaves. - No second TCP has been on the far end of a stream: the peer of `userland/netstack/node/tests/streams.rs` is the tests' own script, which acknowledges what arrives in order and loses, reorders and repeats nothing; what is not ours there is `etherparse`, which reads every segment the node emits and builds every one it receives. Exit: netd runs on the node against the host kernel's TCP through slirp in a guest. -- The node has no bound on its streams, where netd refuses a connect past `max_piped_connections`, and a connection whose client let go is [tcp]'s alone and counted by nobody. Exit: listeners share the bound, so it lands with them or with the move, and it says whether a connection [tcp] is finishing alone holds a place. +- The node's places are one number for every client: a program that connects or listens in a loop takes them all, and so does a peer that keeps the connections [tcp] is finishing alive, each of which holds its stream's place (next line). The node has no word for the program behind a request, as `issues/netstack-lookup-slots-have-no-per-client-share.md` records of netd. Exit: that issue's. +- A listener's handshakes in progress are [tcp]'s `limits::LISTEN_PENDING`, each held until `limits::SYNACK_GIVE_UP`, and a SYN past them is dropped: 256 SYNs a minute from addresses that never answer shut a port to every other peer for as long as they keep coming. [tcp] has no SYN cookie and lets no older handshake go for a newer one. Exit: a test in `toyos-net-shard/tcp/tests/` in which a peer's handshake completes while LS-12's flood runs. +- The node counts a wake unspent until an accept arrives, and cannot tell an accept on its way from one that will never be sent: an owner that reads a wake and sends no accept is owed one wake fewer from then on, and the last connection waiting is not announced. That is the node's half of `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`. Exit: that issue's, by a client that cannot spend a wake without its accept arriving, or by an accept that waits in netd and needs no wake. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. +- `Tcp::ready` and `Tcp::orphans` are additions to [tcp] the specifications do not hold, made for the node's wakes and places; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. Exit: the specifications hold both calls and the tests name their scenarios. - `node.address-refused` has no test: `toyos-dhcp` accepts no address or prefix [ip] refuses, by the same `toyos-net-wire` checks in both, so the refusal cannot be reached from the wire. Exit: the client hands [ip] a type that carries the check, and the counter goes. The listener defects are this track's: `issues/a-handshake-nobody-finishes-holds-a-listeners-port-shut.md` and `issues/a-connect-between-two-accepts-is-reset.md`, on smoltcp until stage 5, and `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`, in std's accept. diff --git a/userland/netstack/node/Cargo.toml b/userland/netstack/node/Cargo.toml index b854d416dc5..d3a57e806ee 100644 --- a/userland/netstack/node/Cargo.toml +++ b/userland/netstack/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toyos-net-node" -description = "Netstack's decisions, pure: one interface of ToyOS's own stack, its DHCP lease and its clients' streams, frames and time in, frames out against the device's credit." +description = "Netstack's decisions, pure: one interface of ToyOS's own stack, its DHCP lease, its clients' streams and listeners and the bound on both, frames and time in, frames out against the device's credit." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/userland/netstack/node/src/lease/tcp.rs b/userland/netstack/node/src/lease/tcp.rs index ec75535e602..0a6fb9debdf 100644 --- a/userland/netstack/node/src/lease/tcp.rs +++ b/userland/netstack/node/src/lease/tcp.rs @@ -1,8 +1,10 @@ -//! [tcp]'s calls on the interface, for `streams`. They reach no address, gateway or resolver: the -//! shard stays `lease`'s to configure. +//! [tcp]'s calls on the interface, for `streams` and `listeners`. They reach no address, gateway +//! or resolver: the shard stays `lease`'s to configure. + +use core::net::Ipv4Addr; use toyos_net_shard::ConnectError; -use toyos_net_tcp::{ConnId, Endpoint, Error, Options, Received, Status}; +use toyos_net_tcp::{ConnId, Endpoint, Error, ListenerId, Options, Received, Status, Tuple}; use toyos_net_wire::{Instant, Port}; use super::Stack; @@ -61,4 +63,35 @@ impl Stack { pub(crate) fn set_options(&mut self, now: Instant, id: ConnId, options: Options) { held(self.shard.set_options(now, id, options)); } + + /// Connections `close` left [tcp] to finish alone, each until it ends. + pub(crate) fn tcp_orphans(&self) -> usize { + self.shard.orphans() + } + + /// A passive open on every address the interface holds or comes to hold, at `port` or at one + /// of `random`'s candidates. `None` is a port that is taken, or no candidate free. + pub(crate) fn tcp_listen(&mut self, port: Option, random: impl FnMut() -> u16) -> Option<(ListenerId, Port)> { + match self.shard.listen(Ipv4Addr::UNSPECIFIED, port, random) { + Ok(id) => Some((id, held(self.shard.listener_port(id)))), + Err(Error::AddrInUse) => None, + Err(refusal) => unreachable!("[tcp] refused a passive open for more than its port: {refusal:?}"), + } + } + + /// How many connections finished their handshake at `id` and wait to be accepted. + pub(crate) fn tcp_ready(&mut self, id: ListenerId) -> usize { + held(self.shard.ready(id)) + } + + /// The oldest connection waiting at `id`, the caller's from here, and its two endpoints. + pub(crate) fn tcp_accept(&mut self, id: ListenerId) -> Option<(ConnId, Tuple)> { + let conn = held(self.shard.accept(id))?; + Some((conn, held(self.shard.tuple(conn)))) + } + + /// [tcp] resets every connection still waiting at `id`; `id` names nothing after. + pub(crate) fn tcp_close_listener(&mut self, now: Instant, id: ListenerId) { + held(self.shard.close_listener(now, id)); + } } diff --git a/userland/netstack/node/src/lib.rs b/userland/netstack/node/src/lib.rs index 4fc03ff1176..edc8e364a14 100644 --- a/userland/netstack/node/src/lib.rs +++ b/userland/netstack/node/src/lib.rs @@ -6,7 +6,8 @@ //! //! The node carries out what the client asks and tells it what became of its address. What the //! interface holds is `lease`'s to write and nobody else's: see that module for the three rules -//! its types keep. A client's TCP connection and its two pipes are `streams`'. +//! its types keep. A client's TCP connection and its two pipes are `streams`', a passive open and +//! its owner's wakes `listeners`', and how many of either the node holds `places`'. //! //! **Untrusted input.** A received frame is never read here: every byte goes through //! `toyos-net-wire`'s parsers inside the shard, and a DHCP payload through the client's. What @@ -30,9 +31,12 @@ extern crate alloc; mod lease; +mod listeners; +mod places; mod streams; -pub use streams::{FromClient, PipeEnd, PipeRefusal, Pipes, StreamEvent, StreamId, ToClient, Watch}; +pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId, Wake}; +pub use streams::{ConnectRefused, FromClient, PipeEnd, PipeRefusal, Pipes, StreamEvent, StreamId, ToClient, Watch}; use alloc::vec; use alloc::vec::Vec; @@ -65,10 +69,14 @@ pub struct Node { /// Room for the largest message the client accepts. datagram: Vec, streams: streams::Streams, + listeners: listeners::Listeners, + /// How many streams, listeners and connections [tcp] finishes alone the node holds at most. + places: usize, } impl Node { - /// One interface, link down and unaddressed. The client begins here, and its first DISCOVER + /// One interface, link down and unaddressed, with no place for a stream or a listener until + /// [`Self::set_places`]. The client begins here, and its first DISCOVER /// is not sent: an exchange starts over when the link comes up ([`Self::link`]). pub fn new(now: Instant, config: Config, host_name: Option, draw: impl FnMut() -> u32) -> Result { let mac = config.mac; @@ -81,6 +89,8 @@ impl Node { events: Vec::new(), datagram: vec![0; usize::from(toyos_dhcp::limits::MAX_MESSAGE)], streams: streams::Streams::default(), + listeners: listeners::Listeners::default(), + places: 0, }) } diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs new file mode 100644 index 00000000000..ced370d03b9 --- /dev/null +++ b/userland/netstack/node/src/listeners.rs @@ -0,0 +1,177 @@ +//! Listeners: a passive open of [tcp]'s and the pipe its owner reads its wakes from. The queues +//! are [tcp]'s: handshakes in progress, and connections that finished theirs and wait, oldest +//! first. A connection that waits is nobody's stream yet; an accept makes the oldest one a +//! stream on the pipes the accept hands over. +//! +//! **The owner holds one unspent wake for each connection that waits and that there is a place +//! for, and an accept spends one whatever it answers.** The owner reads a wake and then accepts, +//! once; the node writes what is missing whenever a pass ends ([`Node::bridge`]), from what +//! [tcp] holds and what `places` has left at that moment, so no wake depends on the pass that +//! saw a connection arrive. A wake left over by a connection its peer reset before the accept +//! stands for the next one, and until then its accept is answered [`AcceptRefused::Nothing`]. +//! The node cannot tell an accept that is on its way from one that will never be sent: an owner +//! that reads a wake and sends no accept is owed one wake fewer from then on. +//! +//! **A listener lives until its owner lets go**: [`Node::close_listener`], or a wake its pipe +//! refuses, for whatever reason. [tcp] then resets every connection that still waits, so each +//! peer learns at once, and the port is free. + +use alloc::boxed::Box; +use alloc::collections::{BTreeMap, VecDeque}; + +use toyos_net_tcp::Endpoint; +use toyos_net_wire::{Instant, Port}; + +use crate::streams::{PipeRefusal, Pipes, StreamId}; +use crate::Node; + +/// The write end of the pipe a listener's owner reads its wakes from. Dropped, the owner reads +/// the end. +pub trait Wake { + /// Writes one wake. + fn wake(&mut self) -> Result<(), PipeRefusal>; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ListenerId(u64); + +/// Why a listen made no listener. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ListenRefused { + /// The node holds all it has places for (`places`). + Full, + /// The port is another listener's, or no drawn port was free. + InUse, +} + +/// The answer to an accept that took a connection: its stream, established. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Accepted { + pub id: StreamId, + pub remote: Endpoint, + pub local: Port, +} + +/// Why an accept took no connection. Whatever waits still waits, and the pipe ends are dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AcceptRefused { + /// The id names no listener. + NoListener, + /// The request carried no pipes. + NoPipes, + /// The node holds all it has places for (`places`), whatever waits. + Full, + /// No connection waits. + Nothing, +} + +struct Listener { + bound: toyos_net_tcp::ListenerId, + owner: Box, + /// Wakes written that no accept has spent. + unspent: usize, +} + +#[derive(Default)] +pub(crate) struct Listeners { + live: BTreeMap, + /// The next listener's id: none is used twice. + next: u64, + refused: VecDeque<(ListenerId, PipeRefusal)>, +} + +impl Node { + /// A passive open on `port`, or on a port chosen from `draw`'s candidates, at every address + /// the interface holds or comes to hold. Answers the listener and its port. Refused, nothing + /// was made and `owner` is dropped. + pub fn listen(&mut self, port: Option, owner: Box, mut draw: impl FnMut() -> u32) -> Result<(ListenerId, Port), ListenRefused> { + if self.room() == 0 { + return Err(ListenRefused::Full); + } + let candidate = || { + let [low, high, ..] = draw().to_le_bytes(); + u16::from_le_bytes([low, high]) + }; + let Some((bound, port)) = self.stack.tcp_listen(port, candidate) else { return Err(ListenRefused::InUse) }; + let id = ListenerId(self.listeners.next); + self.listeners.next = self.listeners.next.saturating_add(1); + self.listeners.live.insert(id, Listener { bound, owner, unspent: 0 }); + Ok((id, port)) + } + + /// The owner's accept: the oldest connection waiting at `id` becomes a stream on `pipes`, + /// and what it already received moves at once. + pub fn accept(&mut self, now: Instant, id: ListenerId, pipes: Option) -> Result { + let Some(listener) = self.listeners.live.get_mut(&id) else { return Err(AcceptRefused::NoListener) }; + listener.unspent = listener.unspent.saturating_sub(1); + let bound = listener.bound; + let answer = self.take(bound, pipes); + self.bridge(now); + answer + } + + fn take(&mut self, bound: toyos_net_tcp::ListenerId, pipes: Option) -> Result { + let Some(pipes) = pipes else { return Err(AcceptRefused::NoPipes) }; + if self.room() == 0 { + return Err(AcceptRefused::Full); + } + let Some((conn, tuple)) = self.stack.tcp_accept(bound) else { return Err(AcceptRefused::Nothing) }; + let id = self.streams.accepted(conn, pipes); + Ok(Accepted { id, remote: tuple.remote, local: tuple.local.port }) + } + + /// The owner lets go of the listener: netstack calls it for a close, and when the kernel + /// says nobody holds the other end of the wake pipe. `false` is an id that names no listener. + pub fn close_listener(&mut self, now: Instant, id: ListenerId) -> bool { + let closed = self.end_listener(now, id); + if closed { + self.wake_owners(now); + } + closed + } + + fn end_listener(&mut self, now: Instant, id: ListenerId) -> bool { + let Some(listener) = self.listeners.live.remove(&id) else { return false }; + self.stack.tcp_close_listener(now, listener.bound); + true + } + + /// Writes every owner the wakes it is owed, and ends a listener whose pipe refuses one. One + /// lookup in [tcp] a listener: nothing of [tcp]'s is moved or offered for sending. + pub(crate) fn wake_owners(&mut self, now: Instant) { + // A listener ended here gives its place back, which another's owner may be owed a wake + // for. + while self.wake_each(now) {} + } + + /// One round over the listeners, up to the first whose pipe refuses a wake; `true` ended it. + fn wake_each(&mut self, now: Instant) -> bool { + let room = self.room(); + let stack = &mut self.stack; + let refused = self.listeners.live.iter_mut().find_map(|(id, listener)| { + let owed = stack.tcp_ready(listener.bound).min(room); + while listener.unspent < owed { + if let Err(refusal) = listener.owner.wake() { + return Some((*id, refusal)); + } + listener.unspent = listener.unspent.saturating_add(1); + } + None + }); + let Some((id, refusal)) = refused else { return false }; + self.end_listener(now, id); + self.listeners.refused.push_back((id, refusal)); + true + } + + /// The listeners the node holds. + pub fn listeners(&self) -> usize { + self.listeners.live.len() + } + + /// The listeners ended since the last call because their owner's pipe refused a wake, and + /// what it answered. + pub fn drain_refused_listeners(&mut self) -> impl Iterator + '_ { + self.listeners.refused.drain(..) + } +} diff --git a/userland/netstack/node/src/places.rs b/userland/netstack/node/src/places.rs new file mode 100644 index 00000000000..4f0a8362d62 --- /dev/null +++ b/userland/netstack/node/src/places.rs @@ -0,0 +1,34 @@ +//! Places: the one bound on what clients make the node hold. A stream holds a place from its +//! connect or its accept until the node lets it go, a listener from its listen until it is +//! closed, and a connection the node closed holds the place its stream had until [tcp] has +//! finished it, because it is still two buffers of [tcp]'s and its peer decides for how long. +//! With no place left a connect, a listen and an accept are refused with nothing made. +//! +//! The number is the shell's: each stream is two of its client's pipes kept alive and watched, +//! each listener one. A new node has none. +//! +//! What a peer makes the node hold without a client asking is not counted here and is bounded +//! by [tcp] for each listener: `limits::LISTEN_PENDING` handshakes in progress, each given up +//! after `limits::SYNACK_GIVE_UP`, and `limits::LISTEN_READY` connections waiting to be +//! accepted, each with at most one receive buffer of text. + +use toyos_net_wire::Instant; + +use crate::Node; + +impl Node { + /// How many places the node has from here on. Nothing held is let go for a smaller number. + pub fn set_places(&mut self, now: Instant, places: usize) { + self.places = places; + self.wake_owners(now); + } + + /// The places taken: streams, listeners, and connections [tcp] is finishing alone. + pub fn held(&self) -> usize { + self.streams().saturating_add(self.listeners()).saturating_add(self.stack.tcp_orphans()) + } + + pub(crate) fn room(&self) -> usize { + self.places.saturating_sub(self.held()) + } +} diff --git a/userland/netstack/node/src/streams.rs b/userland/netstack/node/src/streams.rs index 4217b67f576..c142eb55895 100644 --- a/userland/netstack/node/src/streams.rs +++ b/userland/netstack/node/src/streams.rs @@ -66,7 +66,15 @@ pub trait FromClient { fn read(&mut self, out: &mut [u8]) -> Result; } -/// The two ends a client's connect hands over. +/// Why a connect made no stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConnectRefused { + /// The node holds all it has places for (`places`). + Full, + Stack(ConnectError), +} + +/// The two ends a client's connect or accept hands over. pub struct Pipes { pub to_client: Box, pub from_client: Box, @@ -128,6 +136,21 @@ struct Stream { } impl Stream { + /// An established connection on its client's pipes. + fn established(conn: ConnId, pipes: Pipes) -> Self { + Self { + conn, + to_client: Some(pipes.to_client), + from_client: Some(pipes.from_client), + connecting: false, + deadline: None, + done_writing: false, + held: false, + room: false, + nodelay: false, + } + } + /// One pass. `false` lets the stream go: its pipe ends drop with it, and its connection is /// [tcp]'s alone or gone. fn pass(&mut self, id: StreamId, now: Instant, stack: &mut Stack, events: &mut VecDeque) -> bool { @@ -237,6 +260,20 @@ pub(crate) struct Streams { } impl Streams { + /// Holds `stream` under an id of its own. + fn hold(&mut self, stream: Stream) -> StreamId { + let id = StreamId(self.next); + self.next = self.next.saturating_add(1); + self.live.insert(id, stream); + id + } + + /// Holds a connection a listener's owner accepted, established, on the pipes its accept + /// handed over. + pub(crate) fn accepted(&mut self, conn: ConnId, pipes: Pipes) -> StreamId { + self.hold(Stream::established(conn, pipes)) + } + pub(crate) fn next_deadline(&self) -> Option { self.live.values().filter_map(|stream| stream.deadline).min() } @@ -245,23 +282,13 @@ impl Streams { impl Node { /// An active open to `remote`, answered by a [`StreamEvent`] once the handshake ends or /// `timeout` passes. Refused, nothing was sent and the pipe ends are dropped. - pub fn connect(&mut self, now: Instant, remote: Endpoint, timeout: Option, pipes: Pipes) -> Result { - let conn = self.stack.connect(now, remote)?; - let id = StreamId(self.streams.next); - self.streams.next = self.streams.next.saturating_add(1); - let stream = Stream { - conn, - to_client: Some(pipes.to_client), - from_client: Some(pipes.from_client), - connecting: true, - deadline: timeout.map(|within| now.after(within)), - done_writing: false, - held: false, - room: false, - nodelay: false, - }; - self.streams.live.insert(id, stream); - Ok(id) + pub fn connect(&mut self, now: Instant, remote: Endpoint, timeout: Option, pipes: Pipes) -> Result { + if self.room() == 0 { + return Err(ConnectRefused::Full); + } + let conn = self.stack.connect(now, remote).map_err(ConnectRefused::Stack)?; + let deadline = timeout.map(|within| now.after(within)); + Ok(self.streams.hold(Stream { connecting: true, deadline, ..Stream::established(conn, pipes) })) } /// One pass over every stream: netstack calls it when a pipe it watches is ready. @@ -269,6 +296,7 @@ impl Node { let Streams { live, events, .. } = &mut self.streams; let stack = &mut self.stack; live.retain(|id, stream| stream.pass(*id, now, stack, events)); + self.wake_owners(now); } /// The client lets go of the stream: nobody reads it, and what its pipe still holds is sent @@ -279,6 +307,7 @@ impl Node { self.stack.abort(now, stream.conn); self.streams.live.remove(&id); self.streams.events.push_back(StreamEvent::Closed { id }); + self.wake_owners(now); return; } stream.to_client = None; @@ -329,6 +358,7 @@ impl Node { if held { self.stack.abort(now, stream.conn); self.streams.live.remove(&id); + self.wake_owners(now); } } diff --git a/userland/netstack/node/tests/common/mod.rs b/userland/netstack/node/tests/common/mod.rs index 52e8194f543..0408830333f 100644 --- a/userland/netstack/node/tests/common/mod.rs +++ b/userland/netstack/node/tests/common/mod.rs @@ -29,6 +29,9 @@ pub const R: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 254); pub const DNS: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 53); pub const MASK: Ipv4Addr = Ipv4Addr::new(255, 255, 255, 0); +/// The places `Wire`'s node has for streams and listeners. +pub const PLACES: usize = 8; + pub const DISCOVER: u8 = 1; pub const OFFER: u8 = 2; pub const REQUEST: u8 = 3; @@ -272,7 +275,8 @@ impl Wire { let now = Instant::from_millis(3_600_000); let config = Config { mac: IndividualMac::new(MacAddr(mac)).unwrap(), receive_buffer: 65_535, send_buffer: 65_535, secrets }; let mut draws = 0x5a00_0000; - let node = Node::new(now, config, HostName::new("toyos"), draw(&mut draws)).unwrap(); + let mut node = Node::new(now, config, HostName::new("toyos"), draw(&mut draws)).unwrap(); + node.set_places(now, PLACES); Self { node, mac, now, sent: Vec::new(), draws } } diff --git a/userland/netstack/node/tests/listeners.rs b/userland/netstack/node/tests/listeners.rs new file mode 100644 index 00000000000..3934e62ffee --- /dev/null +++ b/userland/netstack/node/tests/listeners.rs @@ -0,0 +1,702 @@ +//! Listeners and the node's places, against peers the test scripts at 192.0.2.2 and pipe ends it +//! fakes. No scenario ids: the specifications' listening scenarios are `toyos-net-tcp`'s; these +//! are what the node does between [tcp]'s queues, an owner's wakes and its accepts. +//! +//! What is not ours: every segment the node emits is read by `etherparse` as it leaves +//! ([`Net::hears`]), its IPv4 and TCP checksums and lengths that crate's sums, and every segment +//! a peer sends is built by it. The numbers each test asserts are RFC 9293's for LISTEN and +//! SYN-RECEIVED, cited where asserted. Two tests are recorded failures of the stack this one +//! replaces, played the same way and read the other way: each says which. The peers are ours: +//! each says only what a test tells it to, and nothing answers the node by itself but ARP. + +mod common; + +use std::cell::RefCell; +use std::collections::{BTreeSet, VecDeque}; +use std::net::Ipv4Addr; +use std::rc::Rc; +use std::time::Duration; + +use common::{arp, terms, Wire, A, MAC, MAC_B, MAC_R, R}; +use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, PipeRefusal, Pipes, StreamId, ToClient, Wake}; +use toyos_net_tcp::{limits, Counter, Endpoint}; +use toyos_net_wire::{Instant, Port}; + +const B: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); +/// The port the tests listen on, and another. +const SSH: u16 = 22; +const TELNET: u16 = 23; +/// The peers' ports. +const P1: u16 = 40_001; +const P2: u16 = 40_002; +const P3: u16 = 40_003; +/// Every peer's initial sequence number. +const ISS: u32 = 5000; + +const SYN: u8 = 1; +const FIN: u8 = 2; +const RST: u8 = 4; + +// ---- an owner's wake pipe ---- + +#[derive(Debug, Default)] +struct Notified { + /// The wakes written and not refused. + wakes: usize, + /// What the pipe answers a wake instead of taking it. + refusal: Option, + dropped: bool, +} + +type Owner = Rc>; + +struct WakeEnd(Owner); + +impl Wake for WakeEnd { + fn wake(&mut self) -> Result<(), PipeRefusal> { + let mut notified = self.0.borrow_mut(); + if let Some(refusal) = notified.refusal { + return Err(refusal); + } + notified.wakes += 1; + Ok(()) + } +} + +impl Drop for WakeEnd { + fn drop(&mut self) { + self.0.borrow_mut().dropped = true; + } +} + +fn wakes(owner: &Owner) -> usize { + owner.borrow().wakes +} + +// ---- a client's pipes ---- + +#[derive(Debug, Default)] +struct Ends { + /// What the node wrote: the pipe takes everything. + inbox: Vec, + /// What the client wrote and the node has not read. + outbox: VecDeque, + /// How many of the two ends the node dropped. + dropped: usize, +} + +type Client = Rc>; + +struct WriteEnd(Client); +struct ReadEnd(Client); + +impl ToClient for WriteEnd { + fn write(&mut self, bytes: &[u8]) -> Result { + self.0.borrow_mut().inbox.extend_from_slice(bytes); + Ok(bytes.len()) + } +} + +impl FromClient for ReadEnd { + fn read(&mut self, out: &mut [u8]) -> Result { + let mut ends = self.0.borrow_mut(); + let read = out.len().min(ends.outbox.len()); + if read == 0 { + return Err(PipeRefusal::WouldBlock); + } + for slot in &mut out[..read] { + *slot = ends.outbox.pop_front().unwrap(); + } + Ok(read) + } +} + +impl Drop for WriteEnd { + fn drop(&mut self) { + self.0.borrow_mut().dropped += 1; + } +} + +impl Drop for ReadEnd { + fn drop(&mut self) { + self.0.borrow_mut().dropped += 1; + } +} + +fn client() -> (Client, Pipes) { + let client = Client::default(); + let pipes = Pipes { to_client: Box::new(WriteEnd(client.clone())), from_client: Box::new(ReadEnd(client.clone())) }; + (client, pipes) +} + +// ---- the peers ---- + +/// A segment the node emitted, as `etherparse` read it. +#[derive(Clone, Debug)] +struct Segment { + /// The node's port, and the peer's. + from: u16, + to: u16, + seq: u32, + ack: Option, + syn: bool, + fin: bool, + rst: bool, + text: Vec, +} + +/// A segment from the peer's port `from` to the node's port `to` in its frame, built by +/// `etherparse`: window 65,535, an MSS option on a SYN, PSH with text. +fn frame(from: u16, to: u16, seq: u32, ack: Option, flags: u8, text: &[u8]) -> Vec { + let mut step = PacketBuilder::ethernet2(MAC_B, MAC).ipv4(B.octets(), A.octets(), 64).tcp(from, to, seq, 65_535); + if flags & SYN != 0 { + step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(); + } + if let Some(ack) = ack { + step = step.ack(ack); + } + if flags & FIN != 0 { + step = step.fin(); + } + if flags & RST != 0 { + step = step.rst(); + } + if !text.is_empty() { + step = step.psh(); + } + let mut frame = Vec::new(); + step.write(&mut frame, text).unwrap(); + frame +} + +fn v4(bytes: &[u8]) -> Ipv4Addr { + Ipv4Addr::from(<[u8; 4]>::try_from(bytes).expect("four bytes")) +} + +fn draw(draws: &mut u32) -> impl FnMut() -> u32 + '_ { + move || { + *draws = draws.wrapping_add(1); + *draws + } +} + +// ---- the node between them ---- + +struct Net { + node: Node, + now: Instant, + draws: u32, + /// Every TCP segment the node emitted, in order. + heard: Vec, +} + +impl Net { + /// The node holding its lease of 192.0.2.1/24 for an hour, with `common::PLACES` places. + fn new() -> Self { + let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); + Self { node, now, draws: 0x7e00_0000, heard: Vec::new() } + } + + /// Offers the node all the credit it wants until it sends nothing more, its ARP requests + /// for the peers' address and the router's answered. + fn pump(&mut self) { + for _ in 0..100_000 { + let mut frames = Vec::new(); + self.node.transmit(self.now, usize::MAX, |frame| frames.push(frame.to_vec())); + if frames.is_empty() { + return; + } + for frame in &frames { + if let Some(answer) = self.hears(frame) { + self.node.receive(self.now, &answer, draw(&mut self.draws)); + } + } + } + panic!("100,000 opportunities and the node still sends"); + } + + /// The outside reading of a frame the node emitted, and the ARP answer it gets if it asks + /// for one. Ethernet II from the node's MAC; then ARP, a datagram of the DHCP client's, or + /// a TCP segment to a peer, with the lengths and checksums `etherparse` computes. + fn hears(&mut self, frame: &[u8]) -> Option> { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; + assert_eq!(ethernet.source(), MAC, "from the node's MAC"); + match (&packet.net, &packet.transport) { + (Some(NetSlice::Arp(asked)), None) => { + let asks = asked.operation() == ArpOperation::REQUEST && v4(asked.sender_protocol_addr()) == A; + match v4(asked.target_protocol_addr()) { + target if asks && target == B => Some(arp(MAC, false, MAC_B, B, A)), + target if asks && target == R => Some(arp(MAC, false, MAC_R, R, A)), + _ => None, + } + } + (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(_))) => None, + (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) => { + let header = ip.header(); + assert_eq!(ethernet.destination(), MAC_B, "to the peers' MAC"); + assert_eq!(header.header_checksum(), header.to_header().calc_header_checksum(), "the IPv4 header checksum"); + assert_eq!(usize::from(header.total_len()), 20 + tcp.slice().len(), "IPv4's length is the segment's"); + assert_eq!((header.source_addr(), header.destination_addr()), (A, B)); + assert_eq!(tcp.checksum(), tcp.calc_checksum_ipv4(header.source(), header.destination()).unwrap(), "the TCP checksum"); + self.heard.push(Segment { + from: tcp.source_port(), + to: tcp.destination_port(), + seq: tcp.sequence_number(), + ack: tcp.ack().then(|| tcp.acknowledgment_number()), + syn: tcp.syn(), + fin: tcp.fin(), + rst: tcp.rst(), + text: tcp.payload().to_vec(), + }); + None + } + other => panic!("neither ARP, UDP nor TCP: {other:?}"), + } + } + + fn deliver(&mut self, frame: &[u8]) { + self.node.receive(self.now, frame, draw(&mut self.draws)); + self.pump(); + } + + /// Fires every deadline of the next `span`, each at its own instant, and ends `span` on. + fn run(&mut self, span: Duration) { + let end = self.now.after(span); + for _ in 0..100_000 { + match self.node.next_deadline() { + Some(at) if at <= end => { + self.now = self.now.max(at); + self.node.fire(self.now, draw(&mut self.draws)); + self.pump(); + } + _ => { + self.now = end; + return; + } + } + } + panic!("100,000 deadlines without the clock passing {span:?}"); + } + + /// The segments the node sent to the peer at `port`. + fn to(&self, port: u16) -> Vec { + self.heard.iter().filter(|segment| segment.to == port).cloned().collect() + } + + /// The last segment the node sent to the peer at `port`. + fn last(&self, port: u16) -> Segment { + self.to(port).pop().expect("a segment to the peer") + } + + /// The node's initial sequence number toward the peer at `port`: its SYN-ACK's. + fn iss(&self, port: u16) -> u32 { + self.to(port).iter().rev().find(|segment| segment.syn).expect("a SYN-ACK to the peer").seq + } + + /// The peer's SYN to the node's port `to`. + fn syn_to(&mut self, peer: u16, to: u16) { + self.deliver(&frame(peer, to, ISS, None, SYN, &[])); + } + + fn syn(&mut self, peer: u16) { + self.syn_to(peer, SSH); + } + + /// The peer's ACK of the node's SYN-ACK (RFC 9293 §3.5, figure 6, line 4). + fn finish(&mut self, peer: u16) { + let ack = self.iss(peer).wrapping_add(1); + self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), 0, &[])); + } + + fn handshake(&mut self, peer: u16) { + self.syn(peer); + self.finish(peer); + } + + /// The peer's first text. + fn text(&mut self, peer: u16, text: &[u8]) { + let ack = self.iss(peer).wrapping_add(1); + self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), 0, text)); + } + + /// The peer's reset at exactly the sequence number the node expects (RFC 9293 §3.10.7.4, + /// first check), having sent no text. + fn rst(&mut self, peer: u16) { + let ack = self.iss(peer).wrapping_add(1); + self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), RST, &[])); + } + + /// The peer's FIN, acknowledging the node's (RFC 9293 §3.6, case 1), neither having sent + /// text. + fn fin(&mut self, peer: u16) { + let ack = self.iss(peer).wrapping_add(2); + self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), FIN, &[])); + } + + fn listen(&mut self, port: u16) -> (ListenerId, Owner) { + let owner = Owner::default(); + let (id, bound) = self.node.listen(Port::new(port), Box::new(WakeEnd(owner.clone())), draw(&mut self.draws)).expect("a free port and a place"); + assert_eq!(bound.get(), port); + (id, owner) + } + + /// The owner's accept, on a new client's pipes. + fn accept(&mut self, id: ListenerId) -> Result<(Accepted, Client), AcceptRefused> { + let (client, pipes) = client(); + let answer = self.node.accept(self.now, id, Some(pipes)); + self.pump(); + answer.map(|accepted| (accepted, client)) + } + + /// An accept that takes the connection of the peer at `port`. + fn accepts(&mut self, id: ListenerId, port: u16) -> (StreamId, Client) { + let (accepted, client) = self.accept(id).expect("a connection and a place"); + assert_eq!((accepted.remote, accepted.local), (Endpoint { addr: B, port: Port::new(port).unwrap() }, Port::new(SSH).unwrap())); + (accepted.id, client) + } + + /// A connect to the peers' port 80, which nothing answers. + fn connect(&mut self) -> (Result, Client) { + let (client, pipes) = client(); + let answer = self.node.connect(self.now, Endpoint { addr: B, port: Port::new(80).unwrap() }, None, pipes); + self.pump(); + (answer, client) + } +} + +/// A node with `places` places. +fn with_places(places: usize) -> Net { + let mut net = Net::new(); + net.node.set_places(net.now, places); + net +} + +// ---- listen and accept ---- + +// RFC 9293 §3.10.7.2, third check: a SYN in LISTEN is answered +// with RCV.NXT the SYN's sequence number plus one, and the state is SYN-RECEIVED. §3.10.7.4, fifth +// check: the ACK of our SYN makes it ESTABLISHED, and only then is there a connection to accept. +#[test] +fn a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.syn(P1); + let answered = net.to(P1); + let [synack] = &answered[..] else { panic!("one SYN-ACK, not {:?}", net.heard) }; + assert!(synack.syn && !synack.fin && !synack.rst && synack.text.is_empty(), "{synack:?}"); + assert_eq!((synack.from, synack.ack), (SSH, Some(ISS + 1))); + assert_eq!(wakes(&owner), 0, "SYN-RECEIVED is no connection yet"); + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Nothing); + + net.finish(P1); + assert_eq!(wakes(&owner), 1); + net.text(P1, b"hello"); + assert_eq!((net.node.streams(), net.node.listeners(), net.node.held()), (0, 1, 1), "a connection that waits is nobody's stream"); + + let (client, pipes) = client(); + let accepted = net.node.accept(net.now, id, Some(pipes)).unwrap(); + assert_eq!((accepted.remote, accepted.local), (Endpoint { addr: B, port: Port::new(P1).unwrap() }, Port::new(SSH).unwrap())); + assert_eq!(client.borrow().inbox, b"hello", "what arrived before the accept moves in it"); + assert_eq!((net.node.streams(), net.node.listeners(), net.node.held()), (1, 1, 2)); + client.borrow_mut().outbox.extend(b"welcome"); + net.node.bridge(net.now); + net.pump(); + let said = net.last(P1); + assert_eq!((said.text.as_slice(), said.seq), (&b"welcome"[..], synack.seq.wrapping_add(1))); + assert_eq!((wakes(&owner), client.borrow().dropped), (1, 0)); +} + +// The recorded failure of `issues/a-handshake-nobody-finishes-holds-a-listeners-port-shut.md`: +// one SYN and nothing more, and the stack this one replaces answered the next peer's SYN with a +// reset for as long as the first handshake hung, which was for good. Here the next peer is +// answered at once, and the first handshake is given up within [tcp]'s bound, after which its +// late ACK meets LISTEN: RFC 9293 §3.10.7.2, second check, . +#[test] +fn a_handshake_nobody_finishes_leaves_the_port_open_and_is_given_up() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.syn(P1); + net.syn(P2); + let answered = net.to(P2); + let [synack] = &answered[..] else { panic!("a SYN-ACK to the second peer, not {:?}", net.heard) }; + assert!(synack.syn && !synack.rst && synack.ack == Some(ISS + 1), "{synack:?}"); + net.finish(P2); + assert_eq!(wakes(&owner), 1); + net.accepts(id, P2); + + let first = net.iss(P1); + net.run(2 * limits::SYNACK_GIVE_UP); + assert_eq!(net.node.shard().tcp_counters().get(Counter::SynAckGiveUp), 1); + let said = net.to(P1).len(); + net.run(Duration::from_secs(600)); + assert_eq!(net.to(P1).len(), said, "nothing more is sent for a handshake given up"); + assert!(net.heard.iter().all(|segment| !segment.rst), "{:?}", net.heard); + + net.deliver(&frame(P1, SSH, ISS + 1, Some(first.wrapping_add(1)), 0, &[])); + let reset = net.last(P1); + assert!(reset.rst && !reset.syn, "{reset:?}"); + assert_eq!((reset.seq, reset.ack), (first.wrapping_add(1), None)); + assert_eq!((wakes(&owner), net.node.streams(), net.node.listeners()), (1, 1, 1)); +} + +// The recorded failure of `issues/a-connect-between-two-accepts-is-reset.md`: a connection waits +// for its accept, and the stack this one replaces answered every other peer's SYN with a reset +// until the owner had taken it. Here each is answered, queued behind the one that waits, and +// accepted oldest first, with one wake a connection. +#[test] +fn a_connect_between_two_accepts_is_queued_not_reset() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + net.handshake(P2); + assert_eq!(wakes(&owner), 2, "one wake a connection"); + net.accepts(id, P1); + net.handshake(P3); + assert_eq!(wakes(&owner), 3); + net.accepts(id, P2); + net.accepts(id, P3); + assert!(net.heard.iter().all(|segment| !segment.rst), "{:?}", net.heard); + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Nothing); + assert_eq!((wakes(&owner), net.node.streams()), (3, 3)); +} + +// ---- wakes ---- + +// RFC 9293 §3.10.7.4, first check: a reset at RCV.NXT ends the connection. Nobody had taken it, +// so [tcp] returns it to no accept, and the wake written for it finds nothing. +#[test] +fn an_accept_spends_a_wake_whatever_it_answers() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + net.rst(P1); + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Nothing); + assert_eq!(wakes(&owner), 1); + + net.handshake(P2); + assert_eq!(wakes(&owner), 2, "the wake that accept spent is not counted against the next connection"); + assert_eq!(net.node.accept(net.now, id, None), Err(AcceptRefused::NoPipes)); + assert_eq!(wakes(&owner), 3, "the connection an accept without pipes left is announced again"); + net.accepts(id, P2); + assert_eq!(wakes(&owner), 3); +} + +#[test] +fn a_wake_left_by_a_connection_its_peer_reset_stands_for_the_next() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + net.rst(P1); + net.handshake(P2); + assert_eq!(wakes(&owner), 1, "the owner holds one wake and one connection waits"); + net.accepts(id, P2); + assert_eq!(wakes(&owner), 1); +} + +// ---- places ---- + +// The close is RFC 9293 §3.6, case 1: the node's FIN, then the peer's FIN acknowledging it. Until +// then the connection is [tcp]'s to finish and holds the place its stream had. +#[test] +fn a_wake_is_owed_only_for_a_connection_there_is_a_place_for() { + let mut net = with_places(2); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + net.handshake(P2); + assert_eq!(wakes(&owner), 1, "two wait, and there is a place for one"); + let (first, _client) = net.accepts(id, P1); + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Full); + assert_eq!((wakes(&owner), net.node.held()), (1, 2), "no place, no wake"); + + net.node.close(net.now, first); + net.pump(); + assert!(net.last(P1).fin); + assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 2, 1), "a connection [tcp] is finishing holds its place"); + net.fin(P1); + assert_eq!((net.node.held(), wakes(&owner)), (1, 2), "the place is back, and the connection that waited is announced"); + net.accepts(id, P2); +} + +#[test] +fn closing_a_connect_gives_its_place_to_a_connection_that_waits() { + let mut net = with_places(2); + let (_, owner) = net.listen(SSH); + let (connecting, _client) = net.connect(); + net.handshake(P1); + assert_eq!(wakes(&owner), 0); + net.node.close(net.now, connecting.unwrap()); + assert_eq!(wakes(&owner), 1); +} + +#[test] +fn a_stream_reset_for_its_pipe_gives_its_place_to_a_connection_that_waits() { + let mut net = with_places(2); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + let (stream, _client) = net.accepts(id, P1); + net.handshake(P2); + assert_eq!(wakes(&owner), 1); + net.node.pipe_broken(net.now, stream, PipeEnd::ToClient); + assert_eq!(wakes(&owner), 2); +} + +#[test] +fn closing_a_listener_gives_its_place_to_a_connection_that_waits_at_another() { + let mut net = with_places(2); + let (_, owner) = net.listen(SSH); + let (other, _) = net.listen(TELNET); + net.handshake(P1); + assert_eq!(wakes(&owner), 0); + assert!(net.node.close_listener(net.now, other)); + assert_eq!(wakes(&owner), 1); +} + +#[test] +fn more_places_wake_the_owner_of_a_connection_that_waits() { + let mut net = with_places(1); + let (_, owner) = net.listen(SSH); + net.handshake(P1); + assert_eq!(wakes(&owner), 0); + net.node.set_places(net.now, 2); + assert_eq!(wakes(&owner), 1); +} + +#[test] +fn a_connect_past_the_places_is_refused_and_sends_nothing() { + let mut net = with_places(2); + let (first, _a) = net.connect(); + let (second, _b) = net.connect(); + assert!(first.is_ok() && second.is_ok(), "{first:?} {second:?}"); + let (third, client) = net.connect(); + assert_eq!(third, Err(ConnectRefused::Full), "a connect not yet answered holds a place"); + assert_eq!((client.borrow().dropped, net.node.held()), (2, 2)); + let sources: BTreeSet = net.to(80).iter().map(|segment| segment.from).collect(); + assert_eq!(sources.len(), 2, "no SYN left for the connect refused: {:?}", net.heard); + + net.node.close(net.now, first.unwrap()); + let (fourth, _c) = net.connect(); + assert!(fourth.is_ok(), "{fourth:?}"); +} + +// RFC 9293 §3.10.7.1: a SYN for a port nothing listens on is answered +// , which is how the peer reads that the refused listen +// made nothing. +#[test] +fn a_listener_holds_a_place_and_a_listen_without_one_makes_nothing() { + let mut net = with_places(1); + let (id, _owner) = net.listen(SSH); + let refused = Owner::default(); + let answer = net.node.listen(Port::new(TELNET), Box::new(WakeEnd(refused.clone())), draw(&mut net.draws)); + assert_eq!(answer.unwrap_err(), ListenRefused::Full); + assert!(refused.borrow().dropped); + net.syn_to(P1, TELNET); + let reset = net.last(P1); + assert!(reset.rst && (reset.seq, reset.ack) == (0, Some(ISS + 1)), "{reset:?}"); + let (connect, _client) = net.connect(); + assert_eq!(connect, Err(ConnectRefused::Full), "the listener's place is a stream's"); + + assert!(net.node.close_listener(net.now, id)); + assert_eq!(net.node.held(), 0); + net.listen(TELNET); +} + +// ---- a listener's end ---- + +// The resets are [tcp]'s for a listener closed under its connections, in RFC 9293 §3.10.5's +// form for SYN-RECEIVED and ESTABLISHED: , SND.NXT our SYN's number plus +// one. +#[test] +fn closing_a_listener_resets_what_waits_and_frees_its_port() { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + net.syn(P1); + net.handshake(P2); + assert!(net.node.close_listener(net.now, id)); + net.pump(); + for peer in [P1, P2] { + let reset = net.last(peer); + assert!(reset.rst && reset.seq == net.iss(peer).wrapping_add(1), "{reset:?}"); + } + assert!(owner.borrow().dropped, "the owner reads the end"); + assert_eq!((net.node.listeners(), net.node.held()), (0, 0)); + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::NoListener); + assert!(!net.node.close_listener(net.now, id)); + + let (_, again) = net.listen(SSH); + net.handshake(P3); + assert_eq!((wakes(&owner), wakes(&again)), (1, 1)); +} + +#[test] +fn a_wake_the_owners_pipe_refuses_ends_the_listener() { + for refusal in [PipeRefusal::Gone, PipeRefusal::WouldBlock, PipeRefusal::Broken] { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + owner.borrow_mut().refusal = Some(refusal); + net.handshake(P1); + let ended: Vec<(ListenerId, PipeRefusal)> = net.node.drain_refused_listeners().collect(); + assert_eq!(ended, [(id, refusal)]); + assert!(owner.borrow().dropped && net.last(P1).rst, "{refusal:?}: {:?}", net.heard); + assert_eq!((net.node.listeners(), net.node.held()), (0, 0), "{refusal:?}"); + net.listen(SSH); + } +} + +// The drawn port is in the dynamic range, 49152 to 65535 (RFC 6335 §6), at the offset the draw's +// low sixteen bits name. +#[test] +fn a_listen_on_a_taken_port_is_refused_and_a_drawn_port_listens() { + let mut net = Net::new(); + let (_, owner) = net.listen(SSH); + let second = Owner::default(); + let answer = net.node.listen(Port::new(SSH), Box::new(WakeEnd(second.clone())), draw(&mut net.draws)); + assert_eq!(answer.unwrap_err(), ListenRefused::InUse); + assert!(second.borrow().dropped); + assert_eq!((net.node.listeners(), net.node.held()), (1, 1)); + net.handshake(P1); + assert_eq!(wakes(&owner), 1, "the listener that holds the port still listens"); + + let (_, port) = net.node.listen(None, Box::new(WakeEnd(Owner::default())), || 0x0001_0005).unwrap(); + assert_eq!(port.get(), 49_152 + 5); + net.syn_to(P2, port.get()); + let synack = net.last(P2); + assert!(synack.syn && synack.from == port.get(), "{synack:?}"); +} + +// ---- the wire is not trusted ---- + +#[test] +fn no_cut_of_a_syn_is_a_syn() { + let whole = frame(P1, SSH, ISS, None, SYN, &[]); + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + for len in 0..whole.len() { + net.deliver(&whole[..len]); + assert!(net.heard.is_empty(), "cut to {len} bytes: {:?}", net.heard); + } + net.handshake(P1); + assert_eq!(wakes(&owner), 1); + net.accepts(id, P1); +} + +#[test] +fn no_flipped_bit_of_a_syn_wakes_an_owner_or_ends_a_listener() { + let whole = frame(P1, SSH, ISS, None, SYN, &[]); + for index in 0..whole.len() { + for bit in 0..8 { + let mut net = Net::new(); + let (id, owner) = net.listen(SSH); + let mut flipped = whole.clone(); + flipped[index] ^= 1 << bit; + net.deliver(&flipped); + // A flip no checksum covers is in the link addresses: taken or refused, it is at most + // a handshake in progress. + assert_eq!((wakes(&owner), net.node.listeners(), net.node.streams()), (0, 1, 0), "byte {index} bit {bit}"); + net.handshake(P2); + assert_eq!(wakes(&owner), 1, "byte {index} bit {bit}"); + net.accepts(id, P2); + } + } +} diff --git a/userland/netstack/node/tests/streams.rs b/userland/netstack/node/tests/streams.rs index 54b9061bcb1..e412af9f042 100644 --- a/userland/netstack/node/tests/streams.rs +++ b/userland/netstack/node/tests/streams.rs @@ -18,7 +18,7 @@ use std::time::Duration; use common::{arp, terms, Wire, A, MAC, MAC_B, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{FromClient, Node, PipeEnd, PipeRefusal, Pipes, StreamEvent, StreamId, ToClient, Watch}; +use toyos_net_node::{ConnectRefused, FromClient, Node, PipeEnd, PipeRefusal, Pipes, StreamEvent, StreamId, ToClient, Watch}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{Endpoint, Failure}; use toyos_net_wire::{Instant, Port}; @@ -437,7 +437,7 @@ fn a_connect_before_the_lease_is_refused_and_sends_nothing() { wire.link(true); let (client, pipes) = client(); let refused = wire.node.connect(wire.now, peer(), None, pipes); - assert!(matches!(refused, Err(ConnectError::Route(_))), "{refused:?}"); + assert!(matches!(refused, Err(ConnectRefused::Stack(ConnectError::Route(_)))), "{refused:?}"); assert_eq!((wire.node.streams(), dropped(&client)), (0, (true, true))); // `common::outside` takes no TCP segment. wire.pump(); From 42a8fb25bd520b2914ad3a9e1294be0a0322b0bb Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 21:35:06 +0200 Subject: [PATCH 3/7] toyos-net-node: an accepted stream starts with its listener's options, and the places bound is tested across peer addresses Stage D's round 2 times any stream its client can see no more, an accepted one included, by its peer's address, and writes a stream's whole option set from the copy the stream holds. Three things about a stream an accept makes follow. Its options. Stream::established seeded every stream with Options::default(). That was true only while nothing gave a listener an option. Stream::established now takes the options, and Streams::accepted writes the listener's to [tcp] and seeds the stream with what it wrote, so the copy set_nodelay rewrites is the connection's by construction. The listener's options are the node's (Node::set_listener_nodelay, listener_nodelay): [tcp] hands its own copy for a listener to a connection at the SYN and has no call that reads a connection's back, so a seed taken from there is wrong for a connection begun before the option was set. A listening socket that takes TCP_NODELAY and hands it to what it accepts is what a host does; netstack on smoltcp refuses the request for a listener. Its peer address. Already tuple.remote.addr; the test that says so accepts seventeen from one address and one from another. Its id after a cut. A stream can now be cut while its client holds it. Every request for that id names nothing, and no id is used twice, so not the stream that took its place either: tested, with the place the cut gave back going to the connection that waited. The track's exit for the bound across addresses is the last test: peers at four addresses bring sixteen connections each to a node with three places. The track line and places.rs's header say what holds. The listeners tests' peers have an address: a segment to an address no peer is at fails the test, as in the streams tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- issues/toyos-has-its-own-network-stack.md | 2 +- userland/netstack/node/src/listeners.rs | 33 ++- userland/netstack/node/src/places.rs | 10 +- userland/netstack/node/src/streams.rs | 25 +- userland/netstack/node/tests/listeners.rs | 313 ++++++++++++++++++---- 5 files changed, 316 insertions(+), 67 deletions(-) diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index 8f8206c97f5..a6654ba9dd6 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -27,7 +27,7 @@ What the node does not yet meet: - The node's places are one number for every client: a program that connects or listens in a loop takes them all, and so does a peer that keeps the connections [tcp] is finishing alive, each of which holds its stream's place (the line below on a client gone with nothing left in its pipe). The node has no word for the program behind a request, as `issues/netstack-lookup-slots-have-no-per-client-share.md` records of netd. Exit: that issue's. - A listener's handshakes in progress are [tcp]'s `limits::LISTEN_PENDING`, each held until `limits::SYNACK_GIVE_UP`, and a SYN past them is dropped: 256 SYNs a minute from addresses that never answer shut a port to every other peer for as long as they keep coming. [tcp] has no SYN cookie and lets no older handshake go for a newer one. Exit: a test in `toyos-net-shard/tcp/tests/` in which a peer's handshake completes while LS-12's flood runs. - The node counts a wake unspent until an accept arrives, and cannot tell an accept on its way from one that will never be sent: an owner that reads a wake and sends no accept is owed one wake fewer from then on, and the last connection waiting is not announced. That is the node's half of `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`. Exit: that issue's, by a client that cannot spend a wake without its accept arriving, or by an accept that waits in netd and needs no wake. -- A stream its client can see no more, its reader gone or the peer's FIN read through and its client writing no more, is reset once its pipe has given up no byte for 100 s, and a peer address restarts that clock for at most 16 such streams at once (`OWNERLESS_PER_PEER`, `userland/netstack/node/src/streams.rs`); one past them has 100 s from the pass that found it so, until one of the 16 is done and it takes its place. What holds in this stage, from the code: per address, 16, an estimate no measurement set. There is no floor on the rate: each of the 16 is kept for as long as its pipe gives up one byte per 100 s, which for a full 2 MiB pipe is 2,097,152 x 100 s, over six years, and then for its tail in [tcp] (the line below on a client gone with nothing left in its pipe). There is no bound across addresses: addresses are not counted, so the node holds 16 times as many as there are addresses holding one, and where a connect's address is its client's choice, an accepted stream's is the peer's, which on the link answers for as many addresses as it likes. That bound is the node's places (`userland/netstack/node/src/places.rs`), of which every stream holds one until the node lets it go; there is no second one. `issues/netstack-cuts-a-departed-clients-unsent-tail-at-the-ceiling.md` stays open until netd runs on the node: on a host its exit's second clause is met (`a_departed_clients_tail_arrives_whole_while_its_peer_takes_it`) and its first for 16 streams an address. Exit: a test of the node's places in which peers at more addresses than the node has places for each hold 16 such streams, and the node holds no stream past its places; the T14 or a guest shows a peer holding one at a byte per 100 s, and the cut takes a floor on the rate, or the owner rules it needs none; and the move closes that issue with its bench row. +- A stream its client can see no more, its reader gone or the peer's FIN read through and its client writing no more, is reset once its pipe has given up no byte for 100 s, and a peer address restarts that clock for at most 16 such streams at once (`OWNERLESS_PER_PEER`, `userland/netstack/node/src/streams.rs`); one past them has 100 s from the pass that found it so, until one of the 16 is done and it takes its place. What holds in this stage, from the code: per address, 16, an estimate no measurement set. There is no floor on the rate: each of the 16 is kept for as long as its pipe gives up one byte per 100 s, which for a full 2 MiB pipe is 2,097,152 x 100 s, over six years, and then for its tail in [tcp] (the line below on a client gone with nothing left in its pipe). The 16 is no bound across addresses: addresses are not counted, and where a connect's address is its client's choice, an accepted stream's is the peer's, which on the link answers for as many addresses as it likes. The bound across them is the node's places (`userland/netstack/node/src/places.rs`), of which every stream holds one until the node lets it go, and there is no second one: in `peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them` (`userland/netstack/node/tests/listeners.rs`) peers at four addresses bring sixteen connections each to a node with three places, which holds two such streams at a time, each kept alive past its 100 s, and refuses every accept and connect past them. `issues/netstack-cuts-a-departed-clients-unsent-tail-at-the-ceiling.md` stays open until netd runs on the node: on a host its exit's second clause is met (`a_departed_clients_tail_arrives_whole_while_its_peer_takes_it`) and its first for 16 streams an address. Exit: the T14 or a guest shows a peer holding one at a byte per 100 s, and the cut takes a floor on the rate, or the owner rules it needs none; and the move closes that issue with its bench row. - A stream its peer reset is gone from the node at once, so for a client that still holds it `Node::set_nodelay` answers `false` and `Node::nodelay` `None`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers, and libc's `getsockopt` of `TCP_NODELAY`, which asks through `toyos::net::tcp_get_option` (`userland/libc/src/socket.rs`) and is refused where both hosts answer. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. - Every frame and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per frame, as netd's `bridge_piped` is today. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netd is a system call: 1,000 system calls a frame with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one frame makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. Exit: at the move, netd's CPU time per frame on the T14 with 1, 100 and 1,000 idle streams beside one bulk stream is measured, and the change lands if 1,000 idle streams cost more than 12.3 µs a frame over what one costs. The 12.3 µs is the time a full frame takes on a wire of 1 Gbit/s, 1,538 bytes of 8 ns with its preamble and gap: past it the passes alone take longer than the bulk stream's frames take to arrive. It is a threshold by arithmetic and no measurement; by estimate, not measured, 1,000 system calls exceed it. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs index 0df6d315e4e..b2e2e64ad68 100644 --- a/userland/netstack/node/src/listeners.rs +++ b/userland/netstack/node/src/listeners.rs @@ -15,11 +15,17 @@ //! **A listener lives until its owner lets go**: [`Node::close_listener`], or a wake its pipe //! refuses, for whatever reason. [tcp] then resets every connection that still waits, so each //! peer learns at once, and the port is free. +//! +//! **A stream starts with the options its listener has when it is accepted**, which the node +//! writes to [tcp] at the accept. The listener's are the node's to hold and not [tcp]'s: [tcp] +//! hands a listener's options to a connection at its SYN and has no call that reads a +//! connection's back, so a stream seeded from them would answer for an option its connection, +//! begun before the option was set, does not have. use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; -use toyos_net_tcp::Endpoint; +use toyos_net_tcp::{Endpoint, Options}; use toyos_net_wire::{Instant, Port}; use crate::streams::{Pipes, StreamId, WriteRefusal}; @@ -70,6 +76,8 @@ struct Listener { owner: Box, /// Wakes written that no accept has spent. unspent: usize, + /// What a stream accepted here starts with. + options: Options, } #[derive(Default)] @@ -95,28 +103,41 @@ impl Node { let Some((bound, port)) = self.stack.tcp_listen(port, candidate) else { return Err(ListenRefused::InUse) }; let id = ListenerId(self.listeners.next); self.listeners.next = self.listeners.next.saturating_add(1); - self.listeners.live.insert(id, Listener { bound, owner, unspent: 0 }); + self.listeners.live.insert(id, Listener { bound, owner, unspent: 0, options: Options::default() }); Ok((id, port)) } + /// Nagle's algorithm off or on (RFC 9293 §3.7.4) for every stream accepted at `id` from here + /// on; one already accepted keeps its own. `false` is an id that names no listener. + pub fn set_listener_nodelay(&mut self, id: ListenerId, nodelay: bool) -> bool { + let Some(listener) = self.listeners.live.get_mut(&id) else { return false }; + listener.options.nodelay = nodelay; + true + } + + pub fn listener_nodelay(&self, id: ListenerId) -> Option { + self.listeners.live.get(&id).map(|listener| listener.options.nodelay) + } + /// The owner's accept: the oldest connection waiting at `id` becomes a stream on `pipes`, /// and what it already received moves at once. pub fn accept(&mut self, now: Instant, id: ListenerId, pipes: Option) -> Result { let Some(listener) = self.listeners.live.get_mut(&id) else { return Err(AcceptRefused::NoListener) }; listener.unspent = listener.unspent.saturating_sub(1); - let bound = listener.bound; - let answer = self.take(bound, pipes); + let (bound, options) = (listener.bound, listener.options); + let answer = self.take(now, bound, options, pipes); self.bridge(now); answer } - fn take(&mut self, bound: toyos_net_tcp::ListenerId, pipes: Option) -> Result { + fn take(&mut self, now: Instant, bound: toyos_net_tcp::ListenerId, options: Options, pipes: Option) -> Result { let Some(pipes) = pipes else { return Err(AcceptRefused::NoPipes) }; if self.room() == 0 { return Err(AcceptRefused::Full); } let Some((conn, tuple)) = self.stack.tcp_accept(bound) else { return Err(AcceptRefused::Nothing) }; - let id = self.streams.accepted(conn, tuple.remote.addr, pipes); + // The peer's address is what `streams` counts a stream its client can see no more by. + let id = self.streams.accepted(now, &mut self.stack, conn, tuple.remote.addr, options, pipes); Ok(Accepted { id, remote: tuple.remote, local: tuple.local.port }) } diff --git a/userland/netstack/node/src/places.rs b/userland/netstack/node/src/places.rs index 33bae32765f..94f51642107 100644 --- a/userland/netstack/node/src/places.rs +++ b/userland/netstack/node/src/places.rs @@ -2,10 +2,12 @@ //! connect or its accept until the node lets it go, a listener from its listen until it is //! closed, and a connection the node closed holds the place its stream had until [tcp] has //! finished it, because it is still two buffers of [tcp]'s and its peer decides for how long. -//! A departed client's connection that its peer keeps alive holds its place through both: it is -//! a stream while its pipe holds bytes (`streams` lets one peer address keep at most -//! `OWNERLESS_PER_PEER` of them alive), and [tcp]'s to finish after. With no place left a -//! connect, a listen and an accept are refused with nothing made. +//! A stream its client can see no more, which its peer keeps alive, holds its place through +//! both: it is a stream while its pipe holds bytes, and [tcp]'s to finish after. `streams` lets +//! one peer address keep at most `OWNERLESS_PER_PEER` of them alive and counts no addresses, so +//! that is each address's share of the cut's clock and no bound on the node: peers at however +//! many addresses hold no more of them than there are places. With no place left a connect, a +//! listen and an accept are refused with nothing made. //! //! The number is the shell's: each stream is two of its client's pipes kept alive and watched, //! each listener one. A new node has none. diff --git a/userland/netstack/node/src/streams.rs b/userland/netstack/node/src/streams.rs index 2ab1783cbd4..091d3fb2384 100644 --- a/userland/netstack/node/src/streams.rs +++ b/userland/netstack/node/src/streams.rs @@ -26,10 +26,10 @@ //! address restarts that clock for at most [`OWNERLESS_PER_PEER`] streams at once; any other has //! [`OWNERLESS_LIFE`] from the pass that first found it so, whatever it gives up, until it //! becomes one of them: a pass over the streams that begins with fewer makes it one, the oldest -//! stream first. The count is an address's and addresses are not counted, so how many such -//! streams the node holds in all is not bounded here, nor is how slowly a peer may take. A -//! client that still holds its reading end, or may still write, is never timed here: what it -//! cannot send is [tcp]'s to give up on. +//! stream first. The count is an address's and addresses are not counted: how many such streams +//! the node holds in all is `places`' to bound, of which each holds one, and how slowly a peer +//! may take is bounded nowhere. A client that still holds its reading end, or may still write, +//! is never timed here: what it cannot send is [tcp]'s to give up on. //! //! Every call that can move a stream ends in a pass: a frame, a deadline, and each call here. A //! transmit opportunity can only fail a connect, whose next hop it found to answer nobody, and @@ -170,8 +170,9 @@ struct Stream { } impl Stream { - /// An established connection on its client's pipes. - fn established(conn: ConnId, remote: Ipv4Addr, pipes: Pipes) -> Self { + /// An established connection on its client's pipes. `options` are the ones [tcp] holds for + /// it. + fn established(conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> Self { Self { conn, remote, @@ -183,7 +184,7 @@ impl Stream { extended: false, held: false, room: false, - options: Options::default(), + options, } } @@ -318,9 +319,10 @@ impl Streams { } /// Holds a connection a listener's owner accepted, established, on the pipes its accept - /// handed over. - pub(crate) fn accepted(&mut self, conn: ConnId, remote: Ipv4Addr, pipes: Pipes) -> StreamId { - self.hold(Stream::established(conn, remote, pipes)) + /// handed over, and writes it its listener's `options`. + pub(crate) fn accepted(&mut self, now: Instant, stack: &mut Stack, conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> StreamId { + stack.tcp_set_options(now, conn, options); + self.hold(Stream::established(conn, remote, options, pipes)) } pub(crate) fn next_deadline(&self) -> Option { @@ -337,7 +339,8 @@ impl Node { } let conn = self.stack.tcp_connect(now, remote).map_err(ConnectRefused::Stack)?; let deadline = timeout.map(|within| now.after(within)); - Ok(self.streams.hold(Stream { connecting: true, deadline, ..Stream::established(conn, remote.addr, pipes) })) + // An active open has [tcp]'s defaults until `set_nodelay`. + Ok(self.streams.hold(Stream { connecting: true, deadline, ..Stream::established(conn, remote.addr, Options::default(), pipes) })) } /// One pass over every stream: netstack calls it when a pipe it watches is ready. diff --git a/userland/netstack/node/tests/listeners.rs b/userland/netstack/node/tests/listeners.rs index 8886859a949..be0ecfaa135 100644 --- a/userland/netstack/node/tests/listeners.rs +++ b/userland/netstack/node/tests/listeners.rs @@ -1,6 +1,7 @@ -//! Listeners and the node's places, against peers the test scripts at 192.0.2.2 and pipe ends it -//! fakes. No scenario ids: the specifications' listening scenarios are `toyos-net-tcp`'s; these -//! are what the node does between [tcp]'s queues, an owner's wakes and its accepts. +//! Listeners and the node's places, against peers the test scripts, at 192.0.2.2 unless a test +//! puts one elsewhere, and pipe ends it fakes. No scenario ids: the specifications' listening +//! scenarios are `toyos-net-tcp`'s; these are what the node does between [tcp]'s queues, an +//! owner's wakes and its accepts. //! //! What is not ours: every segment the node emits is read by `etherparse` as it leaves //! ([`Net::hears`]), its IPv4 and TCP checksums and lengths that crate's sums, and every segment @@ -17,22 +18,49 @@ use std::net::Ipv4Addr; use std::rc::Rc; use std::time::Duration; -use common::{arp, terms, Wire, A, MAC, MAC_B, MAC_R, R}; +use common::{arp, terms, Wire, A, MAC, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, StreamId, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; use toyos_net_tcp::{limits, Counter, Endpoint}; use toyos_net_wire::{Instant, Port}; +/// The addresses peers are at. const B: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); +const C: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 3); +const D: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 4); +const E: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 5); + +/// The MAC the frames of a peer at `addr` come from: 192.0.2.2's is `common::MAC_B`. +fn mac(addr: Ipv4Addr) -> [u8; 6] { + let [.., last] = addr.octets(); + [2, 0, 0, 0, 0, 9 + last] +} + +/// A peer: its address and its port. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Peer { + addr: Ipv4Addr, + port: u16, +} + +impl Peer { + fn endpoint(self) -> Endpoint { + Endpoint { addr: self.addr, port: Port::new(self.port).unwrap() } + } +} + /// The port the tests listen on, and another. const SSH: u16 = 22; const TELNET: u16 = 23; -/// The peers' ports. -const P1: u16 = 40_001; -const P2: u16 = 40_002; -const P3: u16 = 40_003; +const P1: Peer = Peer { addr: B, port: 40_001 }; +const P2: Peer = Peer { addr: B, port: 40_002 }; +const P3: Peer = Peer { addr: B, port: 40_003 }; +/// Where the tests connect to: nothing answers there. +const WEB: Peer = Peer { addr: B, port: 80 }; /// Every peer's initial sequence number. const ISS: u32 = 5000; +/// [tcp]'s send buffer, as `common::Wire` configures it. +const SEND_BUFFER: usize = 65_535; const SYN: u8 = 1; const FIN: u8 = 2; @@ -135,9 +163,9 @@ fn client() -> (Client, Pipes) { /// A segment the node emitted, as `etherparse` read it. #[derive(Clone, Debug)] struct Segment { - /// The node's port, and the peer's. + /// The node's port. from: u16, - to: u16, + to: Peer, seq: u32, ack: Option, syn: bool, @@ -146,10 +174,10 @@ struct Segment { text: Vec, } -/// A segment from the peer's port `from` to the node's port `to` in its frame, built by -/// `etherparse`: window 65,535, an MSS option on a SYN, PSH with text. -fn frame(from: u16, to: u16, seq: u32, ack: Option, flags: u8, text: &[u8]) -> Vec { - let mut step = PacketBuilder::ethernet2(MAC_B, MAC).ipv4(B.octets(), A.octets(), 64).tcp(from, to, seq, 65_535); +/// A segment from the peer to the node's port `to` in its frame, built by `etherparse`: window +/// 65,535, an MSS option on a SYN, PSH with text. +fn frame(from: Peer, to: u16, seq: u32, ack: Option, flags: u8, text: &[u8]) -> Vec { + let mut step = PacketBuilder::ethernet2(mac(from.addr), MAC).ipv4(from.addr.octets(), A.octets(), 64).tcp(from.port, to, seq, 65_535); if flags & SYN != 0 { step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(); } @@ -187,6 +215,8 @@ struct Net { node: Node, now: Instant, draws: u32, + /// The addresses a peer is at: 192.0.2.2, and each one a SYN has come from. + at: BTreeSet, /// Every TCP segment the node emitted, in order. heard: Vec, } @@ -195,11 +225,11 @@ impl Net { /// The node holding its lease of 192.0.2.1/24 for an hour, with `common::PLACES` places. fn new() -> Self { let Wire { node, now, .. } = Wire::leased(&terms(3_600, Some(R))); - Self { node, now, draws: 0x7e00_0000, heard: Vec::new() } + Self { node, now, draws: 0x7e00_0000, at: BTreeSet::from([B]), heard: Vec::new() } } /// Offers the node all the credit it wants until it sends nothing more, its ARP requests - /// for the peers' address and the router's answered. + /// for the peers' addresses and the router's answered. fn pump(&mut self) { for _ in 0..100_000 { let mut frames = Vec::new(); @@ -218,7 +248,8 @@ impl Net { /// The outside reading of a frame the node emitted, and the ARP answer it gets if it asks /// for one. Ethernet II from the node's MAC; then ARP, a datagram of the DHCP client's, or - /// a TCP segment to a peer, with the lengths and checksums `etherparse` computes. + /// a TCP segment to an address a peer is at, with the lengths and checksums `etherparse` + /// computes. A segment to any other address is the node's mistake. fn hears(&mut self, frame: &[u8]) -> Option> { let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; @@ -227,7 +258,7 @@ impl Net { (Some(NetSlice::Arp(asked)), None) => { let asks = asked.operation() == ArpOperation::REQUEST && v4(asked.sender_protocol_addr()) == A; match v4(asked.target_protocol_addr()) { - target if asks && target == B => Some(arp(MAC, false, MAC_B, B, A)), + target if asks && self.at.contains(&target) => Some(arp(MAC, false, mac(target), target, A)), target if asks && target == R => Some(arp(MAC, false, MAC_R, R, A)), _ => None, } @@ -235,14 +266,16 @@ impl Net { (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(_))) => None, (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) => { let header = ip.header(); - assert_eq!(ethernet.destination(), MAC_B, "to the peers' MAC"); + let to = header.destination_addr(); + assert!(self.at.contains(&to), "a segment to {to}, where no peer is"); + assert_eq!(ethernet.destination(), mac(to), "to the peer's MAC"); assert_eq!(header.header_checksum(), header.to_header().calc_header_checksum(), "the IPv4 header checksum"); assert_eq!(usize::from(header.total_len()), 20 + tcp.slice().len(), "IPv4's length is the segment's"); - assert_eq!((header.source_addr(), header.destination_addr()), (A, B)); + assert_eq!(header.source_addr(), A); assert_eq!(tcp.checksum(), tcp.calc_checksum_ipv4(header.source(), header.destination()).unwrap(), "the TCP checksum"); self.heard.push(Segment { from: tcp.source_port(), - to: tcp.destination_port(), + to: Peer { addr: to, port: tcp.destination_port() }, seq: tcp.sequence_number(), ack: tcp.ack().then(|| tcp.acknowledgment_number()), syn: tcp.syn(), @@ -280,61 +313,76 @@ impl Net { panic!("100,000 deadlines without the clock passing {span:?}"); } - /// The segments the node sent to the peer at `port`. - fn to(&self, port: u16) -> Vec { - self.heard.iter().filter(|segment| segment.to == port).cloned().collect() + /// The segments the node sent to `peer`. + fn to(&self, peer: Peer) -> Vec { + self.heard.iter().filter(|segment| segment.to == peer).cloned().collect() } - /// The last segment the node sent to the peer at `port`. - fn last(&self, port: u16) -> Segment { - self.to(port).pop().expect("a segment to the peer") + /// The last segment the node sent to `peer`. + fn last(&self, peer: Peer) -> Segment { + self.to(peer).pop().expect("a segment to the peer") } - /// The node's initial sequence number toward the peer at `port`: its SYN-ACK's. - fn iss(&self, port: u16) -> u32 { - self.to(port).iter().rev().find(|segment| segment.syn).expect("a SYN-ACK to the peer").seq + /// The text of each segment the node sent to `peer` that carried any, in order. + fn texts(&self, peer: Peer) -> Vec> { + self.to(peer).into_iter().map(|segment| segment.text).filter(|text| !text.is_empty()).collect() + } + + /// The node's initial sequence number toward `peer`: its SYN-ACK's. + fn iss(&self, peer: Peer) -> u32 { + self.to(peer).iter().rev().find(|segment| segment.syn).expect("a SYN-ACK to the peer").seq } /// The peer's SYN to the node's port `to`. - fn syn_to(&mut self, peer: u16, to: u16) { + fn syn_to(&mut self, peer: Peer, to: u16) { + self.at.insert(peer.addr); self.deliver(&frame(peer, to, ISS, None, SYN, &[])); } - fn syn(&mut self, peer: u16) { + fn syn(&mut self, peer: Peer) { self.syn_to(peer, SSH); } /// The peer's ACK of the node's SYN-ACK (RFC 9293 §3.5, figure 6, line 4). - fn finish(&mut self, peer: u16) { + fn finish(&mut self, peer: Peer) { let ack = self.iss(peer).wrapping_add(1); self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), 0, &[])); } - fn handshake(&mut self, peer: u16) { + fn handshake(&mut self, peer: Peer) { self.syn(peer); self.finish(peer); } /// The peer's first text. - fn text(&mut self, peer: u16, text: &[u8]) { + fn text(&mut self, peer: Peer, text: &[u8]) { let ack = self.iss(peer).wrapping_add(1); self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), 0, text)); } /// The peer's reset at exactly the sequence number the node expects (RFC 9293 §3.10.7.4, /// first check), having sent no text. - fn rst(&mut self, peer: u16) { + fn rst(&mut self, peer: Peer) { let ack = self.iss(peer).wrapping_add(1); self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), RST, &[])); } /// The peer's FIN, acknowledging the node's (RFC 9293 §3.6, case 1), neither having sent /// text. - fn fin(&mut self, peer: u16) { + fn fin(&mut self, peer: Peer) { let ack = self.iss(peer).wrapping_add(2); self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), FIN, &[])); } + /// The peer's acknowledgment of the node's first `bytes` bytes of text, or of as many of + /// them as it has heard, having sent nothing since its handshake. + fn takes(&mut self, peer: Peer, bytes: u32) { + let first = self.iss(peer).wrapping_add(1); + let ends = self.to(peer).into_iter().filter(|segment| !segment.text.is_empty()); + let heard = ends.map(|segment| segment.seq.wrapping_sub(first).wrapping_add(u32::try_from(segment.text.len()).unwrap())).max().expect("text to the peer"); + self.deliver(&frame(peer, SSH, ISS + 1, Some(first.wrapping_add(bytes.min(heard))), 0, &[])); + } + fn listen(&mut self, port: u16) -> (ListenerId, Owner) { let owner = Owner::default(); let (id, bound) = self.node.listen(Port::new(port), Box::new(WakeEnd(owner.clone())), draw(&mut self.draws)).expect("a free port and a place"); @@ -350,20 +398,40 @@ impl Net { answer.map(|accepted| (accepted, client)) } - /// An accept that takes the connection of the peer at `port`. - fn accepts(&mut self, id: ListenerId, port: u16) -> (StreamId, Client) { + /// An accept that takes `peer`'s connection. + fn accepts(&mut self, id: ListenerId, peer: Peer) -> (StreamId, Client) { let (accepted, client) = self.accept(id).expect("a connection and a place"); - assert_eq!((accepted.remote, accepted.local), (Endpoint { addr: B, port: Port::new(port).unwrap() }, Port::new(SSH).unwrap())); + assert_eq!((accepted.remote, accepted.local), (peer.endpoint(), Port::new(SSH).unwrap())); (accepted.id, client) } - /// A connect to the peers' port 80, which nothing answers. + /// A connect to [`WEB`]. fn connect(&mut self) -> (Result, Client) { let (client, pipes) = client(); - let answer = self.node.connect(self.now, Endpoint { addr: B, port: Port::new(80).unwrap() }, None, pipes); + let answer = self.node.connect(self.now, WEB.endpoint(), None, pipes); self.pump(); (answer, client) } + + /// The client writes `text`, and the pass netstack runs when its pipe has bytes. + fn says(&mut self, client: &Client, text: &[u8]) { + client.borrow_mut().outbox.extend(text); + self.node.bridge(self.now); + self.pump(); + } + + /// The stream's client writes 100,000 bytes and lets go of it: [tcp] has room for + /// [`SEND_BUFFER`] of them, and the rest stays in its pipe. + fn departs(&mut self, id: StreamId, client: &Client) { + client.borrow_mut().outbox.extend(vec![7u8; 100_000]); + self.node.close(self.now, id); + self.pump(); + assert_eq!(client.borrow().outbox.len(), 100_000 - SEND_BUFFER); + } + + fn events(&mut self) -> Vec { + self.node.drain_stream_events().collect() + } } /// A node with `places` places. @@ -397,7 +465,7 @@ fn a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends() { let (client, pipes) = client(); let accepted = net.node.accept(net.now, id, Some(pipes)).unwrap(); - assert_eq!((accepted.remote, accepted.local), (Endpoint { addr: B, port: Port::new(P1).unwrap() }, Port::new(SSH).unwrap())); + assert_eq!((accepted.remote, accepted.local), (P1.endpoint(), Port::new(SSH).unwrap())); assert_eq!(client.borrow().inbox, b"hello", "what arrived before the accept moves in it"); assert_eq!((net.node.streams(), net.node.listeners(), net.node.held()), (1, 1, 2)); client.borrow_mut().outbox.extend(b"welcome"); @@ -591,7 +659,7 @@ fn a_connect_past_the_places_is_refused_and_sends_nothing() { let (third, client) = net.connect(); assert_eq!(third, Err(ConnectRefused::Full), "a connect not yet answered holds a place"); assert_eq!((client.borrow().dropped, net.node.held()), (2, 2)); - let sources: BTreeSet = net.to(80).iter().map(|segment| segment.from).collect(); + let sources: BTreeSet = net.to(WEB).iter().map(|segment| segment.from).collect(); assert_eq!(sources.len(), 2, "no SYN left for the connect refused: {:?}", net.heard); net.node.close(net.now, first.unwrap()); @@ -684,6 +752,161 @@ fn a_listen_on_a_taken_port_is_refused_and_a_drawn_port_listens() { assert!(synack.syn && synack.from == port.get(), "{synack:?}"); } +// ---- the streams an accept makes ---- + +// RFC 9293 §3.7.4: with text unacknowledged a short segment waits, and with Nagle's algorithm +// off it leaves. No peer here acknowledges any text. +#[test] +fn a_stream_starts_with_the_options_its_listener_has_when_it_is_accepted() { + let mut net = Net::new(); + let (id, _owner) = net.listen(SSH); + assert_eq!(net.node.listener_nodelay(id), Some(false)); + // One handshake ends before the option is set and one after; both are accepted after. + net.handshake(P1); + assert!(net.node.set_listener_nodelay(id, true)); + net.handshake(P2); + assert_eq!(net.node.listener_nodelay(id), Some(true)); + let (first, a) = net.accepts(id, P1); + let (second, b) = net.accepts(id, P2); + for (stream, client, peer) in [(first, &a, P1), (second, &b, P2)] { + assert_eq!(net.node.nodelay(stream), Some(true), "{peer:?}"); + net.says(client, b"a"); + net.says(client, b"b"); + assert_eq!(net.texts(peer), [b"a".to_vec(), b"b".to_vec()], "{peer:?}: the second write waits for no acknowledgment"); + } + + // An accepted stream's options are its own from then on, and the listener's its own. + assert!(net.node.set_nodelay(net.now, second, false)); + net.says(&b, b"c"); + assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the third write waits for the first's acknowledgment"); + assert!(net.node.set_listener_nodelay(id, false)); + assert_eq!((net.node.nodelay(first), net.node.nodelay(second), net.node.listener_nodelay(id)), (Some(true), Some(false), Some(false))); + net.says(&a, b"c"); + assert_eq!(net.texts(P1), [b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]); + + net.handshake(P3); + let (third, c) = net.accepts(id, P3); + assert_eq!(net.node.nodelay(third), Some(false)); + net.says(&c, b"a"); + net.says(&c, b"b"); + assert_eq!(net.texts(P3), [b"a".to_vec()]); + + assert!(net.node.close_listener(net.now, id)); + assert_eq!((net.node.set_listener_nodelay(id, true), net.node.listener_nodelay(id)), (false, None)); +} + +// `streams` counts a stream its client can see no more by its peer's address, and lets an +// address keep sixteen alive at once. Seventeen are accepted from 192.0.2.2 and one from +// 192.0.2.3, each client leaves bytes behind, and each peer takes a segment of them fifty +// seconds on: only the seventeenth from 192.0.2.2 has no more than its 100 seconds. +#[test] +fn an_accepted_stream_is_one_of_its_peers_addresss_sixteen() { + let mut net = Net::new(); + let (id, _owner) = net.listen(SSH); + let peers: Vec = (0..17).map(|nth| Peer { addr: B, port: 41_000 + nth }).chain([Peer { addr: C, port: 41_000 }]).collect(); + let mut streams = Vec::new(); + for peer in &peers { + net.handshake(*peer); + let (stream, client) = net.accepts(id, *peer); + net.departs(stream, &client); + streams.push(stream); + } + assert_eq!((net.node.streams(), net.events()), (18, vec![])); + + net.run(Duration::from_secs(50)); + for peer in &peers { + net.takes(*peer, 1_460); + } + net.run(Duration::from_secs(51)); + assert_eq!((net.node.streams(), net.events()), (17, vec![StreamEvent::Cut { id: streams[16] }])); + let reset: Vec = net.heard.iter().filter(|segment| segment.rst).map(|segment| segment.to).collect(); + assert_eq!(reset, [peers[16]]); +} + +// `streams` cuts a stream whose client still holds it, once the peer's FIN ended its reading +// and it shut its writing down: what that client asks of the stream afterwards names nothing, +// and not the stream that took its place either, since no id is used twice. +#[test] +fn a_request_for_a_stream_that_was_cut_names_nothing() { + let mut net = with_places(2); + let (id, owner) = net.listen(SSH); + net.handshake(P1); + let (cut, client) = net.accepts(id, P1); + net.handshake(P2); + let ack = net.iss(P1).wrapping_add(1); + net.deliver(&frame(P1, SSH, ISS + 1, Some(ack), FIN, &[])); + client.borrow_mut().outbox.extend(vec![7u8; 100_000]); + assert!(net.node.shutdown_write(net.now, cut)); + net.pump(); + assert_eq!((net.node.streams(), client.borrow().dropped, wakes(&owner)), (1, 1, 1), "the client reads the end and may still close"); + + net.run(Duration::from_secs(101)); + assert_eq!(net.events(), [StreamEvent::Cut { id: cut }]); + assert!(net.last(P1).rst); + assert_eq!((net.node.streams(), net.node.held(), client.borrow().dropped, wakes(&owner)), (0, 1, 2, 2)); + let (next, other) = net.accepts(id, P2); + assert_ne!(next, cut); + + let said = net.heard.len(); + net.node.close(net.now, cut); + assert!(!net.node.shutdown_write(net.now, cut)); + assert!(!net.node.set_nodelay(net.now, cut, true)); + assert_eq!(net.node.nodelay(cut), None); + net.node.pipe_gone(net.now, cut, PipeEnd::FromClient); + net.node.pipe_broken(net.now, cut, PipeEnd::FromClient); + net.pump(); + assert_eq!((net.node.streams(), net.node.held(), other.borrow().dropped, wakes(&owner)), (1, 2, 0, 2), "the stream in its place stands"); + assert_eq!((net.node.nodelay(next), net.heard.len(), net.events()), (Some(false), said, vec![])); +} + +// The track's exit for the bound across peer addresses. `streams` lets each address keep +// sixteen streams their clients can see no more alive and counts no addresses, so the places +// are what bounds the node. Four addresses, more than the node has places for, bring sixteen +// connections each, and every peer whose connection is accepted keeps its departed client's +// bytes coming for 250 seconds: the node holds two streams at a time, announces no connection +// it has no place for, and refuses every accept and connect past them. +#[test] +fn peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them() { + let mut net = with_places(3); + let (id, owner) = net.listen(SSH); + for nth in 0..16 { + for addr in [B, C, D, E] { + net.handshake(Peer { addr, port: 42_000 + nth }); + } + } + assert_eq!((wakes(&owner), net.node.streams(), net.node.held()), (2, 0, 1), "sixty-four wait, and there are places for two"); + + for (round, pair) in [[B, C], [D, E]].into_iter().enumerate() { + let peers = pair.map(|addr| Peer { addr, port: 42_000 }); + let held = peers.map(|peer| { + let (stream, client) = net.accepts(id, peer); + net.departs(stream, &client); + (stream, client) + }); + let full = |net: &mut Net| { + assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Full, "round {round}"); + assert_eq!(net.connect().0, Err(ConnectRefused::Full), "round {round}"); + assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (2, 3, 2 + 2 * round)); + }; + full(&mut net); + for step in 1..=5 { + net.run(Duration::from_secs(50)); + for peer in peers { + net.takes(peer, 1_460 * step); + } + full(&mut net); + } + assert!(held.iter().all(|(_, client)| client.borrow().outbox.len() < 100_000 - SEND_BUFFER), "round {round}: each pipe gave bytes up"); + + // The peers take no more: both streams are cut, and two of the connections that wait + // are announced. + net.run(Duration::from_secs(101)); + let cut: Vec = held.iter().map(|(stream, _)| StreamEvent::Cut { id: *stream }).collect(); + assert_eq!(net.events(), cut, "round {round}"); + assert_eq!((net.node.streams(), net.node.held(), wakes(&owner)), (0, 1, 4 + 2 * round)); + } +} + // ---- the wire is not trusted ---- #[test] From 643584d4b00a32c05b145ba5a7eb57641a867d0e Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 22:05:31 +0200 Subject: [PATCH 4/7] Review round 1 of #777: a listen carries its address, a listener's option is the host's measured rule, a datagram socket holds a place BLOCKER 2. Node::listen takes the address the program bound and hands it to [tcp]. One the interface does not hold usable is refused (ListenRefused::NotLocal) in Shard::listen by [ip]'s is_assigned, the predicate [udp]'s bind refuses by, so both socket kinds have one rule. BLOCKER 3. Measured on macOS, by libc over loopback and by tests/host.rs through socket2: TCP_NODELAY set on a listening socket before a connection began is on the accepted socket; set after the connection was established and before the accept, it is not. The node gave it to both. It now does what the host does by [tcp]'s own inheritance (LS-10): set_listener_nodelay writes [tcp]'s listener, a connection takes the options at its SYN, and the accept seeds the stream with what Tcp::options, a new read, says the connection has. The write at the accept and its parameters are gone. Linux is not read: the host check's first run of tests/host.rs is that reading, and the new issue says so. BLOCKER 4. The loop over wake_each has its test: a listener ended for a refused wake gives its place to another in the same pass, asserted before any transmit opportunity. A datagram socket holds a place from its bind to its close, and a bind with none left is refused ResourceExhausted with nothing made. A connection that waits to be accepted holds none: places.rs and the track say why, and what the shell must price a listener at. NOTEs: the doc comment `orphans` took from `eligible` is back; the places line of the track names its own exit; the outside reading of a TCP segment is declared once, in tests/common, and read by both test files; issues/netstack-datagram-sockets-and-listeners-have-no-bound.md says what is left. socket2 is a new dev-dependency of toyos-net-node, already in the lock through the workspace's patch: it sets and reads a socket option std has no call for on a listener, on every host, with no unsafe. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- Cargo.lock | 1 + ...cepted-streams-is-answered-from-a-guess.md | 45 +++++ ...ram-sockets-and-listeners-have-no-bound.md | 11 ++ issues/toyos-has-its-own-network-stack.md | 7 +- toyos-net-shard/src/lib.rs | 31 ++- toyos-net-shard/tcp/src/stack.rs | 8 +- toyos-net-shard/tcp/tests/held.rs | 25 ++- userland/netstack/node/Cargo.toml | 5 +- userland/netstack/node/src/datagram.rs | 18 +- userland/netstack/node/src/lease/tcp.rs | 27 +-- userland/netstack/node/src/lib.rs | 11 +- userland/netstack/node/src/listeners.rs | 52 +++-- userland/netstack/node/src/places.rs | 36 ++-- userland/netstack/node/src/streams.rs | 5 +- userland/netstack/node/tests/common/mod.rs | 46 +++++ userland/netstack/node/tests/host.rs | 34 ++++ userland/netstack/node/tests/listeners.rs | 177 ++++++++++++------ userland/netstack/node/tests/streams.rs | 69 ++----- 18 files changed, 430 insertions(+), 178 deletions(-) create mode 100644 issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md create mode 100644 userland/netstack/node/tests/host.rs diff --git a/Cargo.lock b/Cargo.lock index 8fe763bf323..1e4eab6f3e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5918,6 +5918,7 @@ name = "toyos-net-node" version = "0.1.0" dependencies = [ "etherparse", + "socket2", "toyos-dhcp", "toyos-mdns", "toyos-net-ip", diff --git a/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md b/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md new file mode 100644 index 00000000000..04244b4194d --- /dev/null +++ b/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md @@ -0,0 +1,45 @@ +--- +status: open +kind: defect +opened: 2026-10-08 +--- + +# A listening socket's TCP_NODELAY is refused, and an accepted stream's is answered from a guess + +A host takes `TCP_NODELAY` on a listening socket and gives it to the +connections that begin afterwards. Read on macOS 27 on arm64, by libc's +`setsockopt` and `getsockopt` over loopback: set before the connection began, +the accepted socket has the option; set after the connection was established +and before the accept, it does not. `userland/netstack/node/tests/host.rs` +asks the same of every host the suite runs on. + +ToyOS as it ships does neither half: + +- netstack answers `TcpSetOption` and `TcpGetOption` `ERR_NOT_CONNECTED` for + any id that is not a stream (`handle_tcp_set_option`, + `userland/netstack/src/main.rs`), and libc's `setsockopt` sends a listening + socket's id there (`userland/libc/src/socket.rs`), so a C program that sets + the option on its listener is refused. +- std builds every `TcpStream`, an accepted one included, with + `nodelay: false` and answers `nodelay()` from that field without asking + netstack (`sdk/std/sys/net/connection.rs`). Nothing can make that false + today, since no accepted connection has the option. Once a listener's + option reaches the connections it accepts, it is false for each of them. + +The node (`userland/netstack/node/src/listeners.rs`) has the calls, +`Node::set_listener_nodelay` and `Node::listener_nodelay`, and gives an +accepted stream what its connection took from its listener; nothing ships it. + +Not read: Linux. No Linux host was at hand when this was filed. The `host` +check runs `tests/host.rs` on Linux, and its first run on a pull request that +carries the file is that reading. + +**Exit condition**: `tests/host.rs` is green in a `host` check's log, which +is the Linux reading; and on ToyOS a program that sets `TCP_NODELAY` on a +listening socket and accepts one connection begun before and one begun after +reads the option on each as that test has a host answer, by libc's +`getsockopt` and by std's `nodelay()`. + +**Owner**: whoever holds `issues/toyos-has-its-own-network-stack.md`; the +Linux reading is the orchestrator's, at the first `host` run of the pull +request that adds `tests/host.rs`. diff --git a/issues/netstack-datagram-sockets-and-listeners-have-no-bound.md b/issues/netstack-datagram-sockets-and-listeners-have-no-bound.md index b0b895bac95..948d63ed1aa 100644 --- a/issues/netstack-datagram-sockets-and-listeners-have-no-bound.md +++ b/issues/netstack-datagram-sockets-and-listeners-have-no-bound.md @@ -18,6 +18,17 @@ the two that exist. Read from the code, not measured. +On the node (`userland/netstack/node/src/places.rs`) a datagram socket and a +listener each hold a place, and a bind or a listen with none left is refused +with nothing made: `a_datagram_socket_holds_a_place_and_a_bind_without_one_makes_nothing` +and `a_listener_holds_a_place_and_a_listen_without_one_makes_nothing` +(`userland/netstack/node/tests/listeners.rs`). What is left: netstack as it +ships is the code above until it runs on the node; the node's places are one +number for every client, so its tests bind past the bound and see the refusal +but have no second client whose bind is answered, which the track records +with its own exit; and the word the pipe ABI answers a refused listen in is +the move's to map. + **Exit condition**: a bind past a stated bound is refused `ERR_RESOURCE_EXHAUSTED` with nothing made, and a test binds past it and sees the refusal and another client's bind answered. diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index 7b32e922812..a5bcc75dbe1 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -17,7 +17,7 @@ netd runs smoltcp. Its replacement is ToyOS's own stack, built clean-room: reade - Then multi-core, netring (blocked on the owner's ABI ruling), TCP and IP hardening, IPv6, offloads, soak. - This track owns the T14's outbound rows, the machine reaching its router and the internet on the I219, judged from the stick: they arrive on this stack and not on smoltcp (owner: "no smoltcp."), and until they do no T14 row reads the wired card (`issues/the-host-cannot-reach-the-t14-while-it-runs-toyos.md`). -Stage 5 lands in slices, the orchestrator's cut under the owner's words "i want smoltcp out as fast as possible": netd's decisions go into `toyos-net-node` (`userland/netstack/node`), pure and host-tested and shipped in nothing, and then one change moves netd onto it and deletes smoltcp. In the tree: the node, its DHCP lease, its clients' datagram sockets, its mDNS name, streams, listeners and the bound on streams and listeners (`userland/netstack/node/src/places.rs`). Still to build on it: the resolver; then the move. +Stage 5 lands in slices, the orchestrator's cut under the owner's words "i want smoltcp out as fast as possible": netd's decisions go into `toyos-net-node` (`userland/netstack/node`), pure and host-tested and shipped in nothing, and then one change moves netd onto it and deletes smoltcp. In the tree: the node, its DHCP lease, its clients' datagram sockets, its mDNS name, streams, listeners and the one bound on streams, listeners and datagram sockets (`userland/netstack/node/src/places.rs`). Still to build on it: the resolver; then the move. What the node does not yet meet: @@ -25,14 +25,15 @@ What the node does not yet meet: - It counts a DHCP message [udp] refused as `node.dhcp-unsent`, whatever the rule; the `dhcp.renew-unroutable` scenario owed above is not written. Exit: that scenario names the counter, or the node counts the renewal apart. - With the link down the DHCP client keeps its timers: every 4 to 64 s the node has a deadline, builds a DISCOVER that [udp] refuses, and counts it in `dhcp.tx.discover` and `node.dhcp-unsent`, from `Node::new` on. Exit: `toyos-dhcp`'s client is told the link went down and waits for it, and the node's first DISCOVER is the one that leaves. - No second TCP has been on the far end of a stream: the peer of `userland/netstack/node/tests/streams.rs` is the tests' own script, which acknowledges what arrives in order and loses, reorders and repeats nothing; what is not ours there is `etherparse`, which reads every segment the node emits and builds every one it receives. Exit: netd runs on the node against the host kernel's TCP through slirp in a guest. -- The node's places are one number for every client: a program that connects or listens in a loop takes them all, and so does a peer that keeps the connections [tcp] is finishing alive, each of which holds its stream's place (the line below on a client gone with nothing left in its pipe). The node has no word for the program behind a request, as `issues/netstack-lookup-slots-have-no-per-client-share.md` records of netd. Exit: that issue's. +- The node's places are one number for every client: a program that connects, listens or binds in a loop takes them all, and so does a peer that keeps the connections [tcp] is finishing alive, each of which holds its stream's place (the line below on a client gone with nothing left in its pipe). The node has no word for the program behind a request, as `issues/netstack-lookup-slots-have-no-per-client-share.md` records of netd. Exit: a test of the node in which one program at its share of the places leaves another's connect, listen, accept and bind answered. +- A connection that waits to be accepted holds no place: a listener's one place stands for [tcp]'s `limits::LISTEN_READY` of them, each with at most one receive buffer of text taken when its first byte arrives, 128 x 65,535 bytes or 8.4 MB a listener as netd configures [tcp]. Counting each as a place would let any peer of any listener take every place by handshakes alone and refuse every client its connect, listen and bind; as built a peer fills the queue of the listener it reaches. So the places bound the node's memory only if the shell prices a listener at that product. Exit: the move derives netd's places from a memory figure in which a listener costs its 128 receive buffers, and its body says the figure. - A listener's handshakes in progress are [tcp]'s `limits::LISTEN_PENDING`, each held until `limits::SYNACK_GIVE_UP`, and a SYN past them is dropped: 256 SYNs a minute from addresses that never answer shut a port to every other peer for as long as they keep coming. [tcp] has no SYN cookie and lets no older handshake go for a newer one. Exit: a test in `toyos-net-shard/tcp/tests/` in which a peer's handshake completes while LS-12's flood runs. - The node counts a wake unspent until an accept arrives, and cannot tell an accept on its way from one that will never be sent: an owner that reads a wake and sends no accept is owed one wake fewer from then on, and the last connection waiting is not announced. That is the node's half of `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`. Exit: that issue's, by a client that cannot spend a wake without its accept arriving, or by an accept that waits in netd and needs no wake. - A stream its client can see no more, its reader gone or the peer's FIN read through and its client writing no more, is reset once its pipe has given up no byte for 100 s, and a peer address restarts that clock for at most 16 such streams at once (`OWNERLESS_PER_PEER`, `userland/netstack/node/src/streams.rs`); one past them has 100 s from the pass that found it so, until one of the 16 is done and it takes its place. What holds in this stage, from the code: per address, 16, an estimate no measurement set. There is no floor on the rate: each of the 16 is kept for as long as its pipe gives up one byte per 100 s, which for a full 2 MiB pipe is 2,097,152 x 100 s, over six years, and then for its tail in [tcp] (the line below on a client gone with nothing left in its pipe). The 16 is no bound across addresses: addresses are not counted, and where a connect's address is its client's choice, an accepted stream's is the peer's, which on the link answers for as many addresses as it likes. The bound across them is the node's places (`userland/netstack/node/src/places.rs`), of which every stream holds one until the node lets it go, and there is no second one: in `peers_at_more_addresses_than_there_are_places_hold_no_stream_past_them` (`userland/netstack/node/tests/listeners.rs`) peers at four addresses bring sixteen connections each to a node with three places, which holds two such streams at a time, each kept alive past its 100 s, and refuses every accept and connect past them. `issues/netstack-cuts-a-departed-clients-unsent-tail-at-the-ceiling.md` stays open until netd runs on the node: on a host its exit's second clause is met (`a_departed_clients_tail_arrives_whole_while_its_peer_takes_it`) and its first for 16 streams an address. Exit: the T14 or a guest shows a peer holding one at a byte per 100 s, and the cut takes a floor on the rate, or the owner rules it needs none; and the move closes that issue with its bench row. - A stream its peer reset is gone from the node at once, so for a client that still holds it `Node::set_nodelay` answers `false` and `Node::nodelay` `None`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers, and libc's `getsockopt` of `TCP_NODELAY`, which asks through `toyos::net::tcp_get_option` (`userland/libc/src/socket.rs`) and is refused where both hosts answer. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. - Every frame and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per frame, as netd's `bridge_piped` is today. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netd is a system call: 1,000 system calls a frame with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one frame makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. Exit: at the move, netd's CPU time per frame on the T14 with 1, 100 and 1,000 idle streams beside one bulk stream is measured, and the change lands if 1,000 idle streams cost more than 12.3 µs a frame over what one costs. The 12.3 µs is the time a full frame takes on a wire of 1 Gbit/s, 1,538 bytes of 8 ns with its preamble and gap: past it the passes alone take longer than the bulk stream's frames take to arrive. It is a threshold by arithmetic and no measurement; by estimate, not measured, 1,000 system calls exceed it. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. -- `Tcp::ready` and `Tcp::orphans` are additions to [tcp] the specifications do not hold, made for the node's wakes and places; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. Exit: the specifications hold both calls and the tests name their scenarios. +- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. Exit: the specifications hold the three calls and the refusal, and the tests name their scenarios. - The word each of [udp]'s refusals is answered in on the pipe ABI is `Refused` in `userland/netstack/node/src/datagram.rs`, tested by `each_refusal_is_answered_in_the_pipes_word_for_it` against no reader's scenario; `udp.no-ephemeral-port` has a word and no test that reaches it. Exit: the scenario owed below exists, and the test names its id. - A client's datagram to a broadcast address is refused `udp.broadcast-not-permitted` and answered invalid input, where smoltcp sends it: the pipe ABI has no request that permits it, and std's `set_broadcast` (`sdk/std/sys/net/connection.rs`) answers `Ok` to a permission nothing grants. No program in the tree sends to a broadcast address or sets the option; the sources of the third-party C programs an image builds from a recipe have not been searched. The owner's ruling: "Of course carry the permission why would you even ask. The premise is and always has been to do things properly". The move of netd onto the node does not land with this open. Exit, both before the move, as a stage of their own: that search, for `SO_BROADCAST` and a `sendto` to a broadcast address, is posted with its command and output; and the pipe ABI carries the permission from std's and libc's setters to `Udp::set_broadcast`, a send refused for want of it answered in a word that says permission and not input. - A refusal [udp] logs against a client's call waits in the stack until the node's next `receive`, `fire` or `link`, because only those pass over the stack's reports. Exit, at the move and not after it: the datagram calls end in that pass, or netd drains it once per turn of its loop. diff --git a/toyos-net-shard/src/lib.rs b/toyos-net-shard/src/lib.rs index 9245bf168fb..d7ee5fb50b2 100644 --- a/toyos-net-shard/src/lib.rs +++ b/toyos-net-shard/src/lib.rs @@ -98,6 +98,15 @@ pub enum ConnectError { Tcp(toyos_net_tcp::Error), } +/// Why a passive open was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ListenError { + /// The address named is none the interface may be bound to: the rule [udp]'s bind refuses + /// by (`udp.bind-address-not-local`). + NotLocal, + Tcp(toyos_net_tcp::Error), +} + /// [ip]'s MTU, as TCP's configuration takes it. const TCP_MTU: u16 = { let [low, high, rest @ ..] = toyos_net_ip::MTU.to_le_bytes(); @@ -467,10 +476,13 @@ impl Shard { connected } - /// `addr` is the local address to listen on, UNSPECIFIED for any; port 0 takes `random`'s - /// draws. - pub fn listen(&mut self, addr: Ipv4Addr, port: Option, random: impl FnMut() -> u16) -> Result { - self.tcp.listen(addr, port, random) + /// `addr` is the local address to listen on, UNSPECIFIED for any, and otherwise one [ip] + /// holds assigned or announcing, as a datagram socket's is; port 0 takes `random`'s draws. + pub fn listen(&mut self, addr: Ipv4Addr, port: Option, random: impl FnMut() -> u16) -> Result { + if !addr.is_unspecified() && !self.ip.is_assigned(addr) { + return Err(ListenError::NotLocal); + } + self.tcp.listen(addr, port, random).map_err(ListenError::Tcp) } pub fn accept(&mut self, id: ListenerId) -> Result, toyos_net_tcp::Error> { @@ -481,6 +493,12 @@ impl Shard { self.tcp.listener_port(id) } + /// [`Tcp::set_listener_options`]: what a connection whose SYN arrives from here on starts + /// with. + pub fn set_listener_options(&mut self, id: ListenerId, options: toyos_net_tcp::Options) -> Result<(), toyos_net_tcp::Error> { + self.tcp.set_listener_options(id, options) + } + /// [`Tcp::ready`]. pub fn ready(&mut self, id: ListenerId) -> Result { self.tcp.ready(id) @@ -521,6 +539,11 @@ impl Shard { done } + /// [`Tcp::options`]. + pub fn options(&mut self, id: ConnId) -> Result { + self.tcp.options(id) + } + pub fn shutdown_write(&mut self, now: Instant, id: ConnId) -> Result<(), toyos_net_tcp::Error> { let done = self.tcp.shutdown_write(now, id); self.settle(now); diff --git a/toyos-net-shard/tcp/src/stack.rs b/toyos-net-shard/tcp/src/stack.rs index 4f71e2fb481..b0c7a839123 100644 --- a/toyos-net-shard/tcp/src/stack.rs +++ b/toyos-net-shard/tcp/src/stack.rs @@ -215,9 +215,9 @@ pub struct Tcp { demux: BTreeMap, time_waits: BTreeSet<(Instant, Tuple)>, deadlines: BTreeSet<(Instant, u32)>, - /// Connections that became eligible for the caller's round since it last drained them. /// Connections whose user let go and that have not ended. orphans: usize, + /// Connections that became eligible for the caller's round since it last drained them. eligible: Vec, /// Connections freed while offered to the caller's round, since it last drained them. gone: Vec, @@ -974,6 +974,12 @@ impl Tcp { // ---- user calls ---- + /// The options the connection has: the ones its listener had when its SYN arrived, or the + /// defaults of an active open, until [`Self::set_options`] writes others. + pub fn options(&mut self, id: ConnId) -> Result { + Ok(self.conn(id)?.options) + } + pub fn set_options(&mut self, now: Instant, id: ConnId, options: Options) -> Result<(), Error> { self.conn(id)?.options = options; self.settle(id.index, now); diff --git a/toyos-net-shard/tcp/tests/held.rs b/toyos-net-shard/tcp/tests/held.rs index 08d9389e337..35380368e12 100644 --- a/toyos-net-shard/tcp/tests/held.rs +++ b/toyos-net-shard/tcp/tests/held.rs @@ -1,11 +1,11 @@ -//! What [tcp] says it holds, for a caller that bounds it: `orphans` and `ready`. No scenario ids: -//! the specifications hold neither call; the connections counted are `close.rs`'s and -//! `listen.rs`'s, by their scenarios' own segments. +//! What [tcp] says it holds, for a caller that bounds it or keeps a copy: `orphans`, `ready` and +//! `options`. No scenario ids: the specifications hold none of the calls; the connections +//! counted are `close.rs`'s and `listen.rs`'s, by their scenarios' own segments. mod common; use common::*; -use toyos_net_tcp::{limits, State}; +use toyos_net_tcp::{limits, Error, Options, State}; #[test] fn an_orphan_is_counted_until_both_fins() { @@ -64,3 +64,20 @@ fn ready_counts_the_connections_accept_has_yet_to_return() { h.tcp.close_listener(now, listener).unwrap(); assert_eq!(h.tcp.ready(listener), Err(toyos_net_tcp::Error::NoSuchSocket)); } + +#[test] +fn options_are_the_ones_the_connection_has() { + let mut h = listening(); + let listener = h.listener.unwrap(); + let nodelay = Options { nodelay: true, ..Options::default() }; + h.tcp.set_listener_options(listener, nodelay).unwrap(); + h.input(0, seg(5000).syn().mss(1460)); + h.input(1, seg(5001).ack(1001)); + let conn = h.tcp.accept(listener).unwrap().unwrap(); + assert_eq!(h.tcp.options(conn), Ok(nodelay), "its listener's, as its SYN found them"); + let now = h.now(); + h.tcp.set_options(now, conn, Options::default()).unwrap(); + assert_eq!(h.tcp.options(conn), Ok(Options::default())); + h.tcp.abort(now, conn).unwrap(); + assert_eq!(h.tcp.options(conn), Err(Error::NoSuchSocket)); +} diff --git a/userland/netstack/node/Cargo.toml b/userland/netstack/node/Cargo.toml index 1c2589063a7..b880f8589e3 100644 --- a/userland/netstack/node/Cargo.toml +++ b/userland/netstack/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toyos-net-node" -description = "Netstack's decisions, pure: one interface of ToyOS's own stack, its DHCP lease, its clients' datagram sockets, streams and listeners, the bound on streams and listeners, and its mDNS name, frames and time in, frames out against the device's credit." +description = "Netstack's decisions, pure: one interface of ToyOS's own stack, its DHCP lease, its clients' datagram sockets, streams and listeners and the one bound on all three, and its mDNS name, frames and time in, frames out against the device's credit." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" @@ -22,3 +22,6 @@ toyos-net-wire = { path = "../../../toyos-net-wire" } # The outside reader of every frame the node emits: a published Ethernet, ARP, # IPv4, UDP and ICMP parser, used as a black box. etherparse = "0.21.0" +# The host's own TCP as an oracle: sets and reads a socket option std has no call +# for on a listener, on Linux, macOS and Windows alike, with no `unsafe`. +socket2 = "0.6" diff --git a/userland/netstack/node/src/datagram.rs b/userland/netstack/node/src/datagram.rs index 80b7ab9cfe4..b24dead14e8 100644 --- a/userland/netstack/node/src/datagram.rs +++ b/userland/netstack/node/src/datagram.rs @@ -7,6 +7,10 @@ //! nothing. A destination, a port or a length is the client's number and is refused, never //! trusted; a datagram off the wire reaches a socket only through [udp]'s demultiplexer. //! +//! **A socket holds a place** (`places`) from its bind to its close: what it makes the node hold +//! is its two queues, `limits::RX_DATAGRAMS` received and `limits::TX_DATAGRAMS` accepted. A bind +//! with no place left is answered [`Refused::ResourceExhausted`] with nothing made. +//! //! A refusal [udp] logs comes out of [`Node::drain_events`] after the node's next `receive`, //! `fire` or `link`. @@ -32,7 +36,8 @@ pub enum Refused { /// The call names what no datagram of this socket may carry: an address that is not this /// machine's to bind, a destination nothing is sent to, port 0, more than one frame holds. InvalidInput, - /// The socket holds all it may until some of it has left: the same call succeeds later. + /// The socket holds all it may until some of it has left, or the node holds all it has + /// places for: the same call succeeds later. ResourceExhausted, } @@ -98,7 +103,11 @@ impl Node { /// or, with none named, an ephemeral one, which spends the one draw. Returns the socket and /// the port it holds. pub fn udp_bind(&mut self, addr: Ipv4Addr, port: Option, draw: impl FnOnce() -> u32) -> Result<(DatagramId, Port), Refused> { + if self.room() == 0 { + return Err(Refused::ResourceExhausted); + } let (id, port) = self.stack.bind(addr, port, draw).map_err(refused)?; + self.sockets = self.sockets.saturating_add(1); Ok((DatagramId(id), port)) } @@ -115,8 +124,11 @@ impl Node { } /// Closes the socket: its port is free at once, what it received is gone, and what it had - /// accepted still leaves, to [udp]'s bound. + /// accepted still leaves, to [udp]'s bound. Its place is back. pub fn udp_close(&mut self, now: Instant, id: DatagramId) -> Result<(), Refused> { - self.stack.close(now, id.0).map_err(refused) + self.stack.close(now, id.0).map_err(refused)?; + self.sockets = self.sockets.saturating_sub(1); + self.wake_owners(now); + Ok(()) } } diff --git a/userland/netstack/node/src/lease/tcp.rs b/userland/netstack/node/src/lease/tcp.rs index 2f4a420aac8..bd9bcbd731c 100644 --- a/userland/netstack/node/src/lease/tcp.rs +++ b/userland/netstack/node/src/lease/tcp.rs @@ -3,7 +3,7 @@ use core::net::Ipv4Addr; -use toyos_net_shard::ConnectError; +use toyos_net_shard::{ConnectError, ListenError}; use toyos_net_tcp::{ConnId, Endpoint, Error, ListenerId, Options, Received, Status, Tuple}; use toyos_net_wire::{Instant, Port}; @@ -69,14 +69,16 @@ impl Stack { self.shard.orphans() } - /// A passive open on every address the interface holds or comes to hold, at `port` or at one - /// of `random`'s candidates. `None` is a port that is taken, or no candidate free. - pub(crate) fn tcp_listen(&mut self, port: Option, random: impl FnMut() -> u16) -> Option<(ListenerId, Port)> { - match self.shard.listen(Ipv4Addr::UNSPECIFIED, port, random) { - Ok(id) => Some((id, held(self.shard.listener_port(id)))), - Err(Error::AddrInUse) => None, - Err(refusal) => unreachable!("[tcp] refused a passive open for more than its port: {refusal:?}"), - } + /// A passive open at `addr`, 0.0.0.0 meaning every address the interface holds or comes to + /// hold, and at `port` or at one of `random`'s candidates. + pub(crate) fn tcp_listen(&mut self, addr: Ipv4Addr, port: Option, random: impl FnMut() -> u16) -> Result<(ListenerId, Port), ListenError> { + let id = self.shard.listen(addr, port, random)?; + Ok((id, held(self.shard.listener_port(id)))) + } + + /// What a connection whose SYN arrives at `id` from here on starts with. + pub(crate) fn tcp_set_listener_options(&mut self, id: ListenerId, options: Options) { + held(self.shard.set_listener_options(id, options)); } /// How many connections finished their handshake at `id` and wait to be accepted. @@ -84,10 +86,11 @@ impl Stack { held(self.shard.ready(id)) } - /// The oldest connection waiting at `id`, the caller's from here, and its two endpoints. - pub(crate) fn tcp_accept(&mut self, id: ListenerId) -> Option<(ConnId, Tuple)> { + /// The oldest connection waiting at `id`, the caller's from here, its two endpoints and the + /// options it has. + pub(crate) fn tcp_accept(&mut self, id: ListenerId) -> Option<(ConnId, Tuple, Options)> { let conn = held(self.shard.accept(id))?; - Some((conn, held(self.shard.tuple(conn)))) + Some((conn, held(self.shard.tuple(conn)), held(self.shard.options(conn)))) } /// [tcp] resets every connection still waiting at `id`; `id` names nothing after. diff --git a/userland/netstack/node/src/lib.rs b/userland/netstack/node/src/lib.rs index b67d9621496..a02ec2f6b52 100644 --- a/userland/netstack/node/src/lib.rs +++ b/userland/netstack/node/src/lib.rs @@ -7,10 +7,11 @@ //! The node carries out what the client asks and tells it what became of its address. What the //! interface holds is `lease`'s to write and nobody else's: see that module for the three rules //! its types keep. A client's TCP connection and its two pipes are `streams`', a passive open and -//! its owner's wakes `listeners`', and how many of either the node holds `places`'. +//! its owner's wakes `listeners`'. //! //! A client's datagram sockets are `datagram`'s and the machine's `.local` name is `name`'s: -//! both read the lease and write none of it. +//! both read the lease and write none of it. How many streams, listeners and datagram sockets +//! the node holds is `places`'. //! //! **Untrusted input.** A received frame is never read here: every byte goes through //! `toyos-net-wire`'s parsers inside the shard, and a DHCP payload through the client's. What @@ -79,7 +80,10 @@ pub struct Node { datagram: Vec, streams: streams::Streams, listeners: listeners::Listeners, - /// How many streams, listeners and connections [tcp] finishes alone the node holds at most. + /// The clients' datagram sockets bound and not closed. + sockets: usize, + /// How many streams, listeners, datagram sockets and connections [tcp] finishes alone the + /// node holds at most. places: usize, } @@ -100,6 +104,7 @@ impl Node { datagram: vec![0; usize::from(toyos_dhcp::limits::MAX_MESSAGE)], streams: streams::Streams::default(), listeners: listeners::Listeners::default(), + sockets: 0, places: 0, }) } diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs index b2e2e64ad68..6d27272acac 100644 --- a/userland/netstack/node/src/listeners.rs +++ b/userland/netstack/node/src/listeners.rs @@ -16,16 +16,18 @@ //! refuses, for whatever reason. [tcp] then resets every connection that still waits, so each //! peer learns at once, and the port is free. //! -//! **A stream starts with the options its listener has when it is accepted**, which the node -//! writes to [tcp] at the accept. The listener's are the node's to hold and not [tcp]'s: [tcp] -//! hands a listener's options to a connection at its SYN and has no call that reads a -//! connection's back, so a stream seeded from them would answer for an option its connection, -//! begun before the option was set, does not have. +//! **A stream starts with the options its connection has**, and those are its listener's as +//! they were when its SYN arrived: [tcp] hands them over then (LS-10), the node reads them +//! back at the accept, and an option set on the listener afterwards reaches the connections +//! that begin afterwards. A host does the same with `TCP_NODELAY` set on a listening socket +//! (`tests/host.rs` asks the host the tests run on). use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; +use core::net::Ipv4Addr; -use toyos_net_tcp::{Endpoint, Options}; +use toyos_net_shard::ListenError; +use toyos_net_tcp::{Endpoint, Error, Options}; use toyos_net_wire::{Instant, Port}; use crate::streams::{Pipes, StreamId, WriteRefusal}; @@ -46,7 +48,10 @@ pub struct ListenerId(u64); pub enum ListenRefused { /// The node holds all it has places for (`places`). Full, - /// The port is another listener's, or no drawn port was free. + /// The address named is not this machine's to bind: what `Node::udp_bind` refuses by the + /// same rule. + NotLocal, + /// The port is another listener's at that address, or no drawn port was free. InUse, } @@ -76,7 +81,7 @@ struct Listener { owner: Box, /// Wakes written that no accept has spent. unspent: usize, - /// What a stream accepted here starts with. + /// The options last written to [tcp]. options: Options, } @@ -89,10 +94,10 @@ pub(crate) struct Listeners { } impl Node { - /// A passive open on `port`, or on a port chosen from `draw`'s candidates, at every address - /// the interface holds or comes to hold. Answers the listener and its port. Refused, nothing - /// was made and `owner` is dropped. - pub fn listen(&mut self, port: Option, owner: Box, mut draw: impl FnMut() -> u32) -> Result<(ListenerId, Port), ListenRefused> { + /// A passive open at `addr`, 0.0.0.0 meaning every address the interface holds or comes to + /// hold, on `port` or on a port chosen from `draw`'s candidates. Answers the listener and + /// its port. Refused, nothing was made and `owner` is dropped. + pub fn listen(&mut self, addr: Ipv4Addr, port: Option, owner: Box, mut draw: impl FnMut() -> u32) -> Result<(ListenerId, Port), ListenRefused> { if self.room() == 0 { return Err(ListenRefused::Full); } @@ -100,18 +105,25 @@ impl Node { let [low, high, ..] = draw().to_le_bytes(); u16::from_le_bytes([low, high]) }; - let Some((bound, port)) = self.stack.tcp_listen(port, candidate) else { return Err(ListenRefused::InUse) }; + let (bound, port) = match self.stack.tcp_listen(addr, port, candidate) { + Ok(listening) => listening, + Err(ListenError::NotLocal) => return Err(ListenRefused::NotLocal), + Err(ListenError::Tcp(Error::AddrInUse)) => return Err(ListenRefused::InUse), + Err(ListenError::Tcp(refusal)) => unreachable!("[tcp] refused a passive open for more than its port: {refusal:?}"), + }; let id = ListenerId(self.listeners.next); self.listeners.next = self.listeners.next.saturating_add(1); self.listeners.live.insert(id, Listener { bound, owner, unspent: 0, options: Options::default() }); Ok((id, port)) } - /// Nagle's algorithm off or on (RFC 9293 §3.7.4) for every stream accepted at `id` from here - /// on; one already accepted keeps its own. `false` is an id that names no listener. + /// Nagle's algorithm off or on (RFC 9293 §3.7.4) for every connection whose SYN arrives at + /// `id` from here on; one that began before keeps what it has. `false` is an id that names + /// no listener. pub fn set_listener_nodelay(&mut self, id: ListenerId, nodelay: bool) -> bool { let Some(listener) = self.listeners.live.get_mut(&id) else { return false }; listener.options.nodelay = nodelay; + self.stack.tcp_set_listener_options(listener.bound, listener.options); true } @@ -124,20 +136,20 @@ impl Node { pub fn accept(&mut self, now: Instant, id: ListenerId, pipes: Option) -> Result { let Some(listener) = self.listeners.live.get_mut(&id) else { return Err(AcceptRefused::NoListener) }; listener.unspent = listener.unspent.saturating_sub(1); - let (bound, options) = (listener.bound, listener.options); - let answer = self.take(now, bound, options, pipes); + let bound = listener.bound; + let answer = self.take(bound, pipes); self.bridge(now); answer } - fn take(&mut self, now: Instant, bound: toyos_net_tcp::ListenerId, options: Options, pipes: Option) -> Result { + fn take(&mut self, bound: toyos_net_tcp::ListenerId, pipes: Option) -> Result { let Some(pipes) = pipes else { return Err(AcceptRefused::NoPipes) }; if self.room() == 0 { return Err(AcceptRefused::Full); } - let Some((conn, tuple)) = self.stack.tcp_accept(bound) else { return Err(AcceptRefused::Nothing) }; + let Some((conn, tuple, options)) = self.stack.tcp_accept(bound) else { return Err(AcceptRefused::Nothing) }; // The peer's address is what `streams` counts a stream its client can see no more by. - let id = self.streams.accepted(now, &mut self.stack, conn, tuple.remote.addr, options, pipes); + let id = self.streams.accepted(conn, tuple.remote.addr, options, pipes); Ok(Accepted { id, remote: tuple.remote, local: tuple.local.port }) } diff --git a/userland/netstack/node/src/places.rs b/userland/netstack/node/src/places.rs index 94f51642107..1113a98eabc 100644 --- a/userland/netstack/node/src/places.rs +++ b/userland/netstack/node/src/places.rs @@ -1,21 +1,28 @@ //! Places: the one bound on what clients make the node hold. A stream holds a place from its //! connect or its accept until the node lets it go, a listener from its listen until it is -//! closed, and a connection the node closed holds the place its stream had until [tcp] has -//! finished it, because it is still two buffers of [tcp]'s and its peer decides for how long. -//! A stream its client can see no more, which its peer keeps alive, holds its place through -//! both: it is a stream while its pipe holds bytes, and [tcp]'s to finish after. `streams` lets -//! one peer address keep at most `OWNERLESS_PER_PEER` of them alive and counts no addresses, so -//! that is each address's share of the cut's clock and no bound on the node: peers at however -//! many addresses hold no more of them than there are places. With no place left a connect, a -//! listen and an accept are refused with nothing made. +//! closed, a datagram socket from its bind until it is closed, and a connection the node closed +//! holds the place its stream had until [tcp] has finished it, because it is still two buffers +//! of [tcp]'s and its peer decides for how long. A stream its client can see no more, which its +//! peer keeps alive, holds its place through both: it is a stream while its pipe holds bytes, +//! and [tcp]'s to finish after. `streams` lets one peer address keep at most +//! `OWNERLESS_PER_PEER` of them alive and counts no addresses, so that is each address's share +//! of the cut's clock and no bound on the node: peers at however many addresses hold no more of +//! them than there are places. With no place left a connect, a listen, an accept and a bind are +//! refused with nothing made. //! //! The number is the shell's: each stream is two of its client's pipes kept alive and watched, //! each listener one. A new node has none. //! -//! What a peer makes the node hold without a client asking is not counted here and is bounded -//! by [tcp] for each listener: `limits::LISTEN_PENDING` handshakes in progress, each given up -//! after `limits::SYNACK_GIVE_UP`, and `limits::LISTEN_READY` connections waiting to be -//! accepted, each with at most one receive buffer of text. +//! **A place is not one size.** A stream or a connection [tcp] finishes is [tcp]'s two buffers, +//! and a datagram socket [udp]'s two queues of `limits::RX_DATAGRAMS` and `limits::TX_DATAGRAMS` +//! datagrams. A listener's place stands for all a peer can make it hold without a client +//! asking, which [tcp] bounds for each listener: `limits::LISTEN_PENDING` handshakes in +//! progress, each given up after `limits::SYNACK_GIVE_UP`, and `limits::LISTEN_READY` +//! connections waiting to be accepted, each with at most one receive buffer of text, taken when +//! its first byte arrives. The shell prices a listener at that product when it picks the number. +//! A connection that waits holds no place of its own: it would take one by a peer's handshake +//! alone, so any peer of any listener could then refuse every client its connect, listen and +//! bind, where now it can fill only the queue of the listener it reaches. use toyos_net_wire::Instant; @@ -28,9 +35,10 @@ impl Node { self.wake_owners(now); } - /// The places taken: streams, listeners, and connections [tcp] is finishing alone. + /// The places taken: streams, listeners, datagram sockets, and connections [tcp] is + /// finishing alone. pub fn held(&self) -> usize { - self.streams().saturating_add(self.listeners()).saturating_add(self.stack.tcp_orphans()) + self.streams().saturating_add(self.listeners()).saturating_add(self.sockets).saturating_add(self.stack.tcp_orphans()) } pub(crate) fn room(&self) -> usize { diff --git a/userland/netstack/node/src/streams.rs b/userland/netstack/node/src/streams.rs index 091d3fb2384..41e86527a6c 100644 --- a/userland/netstack/node/src/streams.rs +++ b/userland/netstack/node/src/streams.rs @@ -319,9 +319,8 @@ impl Streams { } /// Holds a connection a listener's owner accepted, established, on the pipes its accept - /// handed over, and writes it its listener's `options`. - pub(crate) fn accepted(&mut self, now: Instant, stack: &mut Stack, conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> StreamId { - stack.tcp_set_options(now, conn, options); + /// handed over. `options` are the ones [tcp] says it has. + pub(crate) fn accepted(&mut self, conn: ConnId, remote: Ipv4Addr, options: Options, pipes: Pipes) -> StreamId { self.hold(Stream::established(conn, remote, options, pipes)) } diff --git a/userland/netstack/node/tests/common/mod.rs b/userland/netstack/node/tests/common/mod.rs index e178e383904..4df477177f0 100644 --- a/userland/netstack/node/tests/common/mod.rs +++ b/userland/netstack/node/tests/common/mod.rs @@ -121,6 +121,52 @@ pub fn outside(frame: &[u8], mac: [u8; 6]) -> Seen { } } +/// A TCP segment the node emitted, as `etherparse` read it. +#[derive(Clone, Debug)] +pub struct Segment { + /// Where its frame and its datagram went. + pub to_mac: [u8; 6], + pub to: Ipv4Addr, + /// The node's port, and the peer's. + pub from_port: u16, + pub to_port: u16, + pub seq: u32, + pub ack: Option, + pub syn: bool, + pub fin: bool, + pub rst: bool, + pub window: u16, + pub text: Vec, +} + +/// The outside reading of a TCP segment the node emitted, or `None` of a frame that carries +/// none: Ethernet II from the node's MAC, IPv4 from the node's address, and the lengths and the +/// two checksums `etherparse` computes. +pub fn segment(frame: &[u8]) -> Option { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; + let (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) = (&packet.net, &packet.transport) else { return None }; + let header = ip.header(); + assert_eq!(ethernet.source(), MAC, "from the node's MAC"); + assert_eq!(header.header_checksum(), header.to_header().calc_header_checksum(), "the IPv4 header checksum"); + assert_eq!(usize::from(header.total_len()), 20 + tcp.slice().len(), "IPv4's length is the segment's"); + assert_eq!(header.source_addr(), A); + assert_eq!(tcp.checksum(), tcp.calc_checksum_ipv4(header.source(), header.destination()).unwrap(), "the TCP checksum"); + Some(Segment { + to_mac: ethernet.destination(), + to: header.destination_addr(), + from_port: tcp.source_port(), + to_port: tcp.destination_port(), + seq: tcp.sequence_number(), + ack: tcp.ack().then(|| tcp.acknowledgment_number()), + syn: tcp.syn(), + fin: tcp.fin(), + rst: tcp.rst(), + window: tcp.window_size(), + text: tcp.payload().to_vec(), + }) +} + /// The value of option `code` in a DHCP message (RFC 2132 §2): the first instance. pub fn option(message: &[u8], code: u8) -> Option<&[u8]> { let mut rest = &message[240..]; diff --git a/userland/netstack/node/tests/host.rs b/userland/netstack/node/tests/host.rs new file mode 100644 index 00000000000..1bd0d9914f2 --- /dev/null +++ b/userland/netstack/node/tests/host.rs @@ -0,0 +1,34 @@ +//! What the host these tests run on does with `TCP_NODELAY` set on a listening socket: the +//! reading `listeners`' rule for a listener's option rests on, asked of every host the suite +//! runs on. Nothing of the node runs here: the sockets are the host kernel's, over loopback. +//! `tests/listeners.rs` holds the node to the same two answers +//! (`a_stream_starts_with_the_options_its_connection_took_from_its_listener`). + +use std::net::{Ipv4Addr, TcpListener, TcpStream}; + +use socket2::SockRef; + +/// Whether the socket accepted from a listener has `TCP_NODELAY`, the option set on the +/// listener `before` the connection began, or else once it was established and before the +/// accept. A connect that returned is a handshake that ended: the connection waits in the +/// listener's queue, and the accept takes it without waiting. +fn accepted_with_nodelay(before: bool) -> bool { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("a loopback port"); + assert!(!SockRef::from(&listener).tcp_nodelay().expect("a listener answers for the option")); + if before { + SockRef::from(&listener).set_tcp_nodelay(true).expect("a listener takes the option"); + } + let _peer = TcpStream::connect(listener.local_addr().expect("bound")).expect("the listener's queue takes it"); + if !before { + SockRef::from(&listener).set_tcp_nodelay(true).expect("a listener takes the option"); + } + let (accepted, _) = listener.accept().expect("the connection that waits"); + assert!(SockRef::from(&listener).tcp_nodelay().expect("a listener answers for the option")); + accepted.nodelay().expect("an accepted socket answers for the option") +} + +#[test] +fn a_host_gives_a_connection_the_nodelay_its_listener_had_when_it_began() { + assert!(accepted_with_nodelay(true), "set before the connection began, the accepted socket has it"); + assert!(!accepted_with_nodelay(false), "set after the connection was established, it does not"); +} diff --git a/userland/netstack/node/tests/listeners.rs b/userland/netstack/node/tests/listeners.rs index be0ecfaa135..41a1edb77de 100644 --- a/userland/netstack/node/tests/listeners.rs +++ b/userland/netstack/node/tests/listeners.rs @@ -18,12 +18,14 @@ use std::net::Ipv4Addr; use std::rc::Rc; use std::time::Duration; -use common::{arp, terms, Wire, A, MAC, MAC_R, R}; +use common::{arp, terms, Segment, Wire, A, ELSEWHERE, MAC, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; use toyos_net_tcp::{limits, Counter, Endpoint}; use toyos_net_wire::{Instant, Port}; +const ANY: Ipv4Addr = Ipv4Addr::UNSPECIFIED; + /// The addresses peers are at. const B: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); const C: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 3); @@ -160,20 +162,6 @@ fn client() -> (Client, Pipes) { // ---- the peers ---- -/// A segment the node emitted, as `etherparse` read it. -#[derive(Clone, Debug)] -struct Segment { - /// The node's port. - from: u16, - to: Peer, - seq: u32, - ack: Option, - syn: bool, - fin: bool, - rst: bool, - text: Vec, -} - /// A segment from the peer to the node's port `to` in its frame, built by `etherparse`: window /// 65,535, an MSS option on a SYN, PSH with text. fn frame(from: Peer, to: u16, seq: u32, ack: Option, flags: u8, text: &[u8]) -> Vec { @@ -248,9 +236,15 @@ impl Net { /// The outside reading of a frame the node emitted, and the ARP answer it gets if it asks /// for one. Ethernet II from the node's MAC; then ARP, a datagram of the DHCP client's, or - /// a TCP segment to an address a peer is at, with the lengths and checksums `etherparse` - /// computes. A segment to any other address is the node's mistake. + /// a TCP segment to an address a peer is at, as `common::segment` reads it. A segment to any + /// other address is the node's mistake. fn hears(&mut self, frame: &[u8]) -> Option> { + if let Some(segment) = common::segment(frame) { + assert!(self.at.contains(&segment.to), "a segment to {}, where no peer is", segment.to); + assert_eq!(segment.to_mac, mac(segment.to), "to the peer's MAC"); + self.heard.push(segment); + return None; + } let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; assert_eq!(ethernet.source(), MAC, "from the node's MAC"); @@ -264,28 +258,7 @@ impl Net { } } (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(_))) => None, - (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) => { - let header = ip.header(); - let to = header.destination_addr(); - assert!(self.at.contains(&to), "a segment to {to}, where no peer is"); - assert_eq!(ethernet.destination(), mac(to), "to the peer's MAC"); - assert_eq!(header.header_checksum(), header.to_header().calc_header_checksum(), "the IPv4 header checksum"); - assert_eq!(usize::from(header.total_len()), 20 + tcp.slice().len(), "IPv4's length is the segment's"); - assert_eq!(header.source_addr(), A); - assert_eq!(tcp.checksum(), tcp.calc_checksum_ipv4(header.source(), header.destination()).unwrap(), "the TCP checksum"); - self.heard.push(Segment { - from: tcp.source_port(), - to: Peer { addr: to, port: tcp.destination_port() }, - seq: tcp.sequence_number(), - ack: tcp.ack().then(|| tcp.acknowledgment_number()), - syn: tcp.syn(), - fin: tcp.fin(), - rst: tcp.rst(), - text: tcp.payload().to_vec(), - }); - None - } - other => panic!("neither ARP, UDP nor TCP: {other:?}"), + other => panic!("neither ARP nor UDP: {other:?}"), } } @@ -315,7 +288,7 @@ impl Net { /// The segments the node sent to `peer`. fn to(&self, peer: Peer) -> Vec { - self.heard.iter().filter(|segment| segment.to == peer).cloned().collect() + self.heard.iter().filter(|segment| (segment.to, segment.to_port) == (peer.addr, peer.port)).cloned().collect() } /// The last segment the node sent to `peer`. @@ -349,6 +322,13 @@ impl Net { self.deliver(&frame(peer, SSH, ISS + 1, Some(ack), 0, &[])); } + /// The peer's handshake with the node's port `to`. + fn handshake_to(&mut self, peer: Peer, to: u16) { + self.syn_to(peer, to); + let ack = self.iss(peer).wrapping_add(1); + self.deliver(&frame(peer, to, ISS + 1, Some(ack), 0, &[])); + } + fn handshake(&mut self, peer: Peer) { self.syn(peer); self.finish(peer); @@ -385,7 +365,7 @@ impl Net { fn listen(&mut self, port: u16) -> (ListenerId, Owner) { let owner = Owner::default(); - let (id, bound) = self.node.listen(Port::new(port), Box::new(WakeEnd(owner.clone())), draw(&mut self.draws)).expect("a free port and a place"); + let (id, bound) = self.node.listen(ANY, Port::new(port), Box::new(WakeEnd(owner.clone())), draw(&mut self.draws)).expect("a free port and a place"); assert_eq!(bound.get(), port); (id, owner) } @@ -454,7 +434,7 @@ fn a_listener_answers_a_syn_and_wakes_its_owner_when_the_handshake_ends() { let answered = net.to(P1); let [synack] = &answered[..] else { panic!("one SYN-ACK, not {:?}", net.heard) }; assert!(synack.syn && !synack.fin && !synack.rst && synack.text.is_empty(), "{synack:?}"); - assert_eq!((synack.from, synack.ack), (SSH, Some(ISS + 1))); + assert_eq!((synack.from_port, synack.ack), (SSH, Some(ISS + 1))); assert_eq!(wakes(&owner), 0, "SYN-RECEIVED is no connection yet"); assert_eq!(net.accept(id).unwrap_err(), AcceptRefused::Nothing); @@ -659,7 +639,7 @@ fn a_connect_past_the_places_is_refused_and_sends_nothing() { let (third, client) = net.connect(); assert_eq!(third, Err(ConnectRefused::Full), "a connect not yet answered holds a place"); assert_eq!((client.borrow().dropped, net.node.held()), (2, 2)); - let sources: BTreeSet = net.to(WEB).iter().map(|segment| segment.from).collect(); + let sources: BTreeSet = net.to(WEB).iter().map(|segment| segment.from_port).collect(); assert_eq!(sources.len(), 2, "no SYN left for the connect refused: {:?}", net.heard); net.node.close(net.now, first.unwrap()); @@ -675,7 +655,7 @@ fn a_listener_holds_a_place_and_a_listen_without_one_makes_nothing() { let mut net = with_places(1); let (id, _owner) = net.listen(SSH); let refused = Owner::default(); - let answer = net.node.listen(Port::new(TELNET), Box::new(WakeEnd(refused.clone())), draw(&mut net.draws)); + let answer = net.node.listen(ANY, Port::new(TELNET), Box::new(WakeEnd(refused.clone())), draw(&mut net.draws)); assert_eq!(answer.unwrap_err(), ListenRefused::Full); assert!(refused.borrow().dropped); net.syn_to(P1, TELNET); @@ -738,55 +718,130 @@ fn a_listen_on_a_taken_port_is_refused_and_a_drawn_port_listens() { let mut net = Net::new(); let (_, owner) = net.listen(SSH); let second = Owner::default(); - let answer = net.node.listen(Port::new(SSH), Box::new(WakeEnd(second.clone())), draw(&mut net.draws)); + let answer = net.node.listen(ANY, Port::new(SSH), Box::new(WakeEnd(second.clone())), draw(&mut net.draws)); assert_eq!(answer.unwrap_err(), ListenRefused::InUse); assert!(second.borrow().dropped); assert_eq!((net.node.listeners(), net.node.held()), (1, 1)); net.handshake(P1); assert_eq!(wakes(&owner), 1, "the listener that holds the port still listens"); - let (_, port) = net.node.listen(None, Box::new(WakeEnd(Owner::default())), || 0x0001_0005).unwrap(); + let (_, port) = net.node.listen(ANY, None, Box::new(WakeEnd(Owner::default())), || 0x0001_0005).unwrap(); assert_eq!(port.get(), 49_152 + 5); net.syn_to(P2, port.get()); let synack = net.last(P2); - assert!(synack.syn && synack.from == port.get(), "{synack:?}"); + assert!(synack.syn && synack.from_port == port.get(), "{synack:?}"); +} + +// A listener is at the address its listen named. One the machine does not hold is refused, as +// a datagram socket's bind is, and nothing listens: RFC 9293 §3.10.7.1, a SYN for a port +// nothing listens on is answered . One the machine +// holds listens beside a listener at every address on the same port, and [tcp] hands a SYN to +// the one that named the address (LS-09). +#[test] +fn a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds() { + let mut net = Net::new(); + let refused = Owner::default(); + let answer = net.node.listen(ELSEWHERE, Port::new(SSH), Box::new(WakeEnd(refused.clone())), draw(&mut net.draws)); + assert_eq!(answer.unwrap_err(), ListenRefused::NotLocal); + assert!(refused.borrow().dropped); + assert_eq!((net.node.listeners(), net.node.held()), (0, 0)); + net.syn(P1); + let reset = net.last(P1); + assert!(reset.rst && (reset.seq, reset.ack) == (0, Some(ISS + 1)), "{reset:?}"); + + let (_, any) = net.listen(SSH); + let named = Owner::default(); + let (_, port) = net.node.listen(A, Port::new(SSH), Box::new(WakeEnd(named.clone())), draw(&mut net.draws)).expect("the machine's address, and a port no listener at it holds"); + assert_eq!((port.get(), net.node.listeners()), (SSH, 2)); + net.handshake(P2); + assert_eq!((wakes(&named), wakes(&any)), (1, 0)); + let again = net.node.listen(A, Port::new(SSH), Box::new(WakeEnd(Owner::default())), draw(&mut net.draws)); + assert_eq!(again.unwrap_err(), ListenRefused::InUse); +} + +// A listener ended for a wake its owner's pipe refused gives its place back in the pass that +// ended it: the listener after it is owed a wake for that place before the node is asked +// anything more. The last handshake's ACK is delivered with no transmit opportunity after it, +// since an opportunity ends in a pass of its own. +#[test] +fn a_listener_ended_for_its_wake_gives_its_place_to_another_in_the_same_pass() { + let mut net = with_places(3); + let (first, refusing) = net.listen(SSH); + let (_, owner) = net.listen(TELNET); + net.handshake_to(P1, TELNET); + net.handshake_to(P2, TELNET); + assert_eq!(wakes(&owner), 1, "two wait, and there is a place for one"); + refusing.borrow_mut().refusal = Some(WriteRefusal::Gone); + net.syn(P3); + let ack = net.iss(P3).wrapping_add(1); + net.node.receive(net.now, &frame(P3, SSH, ISS + 1, Some(ack), 0, &[]), draw(&mut net.draws)); + assert_eq!(wakes(&owner), 2); + let ended: Vec<(ListenerId, WriteRefusal)> = net.node.drain_refused_listeners().collect(); + assert_eq!((ended, net.node.listeners(), net.node.held()), (vec![(first, WriteRefusal::Gone)], 1, 1)); +} + +// A datagram socket is something a client makes the node hold, its two queues of sixteen +// datagrams, from its bind to its close. +#[test] +fn a_datagram_socket_holds_a_place_and_a_bind_without_one_makes_nothing() { + let undrawn = || -> u32 { panic!("a named port draws nothing") }; + let mut net = with_places(2); + let (_, owner) = net.listen(SSH); + let (socket, _) = net.node.udp_bind(ANY, Port::new(4_000), undrawn).expect("a free port and a place"); + assert_eq!(net.node.held(), 2); + assert_eq!(net.node.udp_bind(ANY, Port::new(4_001), undrawn), Err(Refused::ResourceExhausted)); + let (connect, _client) = net.connect(); + assert_eq!(connect, Err(ConnectRefused::Full), "the socket's place is a stream's"); + net.handshake(P1); + assert_eq!(wakes(&owner), 0); + + assert_eq!(net.node.udp_close(net.now, socket), Ok(())); + assert_eq!((net.node.held(), wakes(&owner)), (1, 1), "its place is back, and the connection that waited is announced"); + assert_eq!(net.node.udp_close(net.now, socket), Err(Refused::NotConnected)); + assert_eq!(net.node.held(), 1, "a close that names no socket gives no place back"); + net.node.udp_bind(ANY, Port::new(4_001), undrawn).expect("the refused bind left its port free"); + assert_eq!(net.node.held(), 2); } // ---- the streams an accept makes ---- // RFC 9293 §3.7.4: with text unacknowledged a short segment waits, and with Nagle's algorithm -// off it leaves. No peer here acknowledges any text. +// off it leaves. No peer here acknowledges any text. The two answers a host gives for the option +// on a listening socket are `tests/host.rs`'s: a connection has it if its listener had it when +// the connection began, whenever it is accepted. #[test] -fn a_stream_starts_with_the_options_its_listener_has_when_it_is_accepted() { +fn a_stream_starts_with_the_options_its_connection_took_from_its_listener() { let mut net = Net::new(); let (id, _owner) = net.listen(SSH); assert_eq!(net.node.listener_nodelay(id), Some(false)); - // One handshake ends before the option is set and one after; both are accepted after. + // One connection begins before the option is set and one after; both are accepted after. net.handshake(P1); assert!(net.node.set_listener_nodelay(id, true)); net.handshake(P2); assert_eq!(net.node.listener_nodelay(id), Some(true)); let (first, a) = net.accepts(id, P1); let (second, b) = net.accepts(id, P2); - for (stream, client, peer) in [(first, &a, P1), (second, &b, P2)] { - assert_eq!(net.node.nodelay(stream), Some(true), "{peer:?}"); + assert_eq!((net.node.nodelay(first), net.node.nodelay(second)), (Some(false), Some(true))); + for client in [&a, &b] { net.says(client, b"a"); net.says(client, b"b"); - assert_eq!(net.texts(peer), [b"a".to_vec(), b"b".to_vec()], "{peer:?}: the second write waits for no acknowledgment"); } + assert_eq!(net.texts(P1), [b"a".to_vec()], "the second write waits for the first's acknowledgment"); + assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the second write waits for no acknowledgment"); // An accepted stream's options are its own from then on, and the listener's its own. assert!(net.node.set_nodelay(net.now, second, false)); net.says(&b, b"c"); - assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the third write waits for the first's acknowledgment"); - assert!(net.node.set_listener_nodelay(id, false)); - assert_eq!((net.node.nodelay(first), net.node.nodelay(second), net.node.listener_nodelay(id)), (Some(true), Some(false), Some(false))); - net.says(&a, b"c"); - assert_eq!(net.texts(P1), [b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]); + assert_eq!(net.texts(P2), [b"a".to_vec(), b"b".to_vec()], "the third write waits"); + assert!(net.node.set_nodelay(net.now, first, true)); + net.pump(); + assert_eq!(net.texts(P1), [b"a".to_vec(), b"b".to_vec()], "and the first stream's second waits no longer"); + assert_eq!((net.node.nodelay(first), net.node.nodelay(second), net.node.listener_nodelay(id)), (Some(true), Some(false), Some(true))); + assert!(net.node.set_listener_nodelay(id, false)); net.handshake(P3); let (third, c) = net.accepts(id, P3); - assert_eq!(net.node.nodelay(third), Some(false)); + assert_eq!((net.node.nodelay(third), net.node.nodelay(first)), (Some(false), Some(true))); net.says(&c, b"a"); net.says(&c, b"b"); assert_eq!(net.texts(P3), [b"a".to_vec()]); @@ -819,7 +874,7 @@ fn an_accepted_stream_is_one_of_its_peers_addresss_sixteen() { } net.run(Duration::from_secs(51)); assert_eq!((net.node.streams(), net.events()), (17, vec![StreamEvent::Cut { id: streams[16] }])); - let reset: Vec = net.heard.iter().filter(|segment| segment.rst).map(|segment| segment.to).collect(); + let reset: Vec = net.heard.iter().filter(|segment| segment.rst).map(|segment| Peer { addr: segment.to, port: segment.to_port }).collect(); assert_eq!(reset, [peers[16]]); } diff --git a/userland/netstack/node/tests/streams.rs b/userland/netstack/node/tests/streams.rs index 6b398e1cc88..d9bef41f1da 100644 --- a/userland/netstack/node/tests/streams.rs +++ b/userland/netstack/node/tests/streams.rs @@ -16,7 +16,7 @@ use std::net::Ipv4Addr; use std::rc::Rc; use std::time::Duration; -use common::{arp, terms, Wire, A, MAC, MAC_B, MAC_R, R}; +use common::{arp, terms, Segment, Wire, A, MAC, MAC_B, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; use toyos_net_node::{ConnectRefused, FromClient, Node, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Watch, WriteRefusal}; use toyos_net_shard::ConnectError; @@ -128,18 +128,6 @@ fn dropped(client: &Client) -> (bool, bool) { // ---- the far end ---- -/// A segment the node emitted, as `etherparse` read it. -#[derive(Clone, Debug)] -struct Segment { - seq: u32, - ack: Option, - syn: bool, - fin: bool, - rst: bool, - window: u16, - text: Vec, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Manner { /// Completes the handshake and acknowledges what arrives in order. @@ -272,9 +260,24 @@ impl Far { /// The outside reading of a frame the node emitted, and the frames it is answered with. /// Ethernet II from the node's MAC; then ARP, whose requests for this peer and the router /// are answered; a datagram of the DHCP client's, which `lease.rs` reads; or a TCP segment - /// to port 80, with the lengths and checksums `etherparse` computes, heard if it is to this - /// peer's address. + /// to port 80, as `common::segment` reads it, heard if it is to this peer's address. fn hears(&mut self, frame: &[u8]) -> Vec> { + if let Some(segment) = common::segment(frame) { + assert_eq!(segment.to_port, PORT); + if segment.to != self.addr || (self.parked && segment.from_port != self.port) { + return Vec::new(); + } + assert_eq!(segment.to_mac, self.mac, "to the peer's MAC"); + self.segments.push(segment.clone()); + if segment.syn { + self.port = segment.from_port; + self.received.clear(); + self.fin = false; + } else if segment.from_port != self.port { + return Vec::new(); + } + return self.answer(&segment); + } let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; assert_eq!(ethernet.source(), MAC, "from the node's MAC"); @@ -288,37 +291,7 @@ impl Far { } } (Some(NetSlice::Ipv4(_)), Some(TransportSlice::Udp(_))) => Vec::new(), - (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(tcp))) => { - let header = ip.header(); - assert_eq!(header.header_checksum(), header.to_header().calc_header_checksum(), "the IPv4 header checksum"); - assert_eq!(usize::from(header.total_len()), 20 + tcp.slice().len(), "IPv4's length is the segment's"); - assert_eq!(header.source_addr(), A); - assert_eq!(tcp.checksum(), tcp.calc_checksum_ipv4(header.source(), header.destination()).unwrap(), "the TCP checksum"); - assert_eq!(tcp.destination_port(), PORT); - if header.destination_addr() != self.addr || (self.parked && tcp.source_port() != self.port) { - return Vec::new(); - } - assert_eq!(ethernet.destination(), self.mac, "to the peer's MAC"); - let segment = Segment { - seq: tcp.sequence_number(), - ack: tcp.ack().then(|| tcp.acknowledgment_number()), - syn: tcp.syn(), - fin: tcp.fin(), - rst: tcp.rst(), - window: tcp.window_size(), - text: tcp.payload().to_vec(), - }; - self.segments.push(segment.clone()); - if segment.syn { - self.port = tcp.source_port(); - self.received.clear(); - self.fin = false; - } else if tcp.source_port() != self.port { - return Vec::new(); - } - self.answer(&segment) - } - other => panic!("neither ARP, UDP nor TCP: {other:?}"), + other => panic!("neither ARP nor UDP: {other:?}"), } } @@ -401,9 +374,7 @@ impl Net { /// request the same answer, which is one frame. A segment to an address no peer is at is /// the node's mistake, and no peer would hear it. fn hears(&mut self, frame: &[u8]) -> Vec> { - let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); - if let (Some(NetSlice::Ipv4(ip)), Some(TransportSlice::Tcp(_))) = (&packet.net, &packet.transport) { - let to = ip.header().destination_addr(); + if let Some(Segment { to, .. }) = common::segment(frame) { assert!(self.far.addr == to || self.parked.iter().any(|far| far.addr == to), "a segment to {to}, where no peer is"); } let mut answers = self.far.hears(frame); From d1d940dfdf3e54bbd287a83468e808d591e4e87d Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 22:26:31 +0200 Subject: [PATCH 5/7] Review round 2 of #777: a port is one listener's, a listener whose address went is ended, the host oracle waits on its listener A port is one listener's. [tcp] hands a SYN to the listener that named its address before one at every address (LS-09), so with a server at 0.0.0.0:22 any program that reached netstack listened at the machine's address on 22 and was handed every connection from then on: authority nobody moved into it. A listen carries no word for its program, so the node cannot allow the pair to one owner and refuse it to another. Node::listen now refuses a named port any listener holds, whichever address either named, as [udp] holds a port once; LS-09 stays [tcp]'s and is not reachable from the node. The track has the line and its exit: a listen that carries its program. A listener whose address went is ended. One that named the machine's address kept its place and its port when the lease lost the address, was handed nothing and its owner was told nothing. The pass that writes the wakes now ends it when the lease names another address or none: [tcp] resets what waits, the port and the place are free, the owner reads the end of its wake pipe, and the shell reads Node::drain_ended_listeners, which replaces drain_refused_listeners and says why (ListenerEnd). Ended and not kept, because a listener's owner makes no call a refusal could answer: it only waits for wakes, and none would come. A renewal and a link that returns keep the address and the listener. tests/host.rs waits on the event its second answer depends on. A connect returns at the SYN-ACK and the listener's end is established by the ACK after it, so the option set "after the connection was established" rested on loopback delivering that ACK first. The test now sets it only once the listener is readable (mio, in the lock already, default features off), bounded by a ceiling that fails, and asserts the pair once with the host's name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- Cargo.lock | 1 + ...cepted-streams-is-answered-from-a-guess.md | 6 +- issues/toyos-has-its-own-network-stack.md | 3 +- userland/netstack/node/Cargo.toml | 7 +- userland/netstack/node/src/lease/tcp.rs | 6 ++ userland/netstack/node/src/lib.rs | 2 +- userland/netstack/node/src/listeners.rs | 61 +++++++++---- userland/netstack/node/tests/host.rs | 30 ++++-- userland/netstack/node/tests/lease.rs | 91 ++++++++++++++++++- userland/netstack/node/tests/listeners.rs | 58 +++++++++--- 10 files changed, 221 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e4eab6f3e5..f99f7464634 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5918,6 +5918,7 @@ name = "toyos-net-node" version = "0.1.0" dependencies = [ "etherparse", + "mio", "socket2", "toyos-dhcp", "toyos-mdns", diff --git a/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md b/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md index 04244b4194d..4223534d926 100644 --- a/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md +++ b/issues/a-listening-sockets-tcp-nodelay-is-refused-and-an-accepted-streams-is-answered-from-a-guess.md @@ -11,7 +11,11 @@ connections that begin afterwards. Read on macOS 27 on arm64, by libc's `setsockopt` and `getsockopt` over loopback: set before the connection began, the accepted socket has the option; set after the connection was established and before the accept, it does not. `userland/netstack/node/tests/host.rs` -asks the same of every host the suite runs on. +asks the same of every host the suite runs on, and sets the option in the +second case only once the listener is readable, which is the host saying the +connection waits to be accepted: a connect returns before the listener's end +is established, so the answer rests on that event and not on the order +loopback delivers in. A red there prints the host's name and both answers. ToyOS as it ships does neither half: diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index a5bcc75dbe1..fbb134decbb 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -26,6 +26,7 @@ What the node does not yet meet: - With the link down the DHCP client keeps its timers: every 4 to 64 s the node has a deadline, builds a DISCOVER that [udp] refuses, and counts it in `dhcp.tx.discover` and `node.dhcp-unsent`, from `Node::new` on. Exit: `toyos-dhcp`'s client is told the link went down and waits for it, and the node's first DISCOVER is the one that leaves. - No second TCP has been on the far end of a stream: the peer of `userland/netstack/node/tests/streams.rs` is the tests' own script, which acknowledges what arrives in order and loses, reorders and repeats nothing; what is not ours there is `etherparse`, which reads every segment the node emits and builds every one it receives. Exit: netd runs on the node against the host kernel's TCP through slirp in a guest. - The node's places are one number for every client: a program that connects, listens or binds in a loop takes them all, and so does a peer that keeps the connections [tcp] is finishing alive, each of which holds its stream's place (the line below on a client gone with nothing left in its pipe). The node has no word for the program behind a request, as `issues/netstack-lookup-slots-have-no-per-client-share.md` records of netd. Exit: a test of the node in which one program at its share of the places leaves another's connect, listen, accept and bind answered. +- A port is one listener's, whatever address either listen named: a listen on a port a listener holds is refused `ListenRefused::InUse` (`userland/netstack/node/src/listeners.rs`), so [tcp]'s rule that a SYN goes to the listener that named its address before one at every address (LS-09) is not reachable from the node, and one program's two listeners on one port at two addresses, which a host allows one user, are refused too. The node has no word for the program behind a listen: with LS-09 reachable, any program that reaches netstack would take every later connection of another program's listener at every address by naming the machine's address on its port. Exit: a listen carries its program, a second listener on a held port is allowed only to the program that holds every listener on it, and a test of the node finds one program's pair standing, a SYN handed to the one that named its address, and another program's listen on that port refused. - A connection that waits to be accepted holds no place: a listener's one place stands for [tcp]'s `limits::LISTEN_READY` of them, each with at most one receive buffer of text taken when its first byte arrives, 128 x 65,535 bytes or 8.4 MB a listener as netd configures [tcp]. Counting each as a place would let any peer of any listener take every place by handshakes alone and refuse every client its connect, listen and bind; as built a peer fills the queue of the listener it reaches. So the places bound the node's memory only if the shell prices a listener at that product. Exit: the move derives netd's places from a memory figure in which a listener costs its 128 receive buffers, and its body says the figure. - A listener's handshakes in progress are [tcp]'s `limits::LISTEN_PENDING`, each held until `limits::SYNACK_GIVE_UP`, and a SYN past them is dropped: 256 SYNs a minute from addresses that never answer shut a port to every other peer for as long as they keep coming. [tcp] has no SYN cookie and lets no older handshake go for a newer one. Exit: a test in `toyos-net-shard/tcp/tests/` in which a peer's handshake completes while LS-12's flood runs. - The node counts a wake unspent until an accept arrives, and cannot tell an accept on its way from one that will never be sent: an owner that reads a wake and sends no accept is owed one wake fewer from then on, and the last connection waiting is not announced. That is the node's half of `issues/an-accept-that-never-reaches-netstack-strands-its-listener.md`. Exit: that issue's, by a client that cannot spend a wake without its accept arriving, or by an accept that waits in netd and needs no wake. @@ -33,7 +34,7 @@ What the node does not yet meet: - A stream its peer reset is gone from the node at once, so for a client that still holds it `Node::set_nodelay` answers `false` and `Node::nodelay` `None`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers, and libc's `getsockopt` of `TCP_NODELAY`, which asks through `toyos::net::tcp_get_option` (`userland/libc/src/socket.rs`) and is refused where both hosts answer. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. - Every frame and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per frame, as netd's `bridge_piped` is today. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netd is a system call: 1,000 system calls a frame with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one frame makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. Exit: at the move, netd's CPU time per frame on the T14 with 1, 100 and 1,000 idle streams beside one bulk stream is measured, and the change lands if 1,000 idle streams cost more than 12.3 µs a frame over what one costs. The 12.3 µs is the time a full frame takes on a wire of 1 Gbit/s, 1,538 bytes of 8 ns with its preamble and gap: past it the passes alone take longer than the bulk stream's frames take to arrive. It is a threshold by arithmetic and no measurement; by estimate, not measured, 1,000 system calls exceed it. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. -- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. Exit: the specifications hold the three calls and the refusal, and the tests name their scenarios. +- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener that named an address is ended by the node in the pass that finds the lease naming another or none (`a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands`, `userland/netstack/node/tests/lease.rs`), which no specification holds and no host has been read for. Exit: the specifications hold the three calls, the refusal and the end of a listener whose address went, and the tests name their scenarios. - The word each of [udp]'s refusals is answered in on the pipe ABI is `Refused` in `userland/netstack/node/src/datagram.rs`, tested by `each_refusal_is_answered_in_the_pipes_word_for_it` against no reader's scenario; `udp.no-ephemeral-port` has a word and no test that reaches it. Exit: the scenario owed below exists, and the test names its id. - A client's datagram to a broadcast address is refused `udp.broadcast-not-permitted` and answered invalid input, where smoltcp sends it: the pipe ABI has no request that permits it, and std's `set_broadcast` (`sdk/std/sys/net/connection.rs`) answers `Ok` to a permission nothing grants. No program in the tree sends to a broadcast address or sets the option; the sources of the third-party C programs an image builds from a recipe have not been searched. The owner's ruling: "Of course carry the permission why would you even ask. The premise is and always has been to do things properly". The move of netd onto the node does not land with this open. Exit, both before the move, as a stage of their own: that search, for `SO_BROADCAST` and a `sendto` to a broadcast address, is posted with its command and output; and the pipe ABI carries the permission from std's and libc's setters to `Udp::set_broadcast`, a send refused for want of it answered in a word that says permission and not input. - A refusal [udp] logs against a client's call waits in the stack until the node's next `receive`, `fire` or `link`, because only those pass over the stack's reports. Exit, at the move and not after it: the datagram calls end in that pass, or netd drains it once per turn of its loop. diff --git a/userland/netstack/node/Cargo.toml b/userland/netstack/node/Cargo.toml index b880f8589e3..ea9f5b02904 100644 --- a/userland/netstack/node/Cargo.toml +++ b/userland/netstack/node/Cargo.toml @@ -22,6 +22,9 @@ toyos-net-wire = { path = "../../../toyos-net-wire" } # The outside reader of every frame the node emits: a published Ethernet, ARP, # IPv4, UDP and ICMP parser, used as a black box. etherparse = "0.21.0" -# The host's own TCP as an oracle: sets and reads a socket option std has no call -# for on a listener, on Linux, macOS and Windows alike, with no `unsafe`. +# The host's own TCP as an oracle, on Linux, macOS and Windows alike, with no +# `unsafe`: `socket2` sets and reads a socket option std has no call for on a +# listener, and `mio` waits until a listener has a connection to accept, which +# std can learn only by accepting it. +mio = { version = "1", default-features = false, features = ["os-poll", "net"] } socket2 = "0.6" diff --git a/userland/netstack/node/src/lease/tcp.rs b/userland/netstack/node/src/lease/tcp.rs index bd9bcbd731c..5efcc42a2a0 100644 --- a/userland/netstack/node/src/lease/tcp.rs +++ b/userland/netstack/node/src/lease/tcp.rs @@ -76,6 +76,12 @@ impl Stack { Ok((id, held(self.shard.listener_port(id)))) } + /// Whether a listener that named `addr` still listens somewhere: at every address, or at the + /// one the lease names, under probe or held. + pub(crate) fn tcp_listens_at(&self, addr: Ipv4Addr) -> bool { + addr.is_unspecified() || self.address() == Some(addr) + } + /// What a connection whose SYN arrives at `id` from here on starts with. pub(crate) fn tcp_set_listener_options(&mut self, id: ListenerId, options: Options) { held(self.shard.set_listener_options(id, options)); diff --git a/userland/netstack/node/src/lib.rs b/userland/netstack/node/src/lib.rs index a02ec2f6b52..f96ae9732bf 100644 --- a/userland/netstack/node/src/lib.rs +++ b/userland/netstack/node/src/lib.rs @@ -41,7 +41,7 @@ mod name; mod places; mod streams; -pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId, Wake}; +pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerEnd, ListenerId, Wake}; pub use streams::{ConnectRefused, FromClient, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Watch, WriteRefusal}; use alloc::vec; diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs index 6d27272acac..648aaa44091 100644 --- a/userland/netstack/node/src/listeners.rs +++ b/userland/netstack/node/src/listeners.rs @@ -12,9 +12,18 @@ //! The node cannot tell an accept that is on its way from one that will never be sent: an owner //! that reads a wake and sends no accept is owed one wake fewer from then on. //! -//! **A listener lives until its owner lets go**: [`Node::close_listener`], or a wake its pipe -//! refuses, for whatever reason. [tcp] then resets every connection that still waits, so each -//! peer learns at once, and the port is free. +//! **A port is one listener's**, whatever address either listen named: [tcp] hands a SYN to +//! the listener that named its address before one at every address, so a second listen on a +//! held port would take the first's connections, and a listen carries no word for its program +//! by which the node could allow that to the first's owner alone. +//! +//! **A listener lives until its owner lets go or its address goes**: [`Node::close_listener`], +//! a wake its pipe refuses, for whatever reason, or a pass that finds the lease naming another +//! address than the one the listen named, or none. [tcp] then resets every connection that +//! still waits, so each peer learns at once, the port is free and the place is back. The owner +//! reads the end of its wake pipe and the shell reads [`Node::drain_ended_listeners`]. A +//! listener at a lost address is ended and not kept for the address to come back because its +//! owner makes no call a refusal could answer: it only waits for wakes, and none would come. //! //! **A stream starts with the options its connection has**, and those are its listener's as //! they were when its SYN arrived: [tcp] hands them over then (LS-10), the node reads them @@ -51,10 +60,19 @@ pub enum ListenRefused { /// The address named is not this machine's to bind: what `Node::udp_bind` refuses by the /// same rule. NotLocal, - /// The port is another listener's at that address, or no drawn port was free. + /// The port is a listener's, whatever address either named, or no drawn port was free. InUse, } +/// Why the node ended a listener its owner had not closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ListenerEnd { + /// The owner's pipe refused a wake. + Wake(WriteRefusal), + /// The lease names the address the listen named no more. + Address, +} + /// The answer to an accept that took a connection: its stream, established. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Accepted { @@ -78,6 +96,8 @@ pub enum AcceptRefused { struct Listener { bound: toyos_net_tcp::ListenerId, + /// The address its listen named, and the port it holds. + at: Endpoint, owner: Box, /// Wakes written that no accept has spent. unspent: usize, @@ -90,7 +110,7 @@ pub(crate) struct Listeners { live: BTreeMap, /// The next listener's id: none is used twice. next: u64, - refused: VecDeque<(ListenerId, WriteRefusal)>, + ended: VecDeque<(ListenerId, ListenerEnd)>, } impl Node { @@ -101,6 +121,10 @@ impl Node { if self.room() == 0 { return Err(ListenRefused::Full); } + // A drawn port is one [tcp] finds no listener on at any address. + if port.is_some_and(|port| self.listeners.live.values().any(|listener| listener.at.port == port)) { + return Err(ListenRefused::InUse); + } let candidate = || { let [low, high, ..] = draw().to_le_bytes(); u16::from_le_bytes([low, high]) @@ -113,7 +137,7 @@ impl Node { }; let id = ListenerId(self.listeners.next); self.listeners.next = self.listeners.next.saturating_add(1); - self.listeners.live.insert(id, Listener { bound, owner, unspent: 0, options: Options::default() }); + self.listeners.live.insert(id, Listener { bound, at: Endpoint { addr, port }, owner, unspent: 0, options: Options::default() }); Ok((id, port)) } @@ -169,31 +193,35 @@ impl Node { true } - /// Writes every owner the wakes it is owed, and ends a listener whose pipe refuses one. One - /// lookup in [tcp] a listener: nothing of [tcp]'s is moved or offered for sending. + /// Writes every owner the wakes it is owed, and ends a listener whose pipe refuses one or + /// whose address went. One lookup in [tcp] a listener: nothing of [tcp]'s is moved or + /// offered for sending. pub(crate) fn wake_owners(&mut self, now: Instant) { // A listener ended here gives its place back, which another's owner may be owed a wake // for. while self.wake_each(now) {} } - /// One round over the listeners, up to the first whose pipe refuses a wake; `true` ended it. + /// One round over the listeners, up to the first to end; `true` ended one. fn wake_each(&mut self, now: Instant) -> bool { let room = self.room(); let stack = &mut self.stack; - let refused = self.listeners.live.iter_mut().find_map(|(id, listener)| { + let ended = self.listeners.live.iter_mut().find_map(|(id, listener)| { + if !stack.tcp_listens_at(listener.at.addr) { + return Some((*id, ListenerEnd::Address)); + } let owed = stack.tcp_ready(listener.bound).min(room); while listener.unspent < owed { if let Err(refusal) = listener.owner.wake() { - return Some((*id, refusal)); + return Some((*id, ListenerEnd::Wake(refusal))); } listener.unspent = listener.unspent.saturating_add(1); } None }); - let Some((id, refusal)) = refused else { return false }; + let Some((id, end)) = ended else { return false }; self.end_listener(now, id); - self.listeners.refused.push_back((id, refusal)); + self.listeners.ended.push_back((id, end)); true } @@ -202,9 +230,8 @@ impl Node { self.listeners.live.len() } - /// The listeners ended since the last call because their owner's pipe refused a wake, and - /// what it answered. - pub fn drain_refused_listeners(&mut self) -> impl Iterator + '_ { - self.listeners.refused.drain(..) + /// The listeners the node ended since the last call, and why. + pub fn drain_ended_listeners(&mut self) -> impl Iterator + '_ { + self.listeners.ended.drain(..) } } diff --git a/userland/netstack/node/tests/host.rs b/userland/netstack/node/tests/host.rs index 1bd0d9914f2..0638ecc3d7d 100644 --- a/userland/netstack/node/tests/host.rs +++ b/userland/netstack/node/tests/host.rs @@ -3,22 +3,38 @@ //! runs on. Nothing of the node runs here: the sockets are the host kernel's, over loopback. //! `tests/listeners.rs` holds the node to the same two answers //! (`a_stream_starts_with_the_options_its_connection_took_from_its_listener`). +//! +//! **The second answer waits on the listener, not on the connect.** A connect returns when the +//! SYN-ACK arrives; the listener's end is established by the ACK after it, and a host may copy +//! the option then. So the option is set only once the listener is readable, which is the host +//! saying a connection waits to be accepted. -use std::net::{Ipv4Addr, TcpListener, TcpStream}; +use std::net::{Ipv4Addr, TcpStream}; +use std::time::{Duration, Instant}; +use mio::net::TcpListener; +use mio::{Events, Interest, Poll, Token}; use socket2::SockRef; +/// How long a loopback handshake may take before the test fails. +const CEILING: Duration = Duration::from_secs(60); + /// Whether the socket accepted from a listener has `TCP_NODELAY`, the option set on the -/// listener `before` the connection began, or else once it was established and before the -/// accept. A connect that returned is a handshake that ended: the connection waits in the -/// listener's queue, and the accept takes it without waiting. +/// listener `before` the connection began, or else once it waited to be accepted. fn accepted_with_nodelay(before: bool) -> bool { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("a loopback port"); + let mut listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0).into()).expect("a loopback port"); + let mut poll = Poll::new().expect("the host's poller"); + poll.registry().register(&mut listener, Token(0), Interest::READABLE).expect("a listener is a source"); assert!(!SockRef::from(&listener).tcp_nodelay().expect("a listener answers for the option")); if before { SockRef::from(&listener).set_tcp_nodelay(true).expect("a listener takes the option"); } let _peer = TcpStream::connect(listener.local_addr().expect("bound")).expect("the listener's queue takes it"); + let (mut events, start) = (Events::with_capacity(1), Instant::now()); + while events.is_empty() { + let left = CEILING.checked_sub(start.elapsed()).expect("the connection waits at its listener within the ceiling"); + poll.poll(&mut events, Some(left)).expect("the host's poller"); + } if !before { SockRef::from(&listener).set_tcp_nodelay(true).expect("a listener takes the option"); } @@ -29,6 +45,6 @@ fn accepted_with_nodelay(before: bool) -> bool { #[test] fn a_host_gives_a_connection_the_nodelay_its_listener_had_when_it_began() { - assert!(accepted_with_nodelay(true), "set before the connection began, the accepted socket has it"); - assert!(!accepted_with_nodelay(false), "set after the connection was established, it does not"); + let answers = (accepted_with_nodelay(true), accepted_with_nodelay(false)); + assert_eq!(answers, (true, false), "{}: (set before the connection began, set once it waited to be accepted)", std::env::consts::OS); } diff --git a/userland/netstack/node/tests/lease.rs b/userland/netstack/node/tests/lease.rs index 88dcd6f941c..7cf466816ef 100644 --- a/userland/netstack/node/tests/lease.rs +++ b/userland/netstack/node/tests/lease.rs @@ -6,14 +6,16 @@ mod common; +use std::cell::Cell; use std::net::Ipv4Addr; +use std::rc::Rc; use std::time::Duration; use common::*; use toyos_net_ip::AddrState; -use toyos_net_node::{Counter, Event}; +use toyos_net_node::{Counter, Event, ListenRefused, ListenerEnd, ListenerId, Wake, WriteRefusal}; use toyos_net_shard::Refusal; -use toyos_net_wire::Instant; +use toyos_net_wire::{Instant, Port}; const START: Instant = Instant::from_millis(3_600_000); const HOUR: u32 = 3_600; @@ -184,6 +186,91 @@ fn a_second_conflict_takes_a_held_lease_and_declines_it() { assert!(!wire.answers_arp_for(A)); } +/// A listener's owner: whether the node dropped its end of the wake pipe, at which the owner +/// reads the end. +#[derive(Clone, Default)] +struct Owner(Rc>); + +impl Wake for Owner { + fn wake(&mut self) -> Result<(), WriteRefusal> { + Ok(()) + } +} + +impl Drop for Owner { + fn drop(&mut self) { + self.0.set(true); + } +} + +fn listen(wire: &mut Wire, addr: Ipv4Addr, port: u16) -> (ListenerId, Rc>) { + let owner = Owner::default(); + let told = owner.0.clone(); + let (id, _) = wire.node.listen(addr, Port::new(port), Box::new(owner), || panic!("a named port draws nothing")).expect("an address the machine holds and a free port"); + (id, told) +} + +type Loss = fn(&mut Wire); + +/// The three ways a held address goes: the server refuses its renewal, the lease runs out, and +/// another host defends the address twice. +fn losses() -> [(&'static str, Loss); 3] { + fn nak(wire: &mut Wire) { + let id = renewing(wire); + wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message(NAK, id, Ipv4Addr::UNSPECIFIED, &[(54, R.octets().to_vec())]))); + } + fn expiry(wire: &mut Wire) { + assert!(wire.run_until(Duration::from_secs(700), |wire| wire.node.lease().is_none()), "the lease runs out"); + } + fn conflict(wire: &mut Wire) { + let conflict = arp(BROADCAST, false, MAC_B, A, A); + wire.fire(after(100)); + wire.deliver(&conflict); + wire.fire(after(105)); + wire.deliver(&conflict); + } + [("a NAK", nak), ("expiry", expiry), ("a conflict", conflict)] +} + +// A listener that named the machine's address can be reached by nobody once the address is +// gone, and its owner only waits: the node ends it in the pass that finds the address gone, so +// the owner reads the end of its wake pipe, the shell is told why, and the port and the place +// are free. One at every address is at whatever address comes next, and stands. +#[test] +fn a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands() { + for (how, lose) in losses() { + let mut wire = Wire::leased(&terms(600, Some(R))); + let (named, named_told) = listen(&mut wire, A, 22); + let (_, any_told) = listen(&mut wire, Ipv4Addr::UNSPECIFIED, 23); + assert_eq!((wire.node.listeners(), wire.node.held()), (2, 2), "{how}"); + + lose(&mut wire); + assert_eq!((wire.node.lease(), wire.address(A)), (None, None), "{how}"); + let ended: Vec<(ListenerId, ListenerEnd)> = wire.node.drain_ended_listeners().collect(); + assert_eq!((ended, named_told.get(), any_told.get()), (vec![(named, ListenerEnd::Address)], true, false), "{how}"); + assert_eq!((wire.node.listeners(), wire.node.held()), (1, 1), "{how}"); + assert!(!wire.node.close_listener(wire.now, named), "{how}: its id names nothing"); + listen(&mut wire, Ipv4Addr::UNSPECIFIED, 22); + let again = wire.node.listen(A, Port::new(24), Box::new(Owner::default()), || panic!("a named port draws nothing")); + assert_eq!(again.unwrap_err(), ListenRefused::NotLocal, "{how}"); + } +} + +// The address is the lease's for as long as the lease is held: a renewal and a link that went +// and came back leave it, and the listener at it. +#[test] +fn a_listener_at_the_held_address_outlives_a_renewal_and_a_link_that_returns() { + let mut wire = Wire::leased(&terms(600, Some(R))); + let (_, told) = listen(&mut wire, A, 22); + let id = renewing(&mut wire); + wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms(600, Some(R))))); + wire.link(false); + wire.link(true); + let id = xid(wire.last(REQUEST)); + wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); + assert_eq!((wire.node.lease().map(|lease| lease.address), wire.node.listeners(), wire.node.drain_ended_listeners().count(), told.get()), (Some(A), 1, 0, false)); +} + #[test] fn a_router_ip_refuses_is_no_router() { let edge = Ipv4Addr::new(192, 0, 2, 255); diff --git a/userland/netstack/node/tests/listeners.rs b/userland/netstack/node/tests/listeners.rs index 41a1edb77de..53c267fa7f7 100644 --- a/userland/netstack/node/tests/listeners.rs +++ b/userland/netstack/node/tests/listeners.rs @@ -20,7 +20,7 @@ use std::time::Duration; use common::{arp, terms, Segment, Wire, A, ELSEWHERE, MAC, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerEnd, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; use toyos_net_tcp::{limits, Counter, Endpoint}; use toyos_net_wire::{Instant, Port}; @@ -703,8 +703,8 @@ fn a_wake_the_owners_pipe_refuses_ends_the_listener() { let (id, owner) = net.listen(SSH); owner.borrow_mut().refusal = Some(refusal); net.handshake(P1); - let ended: Vec<(ListenerId, WriteRefusal)> = net.node.drain_refused_listeners().collect(); - assert_eq!(ended, [(id, refusal)]); + let ended: Vec<(ListenerId, ListenerEnd)> = net.node.drain_ended_listeners().collect(); + assert_eq!(ended, [(id, ListenerEnd::Wake(refusal))]); assert!(owner.borrow().dropped && net.last(P1).rst, "{refusal:?}: {:?}", net.heard); assert_eq!((net.node.listeners(), net.node.held()), (0, 0), "{refusal:?}"); net.listen(SSH); @@ -735,8 +735,7 @@ fn a_listen_on_a_taken_port_is_refused_and_a_drawn_port_listens() { // A listener is at the address its listen named. One the machine does not hold is refused, as // a datagram socket's bind is, and nothing listens: RFC 9293 §3.10.7.1, a SYN for a port // nothing listens on is answered . One the machine -// holds listens beside a listener at every address on the same port, and [tcp] hands a SYN to -// the one that named the address (LS-09). +// holds listens. #[test] fn a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds() { let mut net = Net::new(); @@ -749,14 +748,47 @@ fn a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds() { let reset = net.last(P1); assert!(reset.rst && (reset.seq, reset.ack) == (0, Some(ISS + 1)), "{reset:?}"); - let (_, any) = net.listen(SSH); let named = Owner::default(); - let (_, port) = net.node.listen(A, Port::new(SSH), Box::new(WakeEnd(named.clone())), draw(&mut net.draws)).expect("the machine's address, and a port no listener at it holds"); - assert_eq!((port.get(), net.node.listeners()), (SSH, 2)); + let (_, port) = net.node.listen(A, Port::new(SSH), Box::new(WakeEnd(named.clone())), draw(&mut net.draws)).expect("the machine's address, and a port no listener holds"); + assert_eq!((port.get(), net.node.listeners()), (SSH, 1)); net.handshake(P2); - assert_eq!((wakes(&named), wakes(&any)), (1, 0)); - let again = net.node.listen(A, Port::new(SSH), Box::new(WakeEnd(Owner::default())), draw(&mut net.draws)); - assert_eq!(again.unwrap_err(), ListenRefused::InUse); + assert_eq!(wakes(&named), 1); +} + +/// A listener at `first` holds the port, and a listen at `second` on it is refused with nothing +/// made: the connection that comes next is the first listener's. +fn a_port_is_one_listeners(first: Ipv4Addr, second: Ipv4Addr) { + let mut net = Net::new(); + let holder = Owner::default(); + let (id, _) = net.node.listen(first, Port::new(SSH), Box::new(WakeEnd(holder.clone())), draw(&mut net.draws)).expect("a port no listener holds"); + let refused = Owner::default(); + let answer = net.node.listen(second, Port::new(SSH), Box::new(WakeEnd(refused.clone())), draw(&mut net.draws)); + assert_eq!(answer.unwrap_err(), ListenRefused::InUse, "{second} beside {first}"); + assert!(refused.borrow().dropped); + assert_eq!((net.node.listeners(), net.node.held()), (1, 1)); + net.handshake(P1); + assert_eq!((wakes(&holder), wakes(&refused)), (1, 0)); + net.accepts(id, P1); +} + +// [tcp] hands a SYN to the listener that named its address before one at every address +// (LS-09), so a second listen on a held port that named the machine's address would take every +// connection of the first from then on. A listen carries no word for its program, so the node +// cannot tell the first listener's owner from another: the port is one listener's, as a +// datagram socket's port is one socket's. +#[test] +fn a_listen_at_the_machines_address_takes_no_port_held_at_every_address() { + a_port_is_one_listeners(ANY, A); +} + +#[test] +fn a_listen_at_every_address_takes_no_port_held_at_the_machines_address() { + a_port_is_one_listeners(A, ANY); +} + +#[test] +fn a_second_listen_at_the_machines_address_takes_no_port_held_there() { + a_port_is_one_listeners(A, A); } // A listener ended for a wake its owner's pipe refused gives its place back in the pass that @@ -776,8 +808,8 @@ fn a_listener_ended_for_its_wake_gives_its_place_to_another_in_the_same_pass() { let ack = net.iss(P3).wrapping_add(1); net.node.receive(net.now, &frame(P3, SSH, ISS + 1, Some(ack), 0, &[]), draw(&mut net.draws)); assert_eq!(wakes(&owner), 2); - let ended: Vec<(ListenerId, WriteRefusal)> = net.node.drain_refused_listeners().collect(); - assert_eq!((ended, net.node.listeners(), net.node.held()), (vec![(first, WriteRefusal::Gone)], 1, 1)); + let ended: Vec<(ListenerId, ListenerEnd)> = net.node.drain_ended_listeners().collect(); + assert_eq!((ended, net.node.listeners(), net.node.held()), (vec![(first, ListenerEnd::Wake(WriteRefusal::Gone))], 1, 1)); } // A datagram socket is something a client makes the node hold, its two queues of sixteen From c27cc36aa18ca4d857acdd5ab90dfd163ef75d91 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 22:35:48 +0200 Subject: [PATCH 6/7] toyos-net-node: a listener stands by the predicate that admitted it Shard::listens_at is the one rule for an address a listener may be at: a listen is refused by it, and the pass that writes the wakes ends a listener it no longer holds for. No listener is made that the next pass ends, and none stands where a listen would be refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- issues/toyos-has-its-own-network-stack.md | 2 +- toyos-net-shard/src/lib.rs | 12 +++++++++--- userland/netstack/node/src/lease/tcp.rs | 6 ------ userland/netstack/node/src/listeners.rs | 17 +++++++++-------- userland/netstack/node/tests/lease.rs | 4 ++-- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index 04718f7528f..0f5f79a8638 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -34,7 +34,7 @@ What the node does not yet meet: - A stream its peer reset is gone from the node at once, so for a client that still holds it `Node::set_nodelay` answers `false` and `Node::nodelay` `None`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers, and libc's `getsockopt` of `TCP_NODELAY`, which asks through `toyos::net::tcp_get_option` (`userland/libc/src/socket.rs`) and is refused where both hosts answer. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. - Every frame and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per frame, as netd's `bridge_piped` is today. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netd is a system call: 1,000 system calls a frame with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one frame makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. Exit: at the move, netd's CPU time per frame on the T14 with 1, 100 and 1,000 idle streams beside one bulk stream is measured, and the change lands if 1,000 idle streams cost more than 12.3 µs a frame over what one costs. The 12.3 µs is the time a full frame takes on a wire of 1 Gbit/s, 1,538 bytes of 8 ns with its preamble and gap: past it the passes alone take longer than the bulk stream's frames take to arrive. It is a threshold by arithmetic and no measurement; by estimate, not measured, 1,000 system calls exceed it. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. -- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener that named an address is ended by the node in the pass that finds the lease naming another or none (`a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands`, `userland/netstack/node/tests/lease.rs`), which no specification holds and no host has been read for. Exit: the specifications hold the three calls, the refusal and the end of a listener whose address went, and the tests name their scenarios. +- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener that named an address is ended by the node in the pass that finds [ip] holding it usable no more, by the predicate that refused the listen (`a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands`, `userland/netstack/node/tests/lease.rs`), which no specification holds and no host has been read for. Exit: the specifications hold the three calls, the refusal and the end of a listener whose address went, and the tests name their scenarios. - The word each of [udp]'s refusals is answered in on the pipe ABI is `Refused` in `userland/netstack/node/src/datagram.rs`, tested by `each_refusal_is_answered_in_the_pipes_word_for_it` against no reader's scenario; `udp.no-ephemeral-port` has a word and no test that reaches it. Exit: the scenario owed below exists, and the test names its id. - A client's datagram to a broadcast address is refused `udp.broadcast-not-permitted` and answered invalid input, where smoltcp sends it: the pipe ABI has no request that permits it, and std's `set_broadcast` (`sdk/std/sys/net/connection.rs`) answers `Ok` to a permission nothing grants. No program in the tree sends to a broadcast address or sets the option; the sources of the third-party C programs an image builds from a recipe have not been searched. The owner's ruling: "Of course carry the permission why would you even ask. The premise is and always has been to do things properly". The move of netd onto the node does not land with this open. Exit, both before the move, as a stage of their own: that search, for `SO_BROADCAST` and a `sendto` to a broadcast address, is posted with its command and output; and the pipe ABI carries the permission from std's and libc's setters to `Udp::set_broadcast`, a send refused for want of it answered in a word that says permission and not input. - A refusal [udp] logs against a client's call waits in the stack until the node's next `receive`, `fire` or `link`, because only those pass over the stack's reports. Exit, at the move and not after it: the datagram calls end in that pass, or netd drains it once per turn of its loop. diff --git a/toyos-net-shard/src/lib.rs b/toyos-net-shard/src/lib.rs index d7ee5fb50b2..ed06f07bc62 100644 --- a/toyos-net-shard/src/lib.rs +++ b/toyos-net-shard/src/lib.rs @@ -476,10 +476,16 @@ impl Shard { connected } - /// `addr` is the local address to listen on, UNSPECIFIED for any, and otherwise one [ip] - /// holds assigned or announcing, as a datagram socket's is; port 0 takes `random`'s draws. + /// Whether `addr` is one a listener may be at: UNSPECIFIED for any, and otherwise one [ip] + /// holds assigned or announcing, as a datagram socket's is. + pub fn listens_at(&self, addr: Ipv4Addr) -> bool { + addr.is_unspecified() || self.ip.is_assigned(addr) + } + + /// `addr` is the local address to listen on, one [`Self::listens_at`]; port 0 takes + /// `random`'s draws. pub fn listen(&mut self, addr: Ipv4Addr, port: Option, random: impl FnMut() -> u16) -> Result { - if !addr.is_unspecified() && !self.ip.is_assigned(addr) { + if !self.listens_at(addr) { return Err(ListenError::NotLocal); } self.tcp.listen(addr, port, random).map_err(ListenError::Tcp) diff --git a/userland/netstack/node/src/lease/tcp.rs b/userland/netstack/node/src/lease/tcp.rs index 5efcc42a2a0..bd9bcbd731c 100644 --- a/userland/netstack/node/src/lease/tcp.rs +++ b/userland/netstack/node/src/lease/tcp.rs @@ -76,12 +76,6 @@ impl Stack { Ok((id, held(self.shard.listener_port(id)))) } - /// Whether a listener that named `addr` still listens somewhere: at every address, or at the - /// one the lease names, under probe or held. - pub(crate) fn tcp_listens_at(&self, addr: Ipv4Addr) -> bool { - addr.is_unspecified() || self.address() == Some(addr) - } - /// What a connection whose SYN arrives at `id` from here on starts with. pub(crate) fn tcp_set_listener_options(&mut self, id: ListenerId, options: Options) { held(self.shard.set_listener_options(id, options)); diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs index 648aaa44091..327af671f75 100644 --- a/userland/netstack/node/src/listeners.rs +++ b/userland/netstack/node/src/listeners.rs @@ -18,12 +18,13 @@ //! by which the node could allow that to the first's owner alone. //! //! **A listener lives until its owner lets go or its address goes**: [`Node::close_listener`], -//! a wake its pipe refuses, for whatever reason, or a pass that finds the lease naming another -//! address than the one the listen named, or none. [tcp] then resets every connection that -//! still waits, so each peer learns at once, the port is free and the place is back. The owner -//! reads the end of its wake pipe and the shell reads [`Node::drain_ended_listeners`]. A -//! listener at a lost address is ended and not kept for the address to come back because its -//! owner makes no call a refusal could answer: it only waits for wakes, and none would come. +//! a wake its pipe refuses, for whatever reason, or a pass that finds the address the listen +//! named one a listen would now be refused for, [`ListenRefused::NotLocal`]. [tcp] then resets +//! every connection that still waits, so each peer learns at once, the port is free and the +//! place is back. The owner reads the end of its wake pipe and the shell reads +//! [`Node::drain_ended_listeners`]. A listener at a lost address is ended and not kept for the +//! address to come back because its owner makes no call a refusal could answer: it only waits +//! for wakes, and none would come. //! //! **A stream starts with the options its connection has**, and those are its listener's as //! they were when its SYN arrived: [tcp] hands them over then (LS-10), the node reads them @@ -69,7 +70,7 @@ pub enum ListenRefused { pub enum ListenerEnd { /// The owner's pipe refused a wake. Wake(WriteRefusal), - /// The lease names the address the listen named no more. + /// The address the listen named is the machine's no more. Address, } @@ -207,7 +208,7 @@ impl Node { let room = self.room(); let stack = &mut self.stack; let ended = self.listeners.live.iter_mut().find_map(|(id, listener)| { - if !stack.tcp_listens_at(listener.at.addr) { + if !stack.shard().listens_at(listener.at.addr) { return Some((*id, ListenerEnd::Address)); } let owed = stack.tcp_ready(listener.bound).min(room); diff --git a/userland/netstack/node/tests/lease.rs b/userland/netstack/node/tests/lease.rs index 7cf466816ef..6933f9ee124 100644 --- a/userland/netstack/node/tests/lease.rs +++ b/userland/netstack/node/tests/lease.rs @@ -256,8 +256,8 @@ fn a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_st } } -// The address is the lease's for as long as the lease is held: a renewal and a link that went -// and came back leave it, and the listener at it. +// [ip] holds the address usable for as long as the lease is held: a renewal and a link that +// went and came back leave it, and the listener at it. #[test] fn a_listener_at_the_held_address_outlives_a_renewal_and_a_link_that_returns() { let mut wire = Wire::leased(&terms(600, Some(R))); From e737052a54e2b78446768181c76cec3702ed52b1 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 8 Oct 2026 23:08:25 +0200 Subject: [PATCH 7/7] Review round 3 of #777: a listener stands while its address is lost Round 2 ended a listener in the first pass after the machine lost the address its listen named, and nothing brought it back. What takes the address is the wire's doing and is not authenticated: a NAK, a server away for longer than the lease has left, two ARP frames inside the defend interval. The lease returns seconds later, as a rule at the same address, and no unchanged program listens again because its address was away. So a loss the wire could already cause for seconds became the loss of a service for good. Removed whole: the check in wake_each, ListenerEnd (the drain carries the refusal of the wake again), Shard::listens_at back into Shard::listen, and Listener::at down to the port the one-listener-a-port rule reads. Nothing replaces it: [ip] takes no segment for an address it does not hold, and [tcp] still has the listener when the address is back. A listener at a lost address stands as a datagram socket bound to it does. a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back, beside the lease's tests, over a NAK, expiry and a conflict, and through a renewal and a link that returns before them: both listeners stand with two places, nothing is drained and neither owner's end is dropped; a SYN to the lost address is answered by nothing and wakes nobody; a listen there on another port is NotLocal; and once the lease is held again a handshake completes and wakes the named listener's owner once. Red against the round 2 source. It replaces round 2's two tests. common::outside reads a TCP segment for it. tests/host.rs: an interrupted poll is one more turn of the loop. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RvnWQFcMuGqTHYhvSnTe8A --- issues/toyos-has-its-own-network-stack.md | 2 +- toyos-net-shard/src/lib.rs | 12 +- userland/netstack/node/src/lib.rs | 2 +- userland/netstack/node/src/listeners.rs | 52 +++----- userland/netstack/node/tests/common/mod.rs | 13 +- userland/netstack/node/tests/host.rs | 6 +- userland/netstack/node/tests/lease.rs | 132 ++++++++++++++------- userland/netstack/node/tests/listeners.rs | 10 +- 8 files changed, 129 insertions(+), 100 deletions(-) diff --git a/issues/toyos-has-its-own-network-stack.md b/issues/toyos-has-its-own-network-stack.md index 0f5f79a8638..50124b20909 100644 --- a/issues/toyos-has-its-own-network-stack.md +++ b/issues/toyos-has-its-own-network-stack.md @@ -34,7 +34,7 @@ What the node does not yet meet: - A stream its peer reset is gone from the node at once, so for a client that still holds it `Node::set_nodelay` answers `false` and `Node::nodelay` `None`: `issues/a-stream-its-peer-reset-refuses-the-option-requests-a-host-answers.md`, carried by the node as built. std's `nodelay()` asks netd nothing: it answers from the value its `TcpStream` holds (`sdk/std/sys/net/connection.rs`). What the node carries is `set_nodelay` on a reset stream, refused where macOS refuses it under another kind and Linux answers, and libc's `getsockopt` of `TCP_NODELAY`, which asks through `toyos::net::tcp_get_option` (`userland/libc/src/socket.rs`) and is refused where both hosts answer. The node cannot hold the stream for the client: its two pipe ends are all it has of the client, letting both go is how the client learns of the reset, and with both gone nothing tells it when the client leaves, so a kept stream would be kept for ever. Exit, outside the node: that issue's guest test is green, and a `getsockopt` of `TCP_NODELAY` on a stream its peer reset is answered, by libc from a value its socket holds as std's does or by a pipe ABI that gives netd a handle whose other end's leaving the kernel reports. - Every frame and every deadline ends in a pass over every stream, every transmit opportunity walks every stream twice (once for the count each peer address keeps alive, once to pass those whose connect is not answered), and `Node::next_deadline` reads every stream: linear in streams per frame, as netd's `bridge_piped` is today. For an idle stream a pass is one `recv_with`, two `status` and one read of its client's pipe, which in netd is a system call: 1,000 system calls a frame with 1,000 idle streams. A pass's `recv_with` also re-files its connection's deadline and offers it to the transmit round, so one frame makes every idle stream eligible and the next opportunity serves each to learn it has nothing. [tcp] cannot say which connections a segment, a timer, an ICMP error or a failed next hop moved: it reports `drain_eligible` and `drain_gone`, which are the round's. The change: [tcp] marks a held connection in `for_conn`, `tick`, `icmp` and `end` and hands the marks out once (`drain_moved`); the node maps a connection to its stream, keeps its deadlines ordered, and passes only the streams [tcp] moved, the one a pipe's wake names, and those due. Exit: at the move, netd's CPU time per frame on the T14 with 1, 100 and 1,000 idle streams beside one bulk stream is measured, and the change lands if 1,000 idle streams cost more than 12.3 µs a frame over what one costs. The 12.3 µs is the time a full frame takes on a wire of 1 Gbit/s, 1,538 bytes of 8 ns with its preamble and gap: past it the passes alone take longer than the bulk stream's frames take to arrive. It is a threshold by arithmetic and no measurement; by estimate, not measured, 1,000 system calls exceed it. - A client that is gone with nothing left in its pipe is finished by [tcp]'s rules for a user who let go: a reset once it has been idle 60 s, where netd on smoltcp reset it 100 s after the client left whatever the peer said. A peer that acknowledges a byte a minute holds such a connection for as long as it has bytes to acknowledge. Exit: the T14 or a guest shows such a peer holding one, and [tcp]'s rule takes a bound from the client's leaving; or the owner rules the idle bound is the one. -- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener that named an address is ended by the node in the pass that finds [ip] holding it usable no more, by the predicate that refused the listen (`a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands`, `userland/netstack/node/tests/lease.rs`), which no specification holds and no host has been read for. Exit: the specifications hold the three calls, the refusal and the end of a listener whose address went, and the tests name their scenarios. +- `Tcp::ready`, `Tcp::orphans` and `Tcp::options` are additions to [tcp] the specifications do not hold, made for the node's wakes, its places and the copy a stream keeps of its connection's options; their tests (`toyos-net-shard/tcp/tests/held.rs`) name no scenario id. `Shard::listen` refuses an address [ip] does not hold usable, by the rule [udp]'s bind refuses by (`udp.bind-address-not-local`), and counts nothing for it; the node's test of it (`a_listener_is_at_the_address_it_named_and_only_one_the_machine_holds`) names no scenario either. A listener whose address the machine loses stands as a datagram socket bound to it does, with its port and its place and its owner told nothing, and answers again once the address is back (`a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back`, `userland/netstack/node/tests/lease.rs`); no host was read for it. Exit: the specifications hold the three calls and the refusal, and the tests name their scenarios. - The word each of [udp]'s refusals is answered in on the pipe ABI is `Refused` in `userland/netstack/node/src/datagram.rs`, tested by `each_refusal_is_answered_in_the_pipes_word_for_it` against no reader's scenario; `udp.no-ephemeral-port` has a word and no test that reaches it. Exit: the scenario owed below exists, and the test names its id. - A client's datagram to a broadcast address is refused `udp.broadcast-not-permitted` and answered invalid input, where smoltcp sends it: the pipe ABI has no request that permits it, and std's `set_broadcast` (`sdk/std/sys/net/connection.rs`) answers `Ok` to a permission nothing grants. No program in the tree sends to a broadcast address or sets the option; the sources of the third-party C programs an image builds from a recipe have not been searched. The owner's ruling: "Of course carry the permission why would you even ask. The premise is and always has been to do things properly". The move of netd onto the node does not land with this open. Exit, both before the move, as a stage of their own: that search, for `SO_BROADCAST` and a `sendto` to a broadcast address, is posted with its command and output; and the pipe ABI carries the permission from std's and libc's setters to `Udp::set_broadcast`, a send refused for want of it answered in a word that says permission and not input. - A refusal [udp] logs against a client's call waits in the stack until the node's next `receive`, `fire` or `link`, because only those pass over the stack's reports. Exit, at the move and not after it: the datagram calls end in that pass, or netd drains it once per turn of its loop. diff --git a/toyos-net-shard/src/lib.rs b/toyos-net-shard/src/lib.rs index ed06f07bc62..d7ee5fb50b2 100644 --- a/toyos-net-shard/src/lib.rs +++ b/toyos-net-shard/src/lib.rs @@ -476,16 +476,10 @@ impl Shard { connected } - /// Whether `addr` is one a listener may be at: UNSPECIFIED for any, and otherwise one [ip] - /// holds assigned or announcing, as a datagram socket's is. - pub fn listens_at(&self, addr: Ipv4Addr) -> bool { - addr.is_unspecified() || self.ip.is_assigned(addr) - } - - /// `addr` is the local address to listen on, one [`Self::listens_at`]; port 0 takes - /// `random`'s draws. + /// `addr` is the local address to listen on, UNSPECIFIED for any, and otherwise one [ip] + /// holds assigned or announcing, as a datagram socket's is; port 0 takes `random`'s draws. pub fn listen(&mut self, addr: Ipv4Addr, port: Option, random: impl FnMut() -> u16) -> Result { - if !self.listens_at(addr) { + if !addr.is_unspecified() && !self.ip.is_assigned(addr) { return Err(ListenError::NotLocal); } self.tcp.listen(addr, port, random).map_err(ListenError::Tcp) diff --git a/userland/netstack/node/src/lib.rs b/userland/netstack/node/src/lib.rs index f842572af0f..60b4e21cec7 100644 --- a/userland/netstack/node/src/lib.rs +++ b/userland/netstack/node/src/lib.rs @@ -44,7 +44,7 @@ mod places; mod resolve; mod streams; -pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerEnd, ListenerId, Wake}; +pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId, Wake}; pub use streams::{ConnectRefused, FromClient, PipeEnd, Pipes, ReadRefusal, StreamEvent, StreamId, ToClient, Watch, WriteRefusal}; use alloc::vec; diff --git a/userland/netstack/node/src/listeners.rs b/userland/netstack/node/src/listeners.rs index 327af671f75..552c962a6ac 100644 --- a/userland/netstack/node/src/listeners.rs +++ b/userland/netstack/node/src/listeners.rs @@ -17,14 +17,11 @@ //! held port would take the first's connections, and a listen carries no word for its program //! by which the node could allow that to the first's owner alone. //! -//! **A listener lives until its owner lets go or its address goes**: [`Node::close_listener`], -//! a wake its pipe refuses, for whatever reason, or a pass that finds the address the listen -//! named one a listen would now be refused for, [`ListenRefused::NotLocal`]. [tcp] then resets -//! every connection that still waits, so each peer learns at once, the port is free and the -//! place is back. The owner reads the end of its wake pipe and the shell reads -//! [`Node::drain_ended_listeners`]. A listener at a lost address is ended and not kept for the -//! address to come back because its owner makes no call a refusal could answer: it only waits -//! for wakes, and none would come. +//! **A listener lives until its owner lets go**: [`Node::close_listener`], or a wake its pipe +//! refuses, for whatever reason. [tcp] then resets every connection that still waits, so each +//! peer learns at once, and the port is free. Nothing the wire does ends one: a listener +//! whose address the machine loses stands, with its port and its place, and is handed the +//! next SYN once the address is the machine's again. //! //! **A stream starts with the options its connection has**, and those are its listener's as //! they were when its SYN arrived: [tcp] hands them over then (LS-10), the node reads them @@ -65,15 +62,6 @@ pub enum ListenRefused { InUse, } -/// Why the node ended a listener its owner had not closed. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ListenerEnd { - /// The owner's pipe refused a wake. - Wake(WriteRefusal), - /// The address the listen named is the machine's no more. - Address, -} - /// The answer to an accept that took a connection: its stream, established. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Accepted { @@ -97,8 +85,7 @@ pub enum AcceptRefused { struct Listener { bound: toyos_net_tcp::ListenerId, - /// The address its listen named, and the port it holds. - at: Endpoint, + port: Port, owner: Box, /// Wakes written that no accept has spent. unspent: usize, @@ -111,7 +98,7 @@ pub(crate) struct Listeners { live: BTreeMap, /// The next listener's id: none is used twice. next: u64, - ended: VecDeque<(ListenerId, ListenerEnd)>, + ended: VecDeque<(ListenerId, WriteRefusal)>, } impl Node { @@ -123,7 +110,7 @@ impl Node { return Err(ListenRefused::Full); } // A drawn port is one [tcp] finds no listener on at any address. - if port.is_some_and(|port| self.listeners.live.values().any(|listener| listener.at.port == port)) { + if port.is_some_and(|port| self.listeners.live.values().any(|listener| listener.port == port)) { return Err(ListenRefused::InUse); } let candidate = || { @@ -138,7 +125,7 @@ impl Node { }; let id = ListenerId(self.listeners.next); self.listeners.next = self.listeners.next.saturating_add(1); - self.listeners.live.insert(id, Listener { bound, at: Endpoint { addr, port }, owner, unspent: 0, options: Options::default() }); + self.listeners.live.insert(id, Listener { bound, port, owner, unspent: 0, options: Options::default() }); Ok((id, port)) } @@ -194,35 +181,31 @@ impl Node { true } - /// Writes every owner the wakes it is owed, and ends a listener whose pipe refuses one or - /// whose address went. One lookup in [tcp] a listener: nothing of [tcp]'s is moved or - /// offered for sending. + /// Writes every owner the wakes it is owed, and ends a listener whose pipe refuses one. One + /// lookup in [tcp] a listener: nothing of [tcp]'s is moved or offered for sending. pub(crate) fn wake_owners(&mut self, now: Instant) { // A listener ended here gives its place back, which another's owner may be owed a wake // for. while self.wake_each(now) {} } - /// One round over the listeners, up to the first to end; `true` ended one. + /// One round over the listeners, up to the first whose pipe refuses a wake; `true` ended it. fn wake_each(&mut self, now: Instant) -> bool { let room = self.room(); let stack = &mut self.stack; let ended = self.listeners.live.iter_mut().find_map(|(id, listener)| { - if !stack.shard().listens_at(listener.at.addr) { - return Some((*id, ListenerEnd::Address)); - } let owed = stack.tcp_ready(listener.bound).min(room); while listener.unspent < owed { if let Err(refusal) = listener.owner.wake() { - return Some((*id, ListenerEnd::Wake(refusal))); + return Some((*id, refusal)); } listener.unspent = listener.unspent.saturating_add(1); } None }); - let Some((id, end)) = ended else { return false }; + let Some((id, refusal)) = ended else { return false }; self.end_listener(now, id); - self.listeners.ended.push_back((id, end)); + self.listeners.ended.push_back((id, refusal)); true } @@ -231,8 +214,9 @@ impl Node { self.listeners.live.len() } - /// The listeners the node ended since the last call, and why. - pub fn drain_ended_listeners(&mut self) -> impl Iterator + '_ { + /// The listeners ended since the last call because their owner's pipe refused a wake, and + /// what it answered. + pub fn drain_ended_listeners(&mut self) -> impl Iterator + '_ { self.listeners.ended.drain(..) } } diff --git a/userland/netstack/node/tests/common/mod.rs b/userland/netstack/node/tests/common/mod.rs index 4df477177f0..b39bd13f051 100644 --- a/userland/netstack/node/tests/common/mod.rs +++ b/userland/netstack/node/tests/common/mod.rs @@ -46,6 +46,7 @@ pub enum Seen { /// A datagram from port 68 to port 67, and its payload. Dhcp { to: [u8; 6], source: Ipv4Addr, destination: Ipv4Addr, message: Vec }, EchoReply { source: Ipv4Addr, destination: Ipv4Addr, id: u16, seq: u16, data: Vec }, + Tcp(Segment), } impl Seen { @@ -53,7 +54,7 @@ impl Seen { pub fn dhcp(&self) -> Option<(u8, &[u8])> { match self { Seen::Dhcp { message, .. } => Some((option(message, 53).expect("a message type")[0], message.as_slice())), - Seen::Arp { .. } | Seen::EchoReply { .. } => None, + Seen::Arp { .. } | Seen::EchoReply { .. } | Seen::Tcp(_) => None, } } @@ -61,6 +62,7 @@ impl Seen { pub fn source(&self) -> Option { match self { Seen::Dhcp { source, .. } | Seen::EchoReply { source, .. } => Some(*source), + Seen::Tcp(_) => Some(A), Seen::Arp { .. } => None, } } @@ -81,8 +83,8 @@ fn v4(bytes: &[u8]) -> Ipv4Addr { } /// The outside reading of a frame the node emitted: Ethernet II from the node's MAC, then ARP, a -/// DHCP client's datagram or an echo reply, each with the lengths and checksums `etherparse` -/// computes. Anything else the node has no business sending here. +/// DHCP client's datagram, an echo reply or a TCP segment, each with the lengths and checksums +/// `etherparse` computes. Anything else the node has no business sending here. pub fn outside(frame: &[u8], mac: [u8; 6]) -> Seen { let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); let Some(LinkSlice::Ethernet2(ethernet)) = &packet.link else { panic!("{:?}", packet.link) }; @@ -114,7 +116,8 @@ pub fn outside(frame: &[u8], mac: [u8; 6]) -> Seen { let Icmpv4Type::EchoReply(echo) = icmp.icmp_type() else { panic!("{:?}", icmp.icmp_type()) }; Seen::EchoReply { source, destination, id: echo.id, seq: echo.seq, data: icmp.payload().to_vec() } } - other => panic!("neither UDP nor ICMP: {other:?}"), + TransportSlice::Tcp(_) => Seen::Tcp(segment(frame).expect("a TCP segment")), + other => panic!("neither UDP, ICMP nor TCP: {other:?}"), } } other => panic!("neither ARP nor IPv4: {other:?}"), @@ -122,7 +125,7 @@ pub fn outside(frame: &[u8], mac: [u8; 6]) -> Seen { } /// A TCP segment the node emitted, as `etherparse` read it. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Segment { /// Where its frame and its datagram went. pub to_mac: [u8; 6], diff --git a/userland/netstack/node/tests/host.rs b/userland/netstack/node/tests/host.rs index 0638ecc3d7d..431e4bb0df7 100644 --- a/userland/netstack/node/tests/host.rs +++ b/userland/netstack/node/tests/host.rs @@ -9,6 +9,7 @@ //! the option then. So the option is set only once the listener is readable, which is the host //! saying a connection waits to be accepted. +use std::io::ErrorKind; use std::net::{Ipv4Addr, TcpStream}; use std::time::{Duration, Instant}; @@ -33,7 +34,10 @@ fn accepted_with_nodelay(before: bool) -> bool { let (mut events, start) = (Events::with_capacity(1), Instant::now()); while events.is_empty() { let left = CEILING.checked_sub(start.elapsed()).expect("the connection waits at its listener within the ceiling"); - poll.poll(&mut events, Some(left)).expect("the host's poller"); + match poll.poll(&mut events, Some(left)) { + Err(interrupted) if interrupted.kind() == ErrorKind::Interrupted => {} + polled => polled.expect("the host's poller"), + } } if !before { SockRef::from(&listener).set_tcp_nodelay(true).expect("a listener takes the option"); diff --git a/userland/netstack/node/tests/lease.rs b/userland/netstack/node/tests/lease.rs index 6933f9ee124..48ac5ee2275 100644 --- a/userland/netstack/node/tests/lease.rs +++ b/userland/netstack/node/tests/lease.rs @@ -12,8 +12,9 @@ use std::rc::Rc; use std::time::Duration; use common::*; +use etherparse::{PacketBuilder, TcpOptionElement}; use toyos_net_ip::AddrState; -use toyos_net_node::{Counter, Event, ListenRefused, ListenerEnd, ListenerId, Wake, WriteRefusal}; +use toyos_net_node::{Counter, Event, ListenRefused, Wake, WriteRefusal}; use toyos_net_shard::Refusal; use toyos_net_wire::{Instant, Port}; @@ -186,34 +187,67 @@ fn a_second_conflict_takes_a_held_lease_and_declines_it() { assert!(!wire.answers_arp_for(A)); } -/// A listener's owner: whether the node dropped its end of the wake pipe, at which the owner -/// reads the end. -#[derive(Clone, Default)] -struct Owner(Rc>); +/// What a listener's owner saw of its wake pipe: the wakes written, and whether the node dropped +/// its end, at which the owner reads the end. +#[derive(Default)] +struct Told { + wakes: Cell, + dropped: Cell, +} + +struct Owner(Rc); impl Wake for Owner { fn wake(&mut self) -> Result<(), WriteRefusal> { + self.0.wakes.set(self.0.wakes.get() + 1); Ok(()) } } impl Drop for Owner { fn drop(&mut self) { - self.0.set(true); + self.0.dropped.set(true); } } -fn listen(wire: &mut Wire, addr: Ipv4Addr, port: u16) -> (ListenerId, Rc>) { - let owner = Owner::default(); - let told = owner.0.clone(); - let (id, _) = wire.node.listen(addr, Port::new(port), Box::new(owner), || panic!("a named port draws nothing")).expect("an address the machine holds and a free port"); - (id, told) +fn listen(wire: &mut Wire, addr: Ipv4Addr, port: u16) -> Result, ListenRefused> { + let told = Rc::new(Told::default()); + wire.node.listen(addr, Port::new(port), Box::new(Owner(told.clone())), || panic!("a named port draws nothing")).map(|_| told) +} + +/// Where the peer of the listeners is, at `MAC_B`. +const PEER: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); +/// The peer's initial sequence number. +const ISS: u32 = 5000; + +/// A segment from the peer's port 40001 to the node's port `to`, built by `etherparse`: a SYN +/// with an MSS option, or the ACK of `ack`. +fn from_peer(to: u16, ack: Option) -> Vec { + let step = PacketBuilder::ethernet2(MAC_B, MAC).ipv4(PEER.octets(), A.octets(), 64); + let step = match ack { + None => step.tcp(40_001, to, ISS, 65_535).syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(), + Some(ack) => step.tcp(40_001, to, ISS + 1, 65_535).ack(ack), + }; + let mut frame = Vec::new(); + step.write(&mut frame, &[]).unwrap(); + frame +} + +/// The peer's SYN to the node's port `to`, the node's ARP request for the peer answered if it +/// makes one. Returns what the node sent for it. +fn syn(wire: &mut Wire, to: u16) -> Vec { + let from = wire.sent.len(); + wire.deliver(&from_peer(to, None)); + if wire.sent[from..].iter().any(|seen| matches!(seen, Seen::Arp { request: true, target, .. } if *target == PEER)) { + wire.deliver(&arp(MAC, false, MAC_B, PEER, A)); + } + wire.sent[from..].to_vec() } type Loss = fn(&mut Wire); /// The three ways a held address goes: the server refuses its renewal, the lease runs out, and -/// another host defends the address twice. +/// another host defends the address twice inside [ip]'s defend interval (RFC 5227 §2.4 (b)). fn losses() -> [(&'static str, Loss); 3] { fn nak(wire: &mut Wire) { let id = renewing(wire); @@ -224,53 +258,63 @@ fn losses() -> [(&'static str, Loss); 3] { } fn conflict(wire: &mut Wire) { let conflict = arp(BROADCAST, false, MAC_B, A, A); - wire.fire(after(100)); + wire.fire(wire.now.after(Duration::from_secs(100))); wire.deliver(&conflict); - wire.fire(after(105)); + wire.fire(wire.now.after(Duration::from_secs(5))); wire.deliver(&conflict); } [("a NAK", nak), ("expiry", expiry), ("a conflict", conflict)] } -// A listener that named the machine's address can be reached by nobody once the address is -// gone, and its owner only waits: the node ends it in the pass that finds the address gone, so -// the owner reads the end of its wake pipe, the shell is told why, and the port and the place -// are free. One at every address is at whatever address comes next, and stands. +// What takes the machine's address is the wire's doing, not a program's: a NAK, a server that +// is away, two ARP frames. So a listener that named the address stands through its loss as a +// datagram socket bound to it does, with its port and its place and its owner told nothing: +// [ip] takes no segment for an address it does not hold, and [tcp] hands the listener the next +// SYN once the address is the machine's again. A renewal and a link that went and came back +// are no loss at all. #[test] -fn a_listener_at_an_address_the_lease_loses_is_ended_and_one_at_every_address_stands() { +fn a_listener_stands_while_its_address_is_lost_and_answers_when_it_is_back() { for (how, lose) in losses() { let mut wire = Wire::leased(&terms(600, Some(R))); - let (named, named_told) = listen(&mut wire, A, 22); - let (_, any_told) = listen(&mut wire, Ipv4Addr::UNSPECIFIED, 23); - assert_eq!((wire.node.listeners(), wire.node.held()), (2, 2), "{how}"); + let named = listen(&mut wire, A, 22).expect("the held address"); + let any = listen(&mut wire, Ipv4Addr::UNSPECIFIED, 23).expect("every address"); + let stand = |wire: &mut Wire, when: &str| { + assert_eq!((wire.node.listeners(), wire.node.held(), wire.node.drain_ended_listeners().count()), (2, 2, 0), "{how}, {when}"); + assert_eq!((named.dropped.get(), any.dropped.get(), named.wakes.get(), any.wakes.get()), (false, false, 0, 0), "{how}, {when}"); + }; + + let id = renewing(&mut wire); + wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms(600, Some(R))))); + wire.link(false); + stand(&mut wire, "the link down"); + wire.link(true); + let id = xid(wire.last(REQUEST)); + wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); + assert_eq!(wire.node.lease().map(|lease| lease.address), Some(A), "{how}"); + stand(&mut wire, "renewed, and the link back"); lose(&mut wire); assert_eq!((wire.node.lease(), wire.address(A)), (None, None), "{how}"); - let ended: Vec<(ListenerId, ListenerEnd)> = wire.node.drain_ended_listeners().collect(); - assert_eq!((ended, named_told.get(), any_told.get()), (vec![(named, ListenerEnd::Address)], true, false), "{how}"); - assert_eq!((wire.node.listeners(), wire.node.held()), (1, 1), "{how}"); - assert!(!wire.node.close_listener(wire.now, named), "{how}: its id names nothing"); - listen(&mut wire, Ipv4Addr::UNSPECIFIED, 22); - let again = wire.node.listen(A, Port::new(24), Box::new(Owner::default()), || panic!("a named port draws nothing")); - assert_eq!(again.unwrap_err(), ListenRefused::NotLocal, "{how}"); + stand(&mut wire, "the address lost"); + assert_eq!(syn(&mut wire, 22), [], "{how}: a SYN to an address the machine does not hold"); + stand(&mut wire, "a SYN to the lost address"); + assert_eq!(listen(&mut wire, A, 24).err(), Some(ListenRefused::NotLocal), "{how}"); + + assert!(wire.run_until(Duration::from_secs(60), |wire| wire.dhcp().last().is_some_and(|(kind, _)| *kind == DISCOVER)), "{how}: discovery starts over"); + let id = xid(wire.last(DISCOVER)); + wire.deliver(&from_server(MAC, A, &message_of(OFFER, id, &terms(600, Some(R))))); + wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); + assert!(wire.run_until(Duration::from_secs(10), |wire| wire.node.lease().is_some()), "{how}: the address is held again"); + stand(&mut wire, "the address back"); + + let answer = syn(&mut wire, 22); + let Some(Seen::Tcp(synack)) = answer.last() else { panic!("{how}: {answer:?}") }; + assert!(synack.syn && (synack.from_port, synack.ack) == (22, Some(ISS + 1)), "{how}: {synack:?}"); + wire.deliver(&from_peer(22, Some(synack.seq.wrapping_add(1)))); + assert_eq!((named.wakes.get(), any.wakes.get(), wire.node.listeners(), wire.node.held()), (1, 0, 2, 2), "{how}"); } } -// [ip] holds the address usable for as long as the lease is held: a renewal and a link that -// went and came back leave it, and the listener at it. -#[test] -fn a_listener_at_the_held_address_outlives_a_renewal_and_a_link_that_returns() { - let mut wire = Wire::leased(&terms(600, Some(R))); - let (_, told) = listen(&mut wire, A, 22); - let id = renewing(&mut wire); - wire.deliver(&from_server(MAC, A, &message_of(ACK, id, &terms(600, Some(R))))); - wire.link(false); - wire.link(true); - let id = xid(wire.last(REQUEST)); - wire.deliver(&from_server(BROADCAST, Ipv4Addr::BROADCAST, &message_of(ACK, id, &terms(600, Some(R))))); - assert_eq!((wire.node.lease().map(|lease| lease.address), wire.node.listeners(), wire.node.drain_ended_listeners().count(), told.get()), (Some(A), 1, 0, false)); -} - #[test] fn a_router_ip_refuses_is_no_router() { let edge = Ipv4Addr::new(192, 0, 2, 255); diff --git a/userland/netstack/node/tests/listeners.rs b/userland/netstack/node/tests/listeners.rs index 53c267fa7f7..a42c736e7cc 100644 --- a/userland/netstack/node/tests/listeners.rs +++ b/userland/netstack/node/tests/listeners.rs @@ -20,7 +20,7 @@ use std::time::Duration; use common::{arp, terms, Segment, Wire, A, ELSEWHERE, MAC, MAC_R, R}; use etherparse::{ArpOperation, LinkSlice, NetSlice, PacketBuilder, SlicedPacket, TcpOptionElement, TransportSlice}; -use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerEnd, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; +use toyos_net_node::{AcceptRefused, Accepted, ConnectRefused, FromClient, ListenRefused, ListenerId, Node, PipeEnd, Pipes, ReadRefusal, Refused, StreamEvent, StreamId, ToClient, Wake, WriteRefusal}; use toyos_net_tcp::{limits, Counter, Endpoint}; use toyos_net_wire::{Instant, Port}; @@ -703,8 +703,8 @@ fn a_wake_the_owners_pipe_refuses_ends_the_listener() { let (id, owner) = net.listen(SSH); owner.borrow_mut().refusal = Some(refusal); net.handshake(P1); - let ended: Vec<(ListenerId, ListenerEnd)> = net.node.drain_ended_listeners().collect(); - assert_eq!(ended, [(id, ListenerEnd::Wake(refusal))]); + let ended: Vec<(ListenerId, WriteRefusal)> = net.node.drain_ended_listeners().collect(); + assert_eq!(ended, [(id, refusal)]); assert!(owner.borrow().dropped && net.last(P1).rst, "{refusal:?}: {:?}", net.heard); assert_eq!((net.node.listeners(), net.node.held()), (0, 0), "{refusal:?}"); net.listen(SSH); @@ -808,8 +808,8 @@ fn a_listener_ended_for_its_wake_gives_its_place_to_another_in_the_same_pass() { let ack = net.iss(P3).wrapping_add(1); net.node.receive(net.now, &frame(P3, SSH, ISS + 1, Some(ack), 0, &[]), draw(&mut net.draws)); assert_eq!(wakes(&owner), 2); - let ended: Vec<(ListenerId, ListenerEnd)> = net.node.drain_ended_listeners().collect(); - assert_eq!((ended, net.node.listeners(), net.node.held()), (vec![(first, ListenerEnd::Wake(WriteRefusal::Gone))], 1, 1)); + let ended: Vec<(ListenerId, WriteRefusal)> = net.node.drain_ended_listeners().collect(); + assert_eq!((ended, net.node.listeners(), net.node.held()), (vec![(first, WriteRefusal::Gone)], 1, 1)); } // A datagram socket is something a client makes the node hold, its two queues of sixteen