diff --git a/build/ci.rs b/build/ci.rs index 1e1a8e0343d..926577d6c66 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -832,6 +832,8 @@ pub(crate) const CONTROLS: &[Control] = &[ )]), red(SDK, "transport-no-resume", Some("transport_loom"), &[Fails("two_holders_read_each_byte_once")]), red(SDK, "transport-ungated-violation", Some("transport_loom"), &[Fails("looks_without_the_gate_lose_no_wake")]), + red(SDK, "transport-grow-room-now", Some("transport_loom"), &[Fails("a_ring_grown_while_it_holds_bytes_reads_each_as_written")]), + red(SDK, "transport-size-before-tail", Some("transport_loom"), &[Fails("a_ring_grown_while_it_holds_bytes_reads_each_as_written")]), red(SDK, "transport-end-keeps-inflight", None, &[Fails( "transport::inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing", )]), diff --git a/issues/a-process-holds-at-most-128-streams.md b/issues/a-process-holds-at-most-128-streams.md new file mode 100644 index 00000000000..d1cd512164c --- /dev/null +++ b/issues/a-process-holds-at-most-128-streams.md @@ -0,0 +1,19 @@ +--- +status: open +kind: defect +opened: 2026-10-14 +--- + +# A process holds at most 128 streams + +The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 +handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` +allocates nothing: past it a connect or an accept is refused +`ResourceExhausted` before anything is made. 128 is libc's `MAX_SOCKETS`, an +estimate no measurement set, and a new limit for a std program. + +**Exit condition**: the table grown on demand, or its bound read off a +measured program's need. + +**Owner**: the orchestrator, who stages +`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`. diff --git a/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md b/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md deleted file mode 100644 index 5840bb152e0..00000000000 --- a/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-14 ---- - -# A stream commits 8 MiB at its connect, and a process holds at most 128 - -Every connect and accept makes its region whole before a byte moves: -`toyos::net::stream::REGION_BYTES`, four 2 MiB pages from `SYS_SHM_CREATE`, -physically contiguous, where the pipes it replaced took a page on first write -and [tcp]'s rings grew as text was held. A program that opens many short -connections pays 8 MiB for each, and a fragmented machine can refuse a -connect a pipe would have taken. #863 accepted the cost with the design. - -The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 -handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` -allocates nothing: past it a connect or an accept is refused -`ResourceExhausted` before anything is made. 128 is libc's `MAX_SOCKETS`, an -estimate no measurement set, and a new limit for a std program. - -**Exit condition**: regions whose pages are allocated as the rings fill, so a -stream commits what it holds; and the table grown on demand, or its bound -read off a measured program's need. - -**Owner**: the orchestrator, who stages -`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`. diff --git a/issues/a-stream-ring-never-shrinks.md b/issues/a-stream-ring-never-shrinks.md new file mode 100644 index 00000000000..40192113f9c --- /dev/null +++ b/issues/a-stream-ring-never-shrinks.md @@ -0,0 +1,23 @@ +--- +status: open +kind: defect +opened: 2026-10-28 +--- + +# A stream's ring never shrinks + +A stream region commits its header and 64 KiB of each ring at the connect, +and each ring's producer doubles it, up to 4 MiB, when it needs more +(`toyos::transport::stream`). Nothing gives a page back: a long-lived stream +that once carried a burst keeps the most it ever held, up to 8 MiB + 4 KiB, +for as long as it lives, counted to its client's job. netstack prices each +place at that most (`PLACE_BYTES`, `system/netstack/src/main.rs`). + +**Owner**: the orchestrator, who assigns it after #1077 lands. + +**Exit**: a ring whose stream has gone idle gives back what it no longer holds, +at a quiescent point both ends agree on (both cursors equal, under the +session's gates), through a kernel call that decommits pages of a memory +object; a host test of the ring and a loom model of the shrink beside a +growth and a write, and a guest test that reads the client's account fall +after an idle stream's shrink. diff --git a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md index e279f776b90..1fc8b64393c 100644 --- a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md +++ b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md @@ -9,10 +9,11 @@ opened: 2026-10-27 Before a ToyOS TCP connect returns, it makes and zeroes memory the connection may never use: -- **The stream region**, 8 MiB (`toyos::net::stream::REGION_BYTES`). - `SYS_SHM_CREATE` takes and zeroes each of its 2048 frames before it answers - (`kernel/src/object/shm.rs`), and netstack maps all of them before the SYN - leaves. +- **The stream region**, 8 MiB (`toyos::net::stream::REGION_BYTES`), whose + 2048 frames `SYS_SHM_CREATE` took and zeroed before it answered. Since + #1077 a connect commits its header and 64 KiB of each ring, 132 KiB, and + each ring's producer commits more as it doubles the ring + (`toyos::transport::stream`); the T14's reading of that is owed. - **The connection's two directions**, 2 MiB each: the request's page on the client's write and the answer's on netstack's, each zeroed whole on that first write (`issues/a-pipe-is-a-two-mebibyte-page-for-a-mapping-nothing-makes.md`). diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index 79fa1b181f7..64979150e89 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -214,18 +214,19 @@ impl Inventory { /// Whether entries cover every page of `[start, end)`. pub fn covered(&self, start: u64, end: u64) -> bool { - self.covered_where(start, end, |_| true) + self.covered_where(start, end, |_, _| true) } /// Whether entries cover every page of `[start, end)` and each that meets - /// it answers `ok`: one descent of the tree, walking down from `end`. - pub fn covered_where(&self, start: u64, end: u64, ok: impl Fn(&Entry) -> bool) -> bool { + /// it answers `ok`, given where it starts: one descent of the tree, + /// walking down from `end`. + pub fn covered_where(&self, start: u64, end: u64, ok: impl Fn(u64, &Entry) -> bool) -> bool { // `[low, end)` is covered by entries that answered `ok`. let mut low = end; let mut below = self.entries.range(..end).rev(); while low > start { let Some((&s, entry)) = below.next() else { return false }; - if entry.end < low || !ok(entry) { + if entry.end < low || !ok(s, entry) { return false; } low = s; @@ -574,7 +575,7 @@ mod tests { let pages = &map[page(start)..page(end)]; assert_eq!(v.covered(start, end), pages.iter().all(Option::is_some), "round {round}: covered"); assert_eq!( - v.covered_where(start, end, |e| e.prot == Prot::ReadWrite), + v.covered_where(start, end, |_, e| e.prot == Prot::ReadWrite), pages.iter().all(|p| p.is_some_and(|(prot, _)| prot == Prot::ReadWrite)), "round {round}: covered by writable entries" ); diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 68829bad25a..2c7ba330bc8 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -64,7 +64,7 @@ pub fn set_counter(boot: u64, mult: u64) { let stated = STATED_MULT.load(Relaxed); let at_boot = Line { counter: 0, stamp: 0, mult: if stated != 0 { stated } else { mult } }.nanos(boot); let page = SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); - let lines = crate::DirectMap::from_phys(page.page(0)).as_mut_ptr::(); + let lines = crate::DirectMap::from_phys(page.page(0).expect("clock: its page is committed whole")).as_mut_ptr::(); // SAFETY: the page is this object's own, which no address space maps yet // and none unmaps after: held below for the machine's life. Every field of // `ClockPage` is an atomic, so a shared reference reads and writes it as diff --git a/kernel/src/inbox/mod.rs b/kernel/src/inbox/mod.rs index 6b7e1474b52..ec60c0c3add 100644 --- a/kernel/src/inbox/mod.rs +++ b/kernel/src/inbox/mod.rs @@ -223,7 +223,7 @@ struct RingState { /// One atomic word of one ring header; never `&RingHeader` — see the block above. fn ring_word(ring: &SharedMemObject, ring_off: u64, field_off: usize) -> &core::sync::atomic::AtomicU32 { let at = ring_off + field_off as u64; - let word = DirectMap::from_phys(ring.page(at) + at % crate::mm::PAGE_SIZE); + let word = DirectMap::from_phys(ring.page(at).expect("inbox: its ring is committed whole") + at % crate::mm::PAGE_SIZE); // SAFETY: the offset is in-bounds and 4-aligned, so inside one page of the // object, which outlives the borrow; `AtomicU32` is sound over memory the // process also writes. diff --git a/kernel/src/mm/pmm.rs b/kernel/src/mm/pmm.rs index d32aa3e0921..c5cc679dcbb 100644 --- a/kernel/src/mm/pmm.rs +++ b/kernel/src/mm/pmm.rs @@ -136,21 +136,9 @@ pub fn release_floor(job: &Job) { /// # Safety /// Each is a frame [`Frame::into_raw`] gave up, that no leaf on any CPU maps /// any more and nothing else frees. -pub unsafe fn free_drawn(job: &Job, frames: &[u64]) { +pub unsafe fn free_drawn(job: &Job, frames: impl IntoIterator) { let pins = pins(); - with(|f| f.free_drawn(job.memory(), frames.iter().copied(), &pins)); -} - -/// Give back every frame of `frames`, drawn for pages `job` committed, under -/// one hold of the allocator's lock. -pub fn free_all(job: &Job, frames: alloc::boxed::Box<[Frame]>) { - let pins = pins(); - with(|f| f.free_drawn(job.memory(), frames.iter().map(|frame| frame.phys), &pins)); - // Given up only once the allocator's lock is let go: the list goes back - // to the heap, which may give a page back through it. - for frame in frames.into_vec() { - let _ = frame.into_raw(); - } + with(|f| f.free_drawn(job.memory(), frames, &pins)); } /// The pin counters, published once by [`init`]: where they are, how many, diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 519b10f4cf6..7256e79374c 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -774,7 +774,7 @@ impl AddressSpace { crate::sched::futex::revoke_frames(frames); // SAFETY: frames owned leaves gave up, which no leaf maps on any // CPU now and no waiter is keyed on. - unsafe { pmm::free_drawn(&self.job, frames) }; + unsafe { pmm::free_drawn(&self.job, frames.iter().copied()) }; freed += frames.len() as u64; }); let inventory = &self.inventory; @@ -822,7 +822,7 @@ impl AddressSpace { Leaf::Empty => match &entry.kind { Kind::Private { committed: false } => unreachable!("space: a reservation at {page:#x} allows an access"), Kind::Object { object, offset, .. } => { - let phys = object.page(offset + (page - start)); + let phys = object.page(offset + (page - start))?; leaves.set(page, Leaf::Mapped(Mapping { phys, prot, owned: false, cache: object.cache() })); phys } @@ -850,10 +850,21 @@ impl AddressSpace { /// Whether every page of `[start, end)` is in a range that allows /// `access` and is memory the kernel can copy through the direct map: /// what a copy the kernel makes for a thread asks first. A device's - /// registers or firmware's framebuffer, an aperture, are no buffer. + /// registers or firmware's framebuffer, an aperture, are no buffer. A + /// page an object maps is one only once its object committed it: a copy + /// into one that is not would be lost, and a fault on it is refused. pub fn allows(&self, start: u64, end: u64, access: Access) -> bool { - self.inventory - .covered_where(start & !(PAGE_SIZE - 1), end, |entry| entry.prot.allows(access) && entry.backed() && copyable(entry)) + let start = start & !(PAGE_SIZE - 1); + self.inventory.covered_where(start, end, |at, entry| { + let committed = || match &entry.kind { + Kind::Object { object, offset, .. } => { + let (lo, hi) = (start.max(at), end.min(entry.end)); + object.committed(offset + (lo - at), hi - lo) + } + Kind::Private { .. } | Kind::Copy { .. } => true, + }; + entry.prot.allows(access) && entry.backed() && copyable(entry) && committed() + }) } /// The frame and protection a present leaf maps `va` with, populating @@ -956,7 +967,7 @@ impl Drop for AddressSpace { let mut freed = 0; self.tables.take_owned(FLOOR, TOP, |frames| { // SAFETY: the space's own frames, which nothing maps once it drops. - unsafe { pmm::free_drawn(&self.job, frames) }; + unsafe { pmm::free_drawn(&self.job, frames.iter().copied()) }; freed += frames.len() as u64; }); self.usage.freed(freed); diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index a7b74d6b77d..54889072508 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -132,7 +132,7 @@ impl Held { } // SAFETY: `phys` is a frame this pager drew, out of its index or never // in it, which nothing maps; a reader's pin defers its reuse. - unsafe { pmm::free_drawn(job, &[phys]) }; + unsafe { pmm::free_drawn(job, [phys]) }; } /// Move each held-aside frame no copy pins any more to the spare ones. @@ -948,7 +948,7 @@ impl ZeroHandles for PagerObject { while let Some(phys) = state.held.spare.pop().or_else(|| state.held.draining.pop()) { // SAFETY: a frame this pager drew and held aside, which nothing // maps; a pin defers its free to its last unpin. - unsafe { pmm::free_drawn(job, &[phys]) }; + unsafe { pmm::free_drawn(job, [phys]) }; } let pool = state.held.core.end(); pmm::unpromise(job, pool); diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index c501dbcd187..888f43f56db 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -10,12 +10,19 @@ //! closing its last handle is how a device's window is taken back from every //! process that mapped it. //! -//! Frames are the object's from its creation to its end, so a fault on a -//! mapping of one never allocates. +//! **A fault never allocates.** A region's pages are committed, each a +//! frame of its own, zeroed, counted to the object's job, at the call that +//! commits them: `SYS_SHM_CREATE`, all or a first part of them, then +//! `SYS_SHM_COMMIT`, which any holder that may map it calls. A committed +//! page is the object's to its end. A fault on a page not committed is +//! refused as an access outside every range is, a syscall's window over one +//! is `BadAddress`, and the kernel's own read of one ([`SharedMemObject::read`], +//! a program image's loader among its callers) answers zeros. use alloc::boxed::Box; use alloc::sync::{Arc, Weak}; use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, Ordering}; use kernel::sched::job::Job; @@ -23,7 +30,7 @@ use toyos_abi::syscall::SyscallError; use crate::mm::charge::{self, Charge, Exhausted}; use crate::mm::policy::CachePolicy; -use crate::mm::pmm::{self, Frame, Run}; +use crate::mm::pmm::{self, Run}; use crate::mm::space::Space; use crate::mm::{PAGE_2M, PAGE_SIZE}; use crate::{DirectMap, UserAddr}; @@ -40,8 +47,9 @@ const CHUNK: usize = 1 << 16; /// What an object's bytes are. enum Backing { /// Frames of its own, one per page, in no order, [`CHUNK`] to a piece so - /// that no piece is past one heap allocation. - Frames(Vec>), + /// that no piece is past one heap allocation: each word its page's + /// frame, or 0 while the page is not committed, since no frame is at 0. + Frames(Vec>), /// 2 MiB pages of its own in one physical run: what a device may be lent /// or granted, and what a kernel-built image addresses as one slice. Run(Run), @@ -65,39 +73,95 @@ pub struct SharedMemObject { backing: Backing, size: u64, cache: CachePolicy, + /// [`Backing::Frames`]' pages not yet committed; 0 for every other + /// backing. A syscall's window over an object asks it, so one committed + /// whole answers without a look at each page. + missing: AtomicU64, } impl SharedMemObject { /// Over `core`, charged with what `backing` keeps on the heap. fn new(core: ObjectCore, backing: Backing, size: u64, cache: CachePolicy) -> Arc { - Arc::new(Self { core, backing, size, cache }) + let missing = match &backing { + Backing::Frames(_) => AtomicU64::new(size / PAGE_SIZE), + _ => AtomicU64::new(0), + }; + Arc::new(Self { core, backing, size, cache, missing }) } /// `size` bytes of zeroed memory in whole pages, frame by frame, counted /// to `job`; `InvalidArgument` for no bytes or a size no page count /// holds, `ResourceExhausted` where frames run out. pub fn create(job: &Arc, size: u64) -> Result, SyscallError> { + Self::reserve(job, size, size) + } + + /// [`Self::create`], with only its first `committed` bytes' pages + /// committed; [`Self::commit`] commits the rest. + pub fn reserve(job: &Arc, size: u64, committed: u64) -> Result, SyscallError> { let pages = pages(size)?; - let (count, full, rest) = (pages.div_ceil(CHUNK as u64), pages / CHUNK as u64, (pages % CHUNK as u64) as usize); - let lists = charge::array::(CHUNK).saturating_mul(full as usize).saturating_add(charge::array::(rest)); - let core = Self::new_core_for(job, charge::array::>(count as usize).saturating_add(lists))?; - // Promised whole before any is taken: frames one at a time until none - // is left would starve every other allocation meanwhile. - if !pmm::promise(job, pages) { - return Err(SyscallError::ResourceExhausted); + if committed > size { + return Err(SyscallError::InvalidArgument); } + let (count, full, rest) = (pages.div_ceil(CHUNK as u64), pages / CHUNK as u64, (pages % CHUNK as u64) as usize); + let lists = charge::array::(CHUNK).saturating_mul(full as usize).saturating_add(charge::array::(rest)); + let core = Self::new_core_for(job, charge::array::>(count as usize).saturating_add(lists))?; let mut pieces = Vec::with_capacity(count as usize); let mut left = pages as usize; while left > 0 { let n = left.min(CHUNK); - let mut piece = Vec::with_capacity(n); - for _ in 0..n { - piece.push(pmm::alloc_promised()); - } - pieces.push(piece.into_boxed_slice()); + pieces.push((0..n).map(|_| AtomicU64::new(0)).collect()); left -= n; } - Ok(Self::new(core, Backing::Frames(pieces), pages * PAGE_SIZE, CachePolicy::Normal)) + let object = Self::new(core, Backing::Frames(pieces), pages * PAGE_SIZE, CachePolicy::Normal); + if committed > 0 { + object.commit(0, committed)?; + } + Ok(object) + } + + /// Commit every page `[offset, offset + len)` touches that is not yet, + /// each a zeroed frame counted to the object's job: refused whole, + /// `ResourceExhausted`, where its frames cannot all be promised. + /// `InvalidArgument` for no bytes, a range past the object, or an + /// aperture; memory with no page to commit answers at once. + pub fn commit(&self, offset: u64, len: u64) -> Result<(), SyscallError> { + if len == 0 || offset.checked_add(len).is_none_or(|end| end > self.size) { + return Err(SyscallError::InvalidArgument); + } + let pieces = match &self.backing { + Backing::Frames(pieces) => pieces, + Backing::Sub { whole, offset: base, .. } => return whole.commit(base + offset, len), + Backing::Run(_) | Backing::Boot { .. } => return Ok(()), + Backing::Aperture { .. } => return Err(SyscallError::InvalidArgument), + }; + let words = || { + let (first, end) = (offset / PAGE_SIZE, (offset + len).div_ceil(PAGE_SIZE)); + (first..end).map(|page| &pieces[page as usize / CHUNK][page as usize % CHUNK]) + }; + let wanted = words().filter(|word| word.load(Ordering::Acquire) == 0).count() as u64; + // Promised whole before any is taken, as a creation's are. + if !pmm::promise(self.core.job(), wanted) { + return Err(SyscallError::ResourceExhausted); + } + let mut drawn = 0; + // A page another commit filled meanwhile was counted here and is + // left; one it fills between the look and the store keeps its frame. + for word in words().filter(|word| word.load(Ordering::Acquire) == 0) { + let frame = pmm::alloc_promised().into_raw(); + drawn += 1; + // Release: a fault that reads the frame finds it zeroed. + match word.compare_exchange(0, frame, Ordering::Release, Ordering::Acquire) { + // Release: whoever reads none missing finds every frame stored. + Ok(_) => { + self.missing.fetch_sub(1, Ordering::Release); + } + // SAFETY: drawn just now and named by nothing. + Err(_) => unsafe { pmm::free_drawn(self.core.job(), [frame]) }, + } + } + pmm::unpromise(self.core.job(), wanted - drawn); + Ok(()) } /// `size` bytes of zeroed memory in whole pages, over one run of whole @@ -199,27 +263,39 @@ impl SharedMemObject { } } - /// The physical address of the page `offset` bytes in, which lies inside it. - pub fn page(&self, offset: u64) -> u64 { + /// Whether every page `[offset, offset + len)` touches is committed. + pub fn committed(&self, offset: u64, len: u64) -> bool { + match &self.backing { + Backing::Sub { whole, offset: base, .. } => whole.committed(base + offset, len), + _ if self.missing.load(Ordering::Acquire) == 0 => true, + _ => (offset & !(PAGE_SIZE - 1)..offset + len).step_by(PAGE_SIZE as usize).all(|at| self.page(at).is_some()), + } + } + + /// The physical address of the page `offset` bytes in, which lies inside + /// it; `None` while that page is not committed. + pub fn page(&self, offset: u64) -> Option { assert!(offset < self.size, "shm koid {}: page {offset:#x} of {:#x} bytes", self.core.koid().raw(), self.size); let page = offset & !(PAGE_SIZE - 1); match &self.backing { Backing::Frames(pieces) => { let frame = (page / PAGE_SIZE) as usize; - pieces[frame / CHUNK][frame % CHUNK].direct_map().phys() + // Acquire: the commit's zeroing is seen before the frame is. + Some(pieces[frame / CHUNK][frame % CHUNK].load(Ordering::Acquire)).filter(|&phys| phys != 0) } - Backing::Run(run) => run.direct_map().phys() + page, - Backing::Boot { phys, .. } | Backing::Aperture { phys, .. } => phys + page, + Backing::Run(run) => Some(run.direct_map().phys() + page), + Backing::Boot { phys, .. } | Backing::Aperture { phys, .. } => Some(phys + page), Backing::Sub { whole, offset, .. } => whole.page(offset + page), } } /// Copy `bytes` in at `offset`, through the direct map. For memory the /// kernel lays out before a process can see it, and the words it alone - /// writes after; never an aperture. + /// writes after; never an aperture, and only over committed pages. pub fn write(&self, offset: u64, bytes: &[u8]) { assert!(self.is_ram(), "shm koid {}: the kernel writes no aperture", self.core.koid().raw()); self.each_piece(offset, bytes.len(), |phys, at, n| { + let phys = phys.unwrap_or_else(|| panic!("shm koid {}: the kernel wrote {:#x}, not committed", self.core.koid().raw(), offset + at as u64)); // SAFETY: `each_piece` bounds `[at, at + n)` inside `bytes` and // `phys..+n` inside one of this object's own pages, which the // direct map covers; no reference forms over a page a process may @@ -228,19 +304,23 @@ impl SharedMemObject { }); } - /// Copy out `out.len()` bytes at `offset`, each read once. + /// Copy out `out.len()` bytes at `offset`, each read once: a page not + /// committed reads as zeros. pub fn read(&self, offset: u64, out: &mut [u8]) { assert!(self.is_ordinary(), "shm koid {}: the kernel reads no aperture", self.core.koid().raw()); let into = out.as_mut_ptr(); - self.each_piece(offset, out.len(), |phys, at, n| { + self.each_piece(offset, out.len(), |phys, at, n| match phys { // SAFETY: as `write`, mirrored; `out` is the caller's own. - unsafe { core::ptr::copy_nonoverlapping(DirectMap::from_phys(phys).as_ptr(), into.add(at), n) } + Some(phys) => unsafe { core::ptr::copy_nonoverlapping(DirectMap::from_phys(phys).as_ptr(), into.add(at), n) }, + // SAFETY: `each_piece` bounds `[at, at + n)` inside `out`. + None => unsafe { core::ptr::write_bytes(into.add(at), 0, n) }, }); } /// `f(phys, at, n)` for each piece of `[offset, offset + len)` inside one - /// page, `at` its offset from `offset`. - fn each_piece(&self, offset: u64, len: usize, mut f: impl FnMut(u64, usize, usize)) { + /// page, `at` its offset from `offset`; `phys` is `None` for a page not + /// committed. + fn each_piece(&self, offset: u64, len: usize, mut f: impl FnMut(Option, usize, usize)) { assert!( offset.checked_add(len as u64).is_some_and(|end| end <= self.size), "shm koid {}: {len} bytes at {offset:#x} of {:#x}", @@ -251,7 +331,7 @@ impl SharedMemObject { while done < len { let at = offset + done as u64; let n = ((PAGE_SIZE - at % PAGE_SIZE) as usize).min(len - done); - f(self.page(at) + at % PAGE_SIZE, done, n); + f(self.page(at).map(|page| page + at % PAGE_SIZE), done, n); done += n; } } @@ -288,15 +368,19 @@ fn pages(size: u64) -> Result { /// first every futex wait on a word in them ends, so none is left keyed on a /// frame the allocator hands out again. Nothing is allocated: each piece is /// sorted in place, its waits ended by one walk of the 64 buckets, and its -/// frames freed under one hold of the allocator's lock. +/// committed frames freed under one hold of the allocator's lock. impl Drop for SharedMemObject { fn drop(&mut self) { match &mut self.backing { Backing::Frames(pieces) => { for mut piece in pieces.drain(..) { - piece.sort_unstable_by_key(|frame| frame.direct_map().phys()); - crate::sched::futex::revoke_sorted(&piece, |frame| frame.direct_map().phys()); - pmm::free_all(self.core.job(), piece); + piece.sort_unstable_by_key(|word| word.load(Ordering::Relaxed)); + let committed = piece.partition_point(|word| word.load(Ordering::Relaxed) == 0); + let frames = &piece[committed..]; + crate::sched::futex::revoke_sorted(frames, |word| word.load(Ordering::Relaxed)); + // SAFETY: the last mapping and handle are gone, so no leaf + // maps a frame of it, and nothing else frees one. + unsafe { pmm::free_drawn(self.core.job(), frames.iter().map(|word| word.load(Ordering::Relaxed))) }; } } Backing::Run(run) => { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 575f26ee549..4599fa11608 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -39,7 +39,7 @@ use super::ipc::{ Incoming, Outgoing, sys_accept, sys_inbox_setup, sys_message_recv, sys_message_send, sys_inbox_submit, sys_namespace_build, sys_namespace_open, sys_pipe, - sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, sys_shm_sub, + sys_port_badge, sys_port_create, sys_port_mint, sys_shm_commit, sys_shm_create, sys_shm_map, sys_shm_sub, }; use super::machine::{ sys_boot_sources, sys_counters, sys_device_inventory, sys_log_read, sys_machine_stops, sys_reboot, sys_shutdown, @@ -471,7 +471,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> let Some(mut into) = into else { return bad_addr }; sys_message_recv(RawHandle(a1 as u32), &mut into, args.flags & MESSAGE_NONBLOCK != 0) } - SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32)), + SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32), a4), + SYS_SHM_COMMIT => sys_shm_commit(RawHandle(a1 as u32), a2, a3), SYS_SHM_MAP => sys_shm_map(RawHandle(a1 as u32), a2), SYS_SHM_SUB => sys_shm_sub(RawHandle(a1 as u32), a2, a3), SYS_CONSOLE_CARRIED => sys_console_carried(RawHandle(a1 as u32), a2), diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 90dfe083089..3fd1f91fae0 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -325,10 +325,10 @@ pub(super) fn sys_port_badge( } /// Make a shared region and hand back the one handle to it, carrying MAP, WRITE, DUP and TRANSFER — naming it is what authorizes mapping it. -/// `SHM_DEVICE` asks for memory a device can be lent, and `SHM_FOR_JOB` -/// counts it to the job `job` names rather than the caller's; any other -/// bit is refused. -pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { +/// `SHM_DEVICE` asks for memory a device can be lent, `SHM_PART` commits +/// only its first `committed` bytes, and `SHM_FOR_JOB` counts it to the job +/// `job` names rather than the caller's; any other bit is refused. +pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle, committed: u64) -> u64 { let charged = match flags & SHM_FOR_JOB { 0 => crate::mm::charge::current(), _ => match process::with_process_data(|data| data.handles.get::(job, Rights::WRITE)) { @@ -338,6 +338,7 @@ pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { }; let created = match flags & !SHM_FOR_JOB { 0 => crate::object::shm::SharedMemObject::create(&charged, size), + SHM_PART => crate::object::shm::SharedMemObject::reserve(&charged, size, committed), SHM_DEVICE => crate::object::shm::SharedMemObject::create_run(&charged, size), _ => return SyscallError::InvalidArgument.to_u64(), }; @@ -348,6 +349,19 @@ pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { process::with_process_data(|data| handle_result(ops::install(&mut data.handles, object))) } +/// `toyos_abi::syscall::shm_commit`, charged to the region's own job. +pub(super) fn sys_shm_commit(h: RawHandle, offset: u64, len: u64) -> u64 { + // A refusal is made with nothing held: a bad handle's ends the process. + let object = match process::with_process_data(|data| data.handles.get::(h, Rights::MAP)) { + Ok(object) => object, + Err(e) => return e.refuse(), + }; + match object.commit(offset, len) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} + /// `toyos_abi::syscall::shm_sub`: the new object's handle carries `h`'s rights. pub(super) fn sys_shm_sub(h: RawHandle, offset: u64, len: u64) -> u64 { // A refusal is made with nothing held: a bad handle's ends the process. diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 390a8bca26b..f22278ba1fd 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -183,6 +183,10 @@ pub const SHM_DEVICE: u64 = 1; /// authority to place a child in a job is the authority to make memory for it. pub const SHM_FOR_JOB: u64 = 1 << 1; +/// [`shm_reserve`]: only the region's first bytes, the call's fourth +/// argument, are committed; [`shm_commit`] commits more. +pub const SHM_PART: u64 = 1 << 2; + /// What a [`SHM_DEVICE`] region is a whole number of. pub const SHM_DEVICE_PAGE: u64 = 2 * 1024 * 1024; @@ -373,6 +377,8 @@ pub const SYS_CURRENT_CPU: u64 = 163; /// Map a firmware window a claim was granted at its mint. See /// [`device_firmware_window`]. pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; +/// Commit pages of a region made with fewer. See [`shm_commit`]. +pub const SYS_SHM_COMMIT: u64 = 165; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -380,13 +386,13 @@ pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 166; +pub const SYSCALL_PROFILE_BINS: usize = 167; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_DEVICE_FIRMWARE_WINDOW < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_SHM_COMMIT < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -2306,6 +2312,28 @@ pub fn shm_create_flags(size: usize, flags: u64) -> Result Result { + check(syscall(SYS_SHM_CREATE, size as u64, SHM_PART, 0, committed as u64)).map(|v| RawHandle(v as u32)) +} + +/// Commit every page `[offset, offset + len)` of the region `shm` names +/// touches that is not yet: zeroed, counted to the job the region is +/// counted to, and refused whole, `ResourceExhausted`, where they cannot all +/// be. A page committed already is left as it is, so any holder may commit +/// what it is about to touch. Needs [`Rights::MAP`] alone: a holder that may +/// only map a region read-only may still spend its owner's memory on it, up +/// to the size the owner made it. `InvalidArgument` for no bytes, a range +/// past the region, or a device's window. +/// +/// [`Rights::MAP`]: crate::handle::Rights::MAP +pub fn shm_commit(shm: RawHandle, offset: usize, len: usize) -> Result<(), SyscallError> { + check(syscall(SYS_SHM_COMMIT, shm.0 as u64, offset as u64, len as u64, 0)).map(|_| ()) +} + /// [`shm_create`], counted to the job `job` names (carrying `WRITE`) rather /// than the caller's: memory a launcher lays out for a child it starts in /// that job, such as its log ring. diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index ac4288d067e..ee3ddbcf1f7 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -95,6 +95,12 @@ transport-no-resume = [] # A look without the gate is believed when it names a violation, so a cursor # another holder moved under it ends the call. transport-ungated-violation = [] +# A stream's consumer loads the ring's size before its tail, so it can read a +# byte written after a growth by the size before it. +transport-size-before-tail = [] +# A producer that grew its ring counts its room in the new capacity at once, +# so it writes over bytes before the switch the consumer has not read. +transport-grow-room-now = [] [lints.rust] # The guest's target: its stream sinks are compiled only for it. diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 7a8880aa05d..7fe46a836eb 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -65,17 +65,26 @@ pub use crate::transport::session::Way; // ---- the region ---- -/// The region a stream is: four 2 MiB pages. -pub const REGION_BYTES: usize = 8 << 20; +/// The region a stream is at most: its header and its two rings each grown +/// whole. A connect makes only the header and each ring's first +/// [`RECEIVE_INITIAL`] and [`SEND_INITIAL`] bytes memory; each ring's producer +/// makes more as it grows it (`crate::transport::stream`). +pub const REGION_BYTES: usize = SEND_RING + SEND_BYTES as usize; /// The header's words: its first 4 KiB. pub const WORDS: usize = 1024; const HEADER_BYTES: usize = WORDS * 4; -/// The receive ring, after the header. +/// The receive ring, after the header, at most. pub const RECEIVE_BYTES: u32 = 4 << 20; pub const RECEIVE_RING: usize = HEADER_BYTES; -/// The send ring, the rest. -pub const SEND_BYTES: u32 = (4 << 20) - HEADER_BYTES as u32; +/// The send ring, after it, at most. +pub const SEND_BYTES: u32 = 4 << 20; pub const SEND_RING: usize = RECEIVE_RING + RECEIVE_BYTES as usize; +/// The receive ring as a connect makes it: [tcp]'s first receive buffer +/// (`toyos_net_tcp::limits::RECEIVE_BUFFER_INITIAL`), rounded up to the power +/// of two a ring is. +pub const RECEIVE_INITIAL: u32 = 64 << 10; +/// The send ring as a connect makes it. +pub const SEND_INITIAL: u32 = 64 << 10; /// Word `n` of the protocol's, on a 64-byte line of its own. const fn line(n: usize) -> usize { @@ -93,11 +102,11 @@ pub const LAYOUT: Layout = Layout::new::< { line(13) }, { line(14) }, >( - Place::new::<{ line(0) }, { line(1) }, { line(2) }, { line(3) }, RECEIVE_RING>(), - Place::new::<{ line(4) }, { line(5) }, { line(6) }, { line(7) }, SEND_RING>(), + Place::new::<{ line(0) }, { line(1) }, { line(2) }, { line(3) }, { line(15) }, RECEIVE_RING>(), + Place::new::<{ line(4) }, { line(5) }, { line(6) }, { line(7) }, { line(16) }, SEND_RING>(), ); -const _: () = assert!(SEND_RING + SEND_BYTES as usize == REGION_BYTES && NODELAY < WORDS); +const _: () = assert!(line(16) < WORDS && NODELAY < WORDS); /// The header, as the one type that may alias words a peer writes. pub type Words = [AtomicU32; WORDS]; @@ -347,6 +356,11 @@ struct Call { } impl Call { + /// Make a ring's first `cap` bytes, at `ring`, memory. + fn commit(&self, ring: usize, cap: u32) -> bool { + self.end.region.is_some_and(|mapped| syscall::shm_commit(mapped.handle, ring, cap as usize).is_ok()) + } + fn words(&self) -> &Words { // SAFETY: the region is mapped at `base`, 2 MiB aligned, for as long as // a call is counted, and its header is `WORDS` words reached only as @@ -521,7 +535,8 @@ pub fn read(handle: BorrowedHandle<'_>, buf: &mut [u8], wait: Wait) -> Result, buf: &[u8], wait: Wait) -> Result( ) -> Result, NetError> { let slot = TABLE.lock().reserve().ok_or(NetError::ResourceExhausted)?; let made = (|| { - let region = SharedMemory::create(REGION_BYTES).map_err(|e| match e { + let exhausted = |e| match e { SyscallError::ResourceExhausted => NetError::ResourceExhausted, _ => NetError::Io, - })?; + }; + let region = SharedMemory::reserve(REGION_BYTES, RECEIVE_RING + RECEIVE_INITIAL as usize).map_err(exhausted)?; + region.commit(SEND_RING, SEND_INITIAL as usize).map_err(exhausted)?; // SAFETY: a fresh region of `REGION_BYTES`, 2 MiB aligned, whose header // is reached only as atomics. let words = unsafe { &*(region.as_ptr() as *const Words) }; words[NODELAY].store(u32::from(nodelay), Ordering::Relaxed); - let (rx, tx) = session::client(words, LAYOUT); + let (rx, tx) = session::client(words, LAYOUT, SEND_INITIAL); let lent = region.share().map_err(|_| NetError::Io)?; let (answer, bells, connection) = ask(lent)?; let (addr, port) = answer.local(); diff --git a/sdk/toyos/src/shm.rs b/sdk/toyos/src/shm.rs index c681793b6b3..de2acdc5091 100644 --- a/sdk/toyos/src/shm.rs +++ b/sdk/toyos/src/shm.rs @@ -62,6 +62,18 @@ impl SharedMemory { Self::adopt(OwnedHandle(syscall::shm_create(len)?), len) } + /// [`create`](Self::create), only the pages of its first `committed` + /// bytes memory until [`Self::commit`] makes more of it. + pub fn reserve(size: usize, committed: usize) -> Result { + Self::adopt(OwnedHandle(syscall::shm_reserve(size, committed)?), size) + } + + /// Make every page `[offset, offset + len)` of the region touches memory + /// that is not yet ([`syscall::shm_commit`]). + pub fn commit(&self, offset: usize, len: usize) -> Result<(), SyscallError> { + syscall::shm_commit(self.handle.raw(), offset, len) + } + /// [`create`](Self::create), counted to the job `job` is a handle to, /// carrying `WRITE`, rather than this process's own: memory laid out /// for a child started in it. diff --git a/sdk/toyos/src/transport/mod.rs b/sdk/toyos/src/transport/mod.rs index 92e94f5a7fe..e5ca57f328a 100644 --- a/sdk/toyos/src/transport/mod.rs +++ b/sdk/toyos/src/transport/mod.rs @@ -102,4 +102,7 @@ pub enum Violation { /// This end's own cursor, as another holder of the end stored it, names no /// position. OwnCursor, + /// A ring's size is past its place, or names a growth further from this + /// end than the old ring. + Size, } diff --git a/sdk/toyos/src/transport/session.rs b/sdk/toyos/src/transport/session.rs index dafe2b8db4f..ad54bd766d2 100644 --- a/sdk/toyos/src/transport/session.rs +++ b/sdk/toyos/src/transport/session.rs @@ -61,7 +61,7 @@ use core::num::NonZeroU16; use core::sync::atomic::Ordering; -use super::stream::{Consumer, Place, Producer, WAKE}; +use super::stream::{Consumer, Place, Producer, Size, WAKE}; use super::{word, Span, Violation, Word}; /// An `end` while its ring runs. @@ -174,7 +174,7 @@ pub struct Layout { impl Layout { /// The receive ring at `rx`, the send ring at `tx`, their `end`s and /// gates, [`shutdown_read`]'s word and the word that says the server hung - /// up. A layout whose words are not fourteen distinct ones below `N` does + /// up. A layout whose words are not sixteen distinct ones below `N` does /// not compile where it is a constant. pub const fn new< const RX_END: usize, @@ -187,14 +187,24 @@ impl Layout { rx: Place, tx: Place, ) -> Self { - let [a, b, c, d] = rx.words(); - let [e, f, g, h] = tx.words(); + let [a, b, c, d, e] = rx.words(); + let [f, g, h, i, j] = tx.words(); let own = [RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]; assert!(below(&own, N), "a word past the words"); - assert!(distinct(&[a, b, c, d, e, f, g, h, RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]), "a session's fourteen words are fourteen"); + assert!(distinct(&[a, b, c, d, e, f, g, h, i, j, RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]), "a session's sixteen words are sixteen"); Self { rx, tx, rx_end: RX_END, tx_end: TX_END, rx_gate: RX_GATE, tx_gate: TX_GATE, read_shut: READ_SHUT, gone: GONE } } + /// Where the receive ring is. + pub const fn receive_place(&self) -> Place { + self.rx + } + + /// Where the send ring is. + pub const fn send_place(&self) -> Place { + self.tx + } + /// The receive ring's `end` as the server last stored it. pub fn received_end(&self, page: &[W; N]) -> u32 { word(page, self.rx_end).load(Ordering::Acquire) @@ -352,6 +362,8 @@ pub trait Half: sealed::Sealed { #[derive(Debug)] pub struct Receiving { ring: Consumer, + /// The ring's capacity this process made sure is memory. + reached: u32, end: usize, read_shut: usize, common: Common, @@ -362,6 +374,8 @@ pub struct Receiving { #[derive(Debug)] pub struct Sending { ring: Producer, + /// The ring's capacity this process made sure is memory. + reached: u32, /// The server's `end`, which a failure makes ready. received: usize, end: usize, @@ -515,14 +529,17 @@ impl Half for Sending { } } -/// The first holder's two halves, the end's cursors stored 0. +/// The first holder's two halves, the end's cursors stored 0 and its send +/// ring `send` bytes, which the region holds as memory. pub fn client( page: &[W; N], layout: Layout, + send: u32, ) -> (Receiving, Sending) { let (mut receiving, mut sending) = holder(layout, Holder::FIRST); receiving.ring = Consumer::new(page, layout.rx); - sending.ring = Producer::new(page, layout.tx); + sending.ring = Producer::new(page, layout.tx, send); + sending.reached = send; (receiving, sending) } @@ -535,6 +552,7 @@ pub fn holder( let common = |gate, shut| Common { gate, shut, gone: layout.gone, holder }; let receiving = Receiving { ring: Consumer::joining(layout.rx), + reached: 0, end: layout.rx_end, read_shut: layout.read_shut, common: common(layout.rx_gate, None), @@ -542,6 +560,7 @@ pub fn holder( }; let sending = Sending { ring: Producer::joining(layout.tx), + reached: 0, received: layout.rx_end, end: layout.tx_end, common: common(layout.tx_gate, Some(layout.tx_end)), @@ -807,13 +826,17 @@ where /// Reads at most `max` bytes, each span copied out by `copy` at its offset /// into the caller's buffer: the count, at least one, or 0 for an orderly end /// with every byte read, or once a holder shut the receiving half down. A read -/// of no bytes answers 0 at once. +/// of no bytes answers 0 at once. Before it copies past a capacity this +/// process has not reached, `reach` makes the ring's first that many bytes +/// memory; one it cannot is a size the server never made, +/// [`Violation::Size`]. pub fn read( page: &[W; N], lock: &L, bells: &B, mode: Mode, max: u32, + mut reach: impl FnMut(u32) -> bool, mut copy: impl FnMut(Span, usize), ) -> Result where @@ -829,8 +852,18 @@ where return (if seen.end == ORDERLY { Ok(0) } else { Err(Refused::Failed(seen.end)) }, false); } let n = seen.ready.min(max); + let cap = half.ring.capacity(); + if cap > half.reached { + if !reach(cap) { + return (Err(Refused::Violation(Violation::Size)), false); + } + half.reached = cap; + } + let Some(spans) = half.ring.span(0, n) else { + return (Err(Refused::Violation(Violation::TailPastDepth)), false); + }; let mut at = 0usize; - for span in half.ring.span(0, n).into_iter().flatten() { + for span in spans { copy(span, at); at = at.saturating_add(span.len); } @@ -844,13 +877,20 @@ where /// Writes at most `len` bytes, each span copied in by `copy` from its offset /// into the caller's buffer: the count, at least one. A shutdown, the server's /// hang-up or a failure it stored is answered, and nothing copied, whatever -/// room there is. A write of no bytes answers 0 at once. +/// room there is. A write of no bytes answers 0 at once. Before it copies +/// past a capacity this process has not reached, as another holder may have +/// grown the ring to, `reach` makes the ring's first that many bytes memory; +/// one it cannot is [`Violation::Size`]. A write the room cannot take whole, +/// or that fills the ring, doubles the ring, up to its place, where `reach` +/// makes the ring's first twice its capacity memory; where it cannot, the +/// ring stays as it is. pub fn write( page: &[W; N], lock: &L, bells: &B, mode: Mode, len: u32, + mut reach: impl FnMut(u32) -> bool, mut copy: impl FnMut(Span, usize), ) -> Result where @@ -867,14 +907,32 @@ where Writable::Failed(end) => (Err(Refused::Failed(end)), false), Writable::Room(room) => { let n = room.min(len); + // A size another holder stored is believed only once it is memory. + let resumed = half.ring.capacity(); + if resumed > half.reached { + if !reach(resumed) { + return (Err(Refused::Violation(Violation::Size)), false); + } + half.reached = resumed; + } + let Some(spans) = half.ring.span(0, n) else { + return (Err(Refused::Violation(Violation::HeadPastTail)), false); + }; let mut at = 0usize; - for span in half.ring.span(0, n).into_iter().flatten() { + for span in spans { copy(span, at); at = at.saturating_add(span.len); } if !half.ring.advance(n) { return (Err(Refused::Violation(Violation::HeadPastTail)), false); } + if (n < len || n == room) && half.ring.can_grow() { + let cap = half.ring.capacity().saturating_mul(2); + if half.reached >= cap || reach(cap) { + half.reached = half.reached.max(cap); + half.ring.grow(page); + } + } (Ok(n), half.ring.publish(page)) } }) @@ -1107,14 +1165,55 @@ pub struct Server { } impl Server { - /// The server's end, its cursors stored 0. - pub fn new(page: &[W; N], layout: Layout) -> Self { - Self { layout, rx: Producer::new(page, layout.rx), tx: Consumer::new(page, layout.tx), ended: OPEN } + /// The server's end, its cursors stored 0 and its receive ring `receive` + /// bytes, which the region holds as memory. + pub fn new(page: &[W; N], layout: Layout, receive: u32) -> Self { + Self { layout, rx: Producer::new(page, layout.rx, receive), tx: Consumer::new(page, layout.tx), ended: OPEN } } /// Bytes published that the client has not released. pub fn held(&mut self, page: &[W; N]) -> Result { - Ok(RX.saturating_sub(self.rx.room(page)?)) + self.rx.room(page)?; + self.rx.held(page) + } + + /// The most the receive ring holds from the client's head [`Self::held`] + /// last read: what netstack may place in it. + pub fn receive_holds(&self) -> u32 { + self.rx.holds() + } + + /// The receive ring's size, which every byte netstack puts in it is laid + /// out by. + pub fn receive_size(&self) -> Size { + self.rx.size() + } + + /// The send ring's size as [`Self::sent`] last judged it, which the + /// caller makes sure is memory before it copies a byte by it. + pub fn send_size(&self) -> Size { + self.tx.size() + } + + /// The receive ring's capacity. + pub fn receive_size_cap(&self) -> u32 { + self.rx.capacity() + } + + /// The send ring's capacity as [`Self::sent`] last judged it. + pub fn send_size_cap(&self) -> u32 { + self.tx.capacity() + } + + /// Whether [`Self::grow_receive`] may double the receive ring now. + pub fn can_grow_receive(&self) -> bool { + self.rx.can_grow() + } + + /// Double the receive ring, the region below twice its capacity made + /// memory first. + pub fn grow_receive(&mut self, page: &[W; N]) { + self.rx.grow(page); } /// Publishes `n` more bytes, already in the ring behind the tail. @@ -1168,7 +1267,7 @@ impl Server { } self.ended = ORDERLY; word(page, self.layout.rx_end).store(ORDERLY, Ordering::Release); - let [_, _, data, _] = self.layout.rx.words(); + let [_, _, data, _, _] = self.layout.rx.words(); Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0) } @@ -1178,8 +1277,8 @@ impl Server { pub fn fail(&mut self, page: &[W; N], code: u32) -> Rings { self.ended = code; word(page, self.layout.rx_end).store(code, Ordering::Release); - let [_, _, data, _] = self.layout.rx.words(); - let [_, _, _, room] = self.layout.tx.words(); + let [_, _, data, _, _] = self.layout.rx.words(); + let [_, _, _, room, _] = self.layout.tx.words(); let receive = Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0); receive.and(Rings::of(Way::Send, word(page, room).swap(0, WAKE) != 0)) } @@ -1227,14 +1326,14 @@ mod tests { use core::cell::{Cell, RefCell}; use core::sync::atomic::AtomicU32; - const N: usize = 14; + const N: usize = 16; const C: u32 = 4; const RX_GATE: usize = 10; const TX_GATE: usize = 11; const TX_END: usize = 9; const LAYOUT: Layout = Layout::new::<8, TX_END, RX_GATE, TX_GATE, 12, 13>( - Place::new::<0, 1, 2, 3, 0>(), - Place::new::<4, 5, 6, 7, { C as usize }>(), + Place::new::<0, 1, 2, 3, 14, 0>(), + Place::new::<4, 5, 6, 7, 15, { C as usize }>(), ); /// The send ring's two wants: the server's and the client's. const TX_DATA: usize = 6; @@ -1298,14 +1397,14 @@ mod tests { /// The first holder's halves and another's, over one page. fn two(page: &[AtomicU32; N]) -> (Halves, Halves) { - let (rx, tx) = client(page, LAYOUT); + let (rx, tx) = client(page, LAYOUT, C); let (other_rx, other_tx) = holder(LAYOUT, second()); ((Held(RefCell::new(rx)), Held(RefCell::new(tx))), (Held(RefCell::new(other_rx)), Held(RefCell::new(other_tx)))) } fn write_one(page: &[AtomicU32; N], tx: &Held>, bells: &B, mode: Mode) -> (Result, u32) { let mut copied = 0; - let wrote = write(page, tx, bells, mode, 1, |span, _| copied += span.len as u32); + let wrote = write(page, tx, bells, mode, 1, |_| true, |span, _| copied += span.len as u32); (wrote, copied) } @@ -1316,8 +1415,8 @@ mod tests { fn a_write_with_room_after_a_reset_answers_it_and_copies_nothing() { const RESET: u32 = 2; let page = page(); - let (_, sending) = client(&page, LAYOUT); - let mut server = Server::new(&page, LAYOUT); + let (_, sending) = client(&page, LAYOUT, C); + let mut server = Server::new(&page, LAYOUT, C); let _ = server.fail(&page, RESET); let (sending, bells) = (Held(RefCell::new(sending)), quiet()); assert_eq!(write_one(&page, &sending, &bells, Mode::Nonblocking), (Err(Refused::Failed(RESET)), 0)); @@ -1328,12 +1427,12 @@ mod tests { #[test] fn a_read_takes_every_byte_before_the_end() { let page = page(); - let (receiving, _) = client(&page, LAYOUT); - let mut server = Server::new(&page, LAYOUT); + let (receiving, _) = client(&page, LAYOUT, C); + let mut server = Server::new(&page, LAYOUT, C); let _ = server.publish(&page, 3); let _ = server.finish(&page); let (receiving, bells) = (Held(RefCell::new(receiving)), quiet()); - let read = |max| read(&page, &receiving, &bells, Mode::Nonblocking, max, |_, _| {}); + let read = |max| read(&page, &receiving, &bells, Mode::Nonblocking, max, |_| true, |_, _| {}); assert_eq!((read(2), read(2), read(2)), (Ok(2), Ok(1), Ok(0))); } @@ -1363,8 +1462,8 @@ mod tests { #[test] fn a_departure_frees_only_the_departed_holders_gates() { let page = page(); - let mut server = Server::new(&page, LAYOUT); - let _ = client(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); + let _ = client(&page, LAYOUT, C); page[RX_GATE].store(2 | WANT, Ordering::Relaxed); page[TX_GATE].store(1, Ordering::Relaxed); assert_eq!(server.departed(&page, second()), Ok(Rings::default()), "nobody armed a want"); @@ -1383,7 +1482,7 @@ mod tests { #[test] fn a_write_after_another_holders_shutdown_answers_shut_and_publishes_nothing() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((_, tx), (_, other_tx)) = two(&page); let bells = quiet(); shutdown(&page, &tx, &bells); @@ -1402,9 +1501,9 @@ mod tests { fn a_shutdown_under_another_holders_gate_is_stored_by_its_release() { let page = page(); let ((_, tx), _) = two(&page); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); // A full send ring: no room rings the shutting holder, only the shut. - let filled = write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_, _| {}); + let filled = write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_| true, |_, _| {}); assert_eq!(filled, Ok(C)); let owner = Common { gate: TX_GATE, shut: Some(TX_END), gone: 13, holder: second() }; assert_eq!(acquire(&page, owner), Ok(())); @@ -1450,7 +1549,7 @@ mod tests { fn a_blocking_write_under_another_holders_gate_waits_for_that_holders_gate() { let page = page(); let ((_, tx), _) = two(&page); - let server = RefCell::new(Server::new(&page, LAYOUT)); + let server = RefCell::new(Server::new(&page, LAYOUT, C)); let owner = Common { gate: TX_GATE, shut: Some(TX_END), gone: 13, holder: second() }; assert_eq!(acquire(&page, owner), Ok(())); let gated = Quiet { @@ -1463,7 +1562,7 @@ mod tests { }; assert_eq!(write_one(&page, &tx, &gated, Mode::Blocking(None)), (Ok(1), 1)); assert_eq!(*gated.gates.borrow(), [Some(2)]); - assert_eq!(write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_, _| {}), Ok(C - 1), "the ring filled"); + assert_eq!(write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_| true, |_, _| {}), Ok(C - 1), "the ring filled"); let full = Quiet { rang: Cell::new(0), gone: false, @@ -1490,7 +1589,7 @@ mod tests { assert_eq!(gated.gate(), Some(2)); let _ = unwatch(&page, &tx, &bells, gated); let _ = release(&page, owner); - assert_eq!(write(&page, &tx, &bells, Mode::Nonblocking, C, |_, _| {}), Ok(C), "the ring filled"); + assert_eq!(write(&page, &tx, &bells, Mode::Nonblocking, C, |_| true, |_, _| {}), Ok(C), "the ring filled"); let Polled::Watching(full) = watch(&page, &tx, &bells) else { panic!("room in a full ring") }; assert_eq!(full.gate(), None, "a watch for room"); let _ = unwatch(&page, &tx, &bells, full); @@ -1498,18 +1597,35 @@ mod tests { /// Once one holder saw the server hang up, a write with room in another /// answers it and copies nothing, and a read answers it after the bytes. + /// A holder that joined believes the size the last holder stored only + /// once it made that much of the ring memory: a size it cannot commit, + /// which another holder may have stored without committing, is + /// `Violation::Size`, and nothing is copied. + #[test] + fn a_joined_writer_commits_the_size_it_resumed_before_it_copies() { + let page = page(); + let ((_, _), (_, other_tx)) = two(&page); + let bells = quiet(); + let mut copied = 0; + let refused = write(&page, &other_tx, &bells, Mode::Nonblocking, 1, |_| false, |span, _| copied += span.len); + assert_eq!((refused, copied), (Err(Refused::Violation(Violation::Size)), 0)); + let mut reached = Vec::new(); + let wrote = write(&page, &other_tx, &bells, Mode::Nonblocking, 1, |cap| { reached.push(cap); true }, |_, _| {}); + assert_eq!((wrote, reached), (Ok(1), vec![C]), "the resumed capacity, committed once"); + } + #[test] fn a_hang_up_one_holder_saw_is_every_holders() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((rx, _), (other_rx, other_tx)) = two(&page); let _ = server.publish(&page, 1); let hung = Quiet:: { rang: Cell::new(0), gone: true, waited: None, gates: RefCell::new(Vec::new()) }; - assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_, _| {}), Ok(1), "the byte before the hang-up"); - assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::Gone)); + assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_| true, |_, _| {}), Ok(1), "the byte before the hang-up"); + assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::Gone)); let quiet = quiet(); assert_eq!(write_one(&page, &other_tx, &quiet, Mode::Nonblocking), (Err(Refused::Gone), 0)); - assert_eq!(read(&page, &other_rx, &quiet, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::Gone)); + assert_eq!(read(&page, &other_rx, &quiet, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::Gone)); assert_eq!(quiet.rang.get(), 0, "nothing was rung"); } @@ -1548,15 +1664,15 @@ mod tests { #[test] fn a_read_shutdown_is_every_holders() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((rx, _), (other_rx, _)) = two(&page); let bells = quiet(); - assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::WouldBlock), "armed"); + assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::WouldBlock), "armed"); shutdown_read(&page, &rx, &bells); assert_eq!(bells.rang.get(), 1); assert_eq!(server.rung(&page), Ok(Rings { receive: true, send: false }), "the reader that armed"); let _ = server.publish(&page, 2); - assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_, _| {}), Ok(0)); + assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| {}), Ok(0)); assert_eq!(server.held(&page), Ok(2), "a byte was read after the shutdown"); } @@ -1565,10 +1681,10 @@ mod tests { #[test] fn a_copy_that_unwinds_lets_the_gate_go_and_rings() { let page = page(); - let (_, tx) = client(&page, LAYOUT); + let (_, tx) = client(&page, LAYOUT, C); let (tx, bells) = (Held(RefCell::new(tx)), quiet()); let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - write(&page, &tx, &bells, Mode::Nonblocking, 1, |_, _| panic!("the copy faulted")) + write(&page, &tx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| panic!("the copy faulted")) })); assert!(unwound.is_err()); assert_eq!(page[TX_GATE].load(Ordering::Relaxed), 0, "the gate is still held"); diff --git a/sdk/toyos/src/transport/stream.rs b/sdk/toyos/src/transport/stream.rs index 7e1a00c0464..49eac5f838e 100644 --- a/sdk/toyos/src/transport/stream.rs +++ b/sdk/toyos/src/transport/stream.rs @@ -1,4 +1,5 @@ -//! A single-producer ring of `C` bytes, its cursors and wants among `N` words. +//! A single-producer ring of at most `C` bytes, its cursors, wants and size +//! among `N` words. //! //! **A producer copies bytes in through the [`Span`]s it is handed, then //! publishes its tail with `Release`; a consumer loads the tail with `Acquire`, @@ -8,11 +9,27 @@ //! peer's cursor only when it asks for its room or what is ready, and stores //! its own once a batch. //! -//! **A cursor is a position modulo twice the ring**, so a full ring and an empty -//! one differ and the ring may be any size. The peer's is believed only as a -//! position at most the ring away from what this end last stored; otherwise -//! [`Violation::TailPastDepth`] or [`Violation::HeadPastTail`] names it. Every -//! span lies in the ring whatever the peer wrote. +//! **A cursor is a stream position, counting every byte and wrapping at +//! 2^32**, and the ring's capacity is a power of two, so a byte's offset is +//! its position below the capacity whichever lap the cursor is on. The peer's +//! cursor is believed only as a position at most the ring away from what this +//! end last stored; otherwise [`Violation::TailPastDepth`] or +//! [`Violation::HeadPastTail`] names it. Every span lies in the ring whatever +//! the peer wrote. +//! +//! **The producer alone grows the ring, by doubling, whatever it holds.** The +//! size word holds the capacity and, while a growth is pending, the position +//! it switches at: the old ring past the head the producer last read, past +//! every byte it may have placed. A byte before it keeps its offset below the +//! old capacity and a byte from it on lies below the new one, so no byte +//! moves. Until the consumer's head passes the switch the producer's room is +//! still counted in the old capacity, so a byte from the switch on lands +//! where no unread byte lies: it would have to be a whole old ring past one. +//! The consumer loads the size after the tail, so it never reads a byte by a +//! size older than the byte. A consumer only touches the ring below the +//! capacity it was told, and judges a size before it believes it: a capacity +//! past `C`, or a switch more than the old ring past what it released, is +//! [`Violation::Size`]. //! //! **A sleeper never misses a wake.** An end about to sleep [`arm`]s its want, //! which exchanges 1 into it and only then looks at the peer's cursor again; it @@ -27,7 +44,7 @@ //! //! **An end may have several holders**, which take turns under a gate of the //! session's. Such a holder keeps no cursor between turns: it [`resume`]s its -//! own from the word the last holder stored, and refuses one past the lap. +//! own, and a producer its size, from the words the last holder stored. //! //! [`arm`]: Consumer::arm //! [`resume`]: Consumer::resume @@ -47,6 +64,9 @@ pub(super) const WAKE: Ordering = Ordering::Relaxed; /// after the consumer's copy out of it, which the consumer's store carries. const GIVE_BACK: Ordering = if cfg!(feature = "transport-head-relaxed") { Ordering::Relaxed } else { Ordering::Release }; const TAKE_BACK: Ordering = if cfg!(feature = "transport-head-relaxed") { Ordering::Relaxed } else { Ordering::Acquire }; +/// A consumer loads the size after the tail it reads by, so it never maps a +/// byte by an older size than the one it was written by. +const SIZE_AFTER_TAIL: bool = !cfg!(feature = "transport-size-before-tail"); /// `n` as an offset into the region; `Place::new` holds `usize` to every `u32`. #[allow(clippy::as_conversions)] @@ -54,19 +74,76 @@ const fn bytes(n: u32) -> usize { n as usize } -/// Where a ring of `C` bytes is: its four words among `N` and the region offset -/// of its first byte. +/// Whether position `at` comes before `switch`: they are within 2^31. +#[allow(clippy::as_conversions, clippy::cast_possible_wrap)] +const fn before(at: u32, switch: u32) -> bool { + (switch.wrapping_sub(at) as i32) > 0 +} + +/// A ring's capacity and the growth to it that may be pending. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Size { + /// Bytes, a power of two. + cap: u32, + /// The position the ring doubled to `cap` at, while a byte before it may + /// be unread. + switch: Option, +} + +/// The size word: log2 of the capacity in its low five bits, bit 5 for a +/// growth pending, and the switch's low 24 bits from bit 8. +const LOG: u32 = 0x1F; +const PENDING: u32 = 1 << 5; +const SWITCH: u32 = 8; + +impl Size { + /// The capacity the byte at `at` lies under. + fn cap_at(&self, at: u32) -> u32 { + match self.switch { + Some(switch) if before(at, switch) => self.cap / 2, + _ => self.cap, + } + } + + fn word(&self) -> u32 { + let log = self.cap.trailing_zeros(); + match self.switch { + Some(switch) => log | PENDING | (switch << SWITCH), + None => log, + } + } + + /// The size `word` says, its switch read as the position within 2^23 of + /// `near`; `None` for a capacity that is no power of two to `max`, or a + /// growth to a capacity of one. + #[allow(clippy::as_conversions, clippy::cast_possible_wrap, clippy::cast_sign_loss)] + fn of(word: u32, near: u32, max: u32) -> Option { + let log = word & LOG; + let cap = 1u32.checked_shl(log).filter(|&cap| cap <= max)?; + if word & PENDING == 0 { + return Some(Self { cap, switch: None }); + } + if cap < 2 { + return None; + } + let apart = (((word >> SWITCH).wrapping_sub(near) << SWITCH) as i32) >> SWITCH; + Some(Self { cap, switch: Some(near.wrapping_add(apart as u32)) }) + } +} + +/// Where a ring of at most `C` bytes is: its five words among `N` and the +/// region offset of its first byte. /// /// ``` /// use toyos::transport::stream::Place; -/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 48, 4096>(); +/// const RING: Place<4096, 80> = Place::new::<0, 16, 32, 48, 64, 4096>(); /// ``` /// /// A place with a word at `N` or past it does not compile: /// /// ```compile_fail,E0080 /// use toyos::transport::stream::Place; -/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 64, 4096>(); +/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 48, 64, 4096>(); /// ``` #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Place { @@ -76,65 +153,77 @@ pub struct Place { data: usize, /// Set by a producer about to sleep on a full ring. room: usize, + /// The producer's [`Size`]. + size: usize, base: usize, } impl Place { - /// What a cursor counts modulo. - const LAP: u32 = C.wrapping_mul(2); - - /// The cursor at word `at`, as an end's own last stored it. - fn own(page: &[W; N], at: usize) -> Result { - let cursor = word(page, at).load(Ordering::Acquire); - if cursor < Self::LAP { Ok(cursor) } else { Err(Violation::OwnCursor) } - } - - pub const fn new( - ) -> Self { + pub const fn new< + const HEAD: usize, + const TAIL: usize, + const DATA: usize, + const ROOM: usize, + const SIZE: usize, + const BASE: usize, + >() -> Self { const { - assert!(C > 0 && C <= 1 << 30, "a ring holds a byte and at most a gibibyte"); + assert!(C.is_power_of_two() && C <= 1 << 22, "a ring holds a power of two bytes, at most 4 MiB"); assert!(usize::BITS >= u32::BITS, "a ring offset is a u32"); - assert!(HEAD < N && TAIL < N && DATA < N && ROOM < N, "a place names a word past its words"); + assert!(HEAD < N && TAIL < N && DATA < N && ROOM < N && SIZE < N, "a place names a word past its words"); assert!( - HEAD != TAIL && HEAD != DATA && HEAD != ROOM && TAIL != DATA && TAIL != ROOM && DATA != ROOM, - "a ring's four words are four" + HEAD != TAIL && HEAD != DATA && HEAD != ROOM && HEAD != SIZE && TAIL != DATA && TAIL != ROOM + && TAIL != SIZE && DATA != ROOM && DATA != SIZE && ROOM != SIZE, + "a ring's five words are five" ); assert!(BASE.checked_add(bytes(C)).is_some(), "a ring ends inside the address space"); } - Self { head: HEAD, tail: TAIL, data: DATA, room: ROOM, base: BASE } + Self { head: HEAD, tail: TAIL, data: DATA, room: ROOM, size: SIZE, base: BASE } } - /// Its four words: head, tail and the two wants. - pub(super) const fn words(&self) -> [usize; 4] { - [self.head, self.tail, self.data, self.room] + /// Its five words: head, tail, the two wants and the size. + pub(super) const fn words(&self) -> [usize; 5] { + [self.head, self.tail, self.data, self.room, self.size] } - /// `at` moved `n` on; both are below the lap. - fn forward(at: u32, n: u32) -> u32 { - let to = at.wrapping_add(n); - if to >= Self::LAP { to.wrapping_sub(Self::LAP) } else { to } + /// The region offset of its first byte. + pub const fn base(&self) -> usize { + self.base } - /// How far `to` is past `from`; both are below the lap. + /// How far `to` is past `from`. fn distance(from: u32, to: u32) -> u32 { - let apart = to.wrapping_sub(from); - if to >= from { apart } else { apart.wrapping_add(Self::LAP) } - } - - /// How far the peer's `claimed` position is from this end's, by `apart`: - /// at most the ring. A word past the lap names no position, so it is - /// farther than any ring. - fn reach(claimed: u32, apart: impl FnOnce(u32) -> u32, broken: Violation) -> Result { - clamp(if claimed < Self::LAP { apart(claimed) } else { u32::MAX }, C, broken) + to.wrapping_sub(from) + } + + /// The region's bytes `len` on from position `at` by `size`, in ring + /// order: at most two before the switch and two after, since what lies + /// before it is at most the old ring and the rest at most the new. + /// `None` where they would be more, which no size this end believed makes. + pub fn spans(&self, size: Size, at: u32, len: u32) -> Option<[Span; 4]> { + let mut spans = [Span { offset: self.base, len: 0 }; 4]; + let (mut at, mut left) = (at, len); + for span in &mut spans { + if left == 0 { + break; + } + let cap = size.cap_at(at); + let to_switch = match size.switch { + Some(switch) if before(at, switch) => switch.wrapping_sub(at), + _ => u32::MAX, + }; + let slot = at & cap.wrapping_sub(1); + let run = left.min(to_switch).min(cap.wrapping_sub(slot)); + *span = Span { offset: self.base.wrapping_add(bytes(slot)), len: bytes(run) }; + at = at.wrapping_add(run); + left = left.wrapping_sub(run); + } + (left == 0).then_some(spans) } - /// The region's bytes `len` on from position `at`, in ring order: the - /// second span is empty unless they wrap. - fn spans(&self, at: u32, len: u32) -> [Span; 2] { - let slot = if at >= C { at.wrapping_sub(C) } else { at }; - let first = len.min(C.wrapping_sub(slot)); - let start = self.base.wrapping_add(bytes(slot)); - [Span { offset: start, len: bytes(first) }, Span { offset: self.base, len: bytes(len.wrapping_sub(first)) }] + /// The size the producer stored, judged by an end at `near`. + fn size(&self, page: &[W; N], near: u32) -> Result { + Size::of(word(page, self.size).load(Ordering::Acquire), near, C).ok_or(Violation::Size) } } @@ -143,51 +232,106 @@ impl Place { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Producer { place: Place, + size: Size, local: u32, published: u32, /// Bytes past `local` that may be written, as last seen. room: u32, + /// The head the room was last counted from. + head: u32, } impl Producer { - /// This end of the ring at `place`, its tail stored 0. - pub fn new(page: &[W; N], place: Place) -> Self { + /// This end of the ring at `place`, its tail stored 0 and its capacity + /// `cap`, a power of two to `C`. + pub fn new(page: &[W; N], place: Place, cap: u32) -> Self { + assert!(cap.is_power_of_two() && cap <= C, "a ring of {cap} bytes, at most {C}"); + let size = Size { cap, switch: None }; + word(page, place.size).store(size.word(), Ordering::Release); word(page, place.tail).store(0, Ordering::Release); - Self { place, local: 0, published: 0, room: C } + Self { place, size, local: 0, published: 0, room: cap, head: 0 } } - /// The end at `place` as its holders left it, its tail resumed before each - /// use ([`Self::resume`]). + /// The end at `place` as its holders left it, its tail and size resumed + /// before each use ([`Self::resume`]). pub const fn joining(place: Place) -> Self { - Self { place, local: 0, published: 0, room: 0 } + Self { place, size: Size { cap: 1, switch: None }, local: 0, published: 0, room: 0, head: 0 } } - /// Takes up the tail the end's last holder stored, nothing advanced past it. + /// Takes up the tail and the size the end's last holder stored, nothing + /// advanced past it. A switch more than the old ring from the tail is no + /// size a holder stored: `OwnCursor`. pub fn resume(&mut self, page: &[W; N]) -> Result<(), Violation> { if cfg!(feature = "transport-no-resume") { return Ok(()); } - let tail = Place::::own(page, self.place.tail)?; - *self = Self { local: tail, published: tail, room: 0, ..*self }; + let tail = word(page, self.place.tail).load(Ordering::Acquire); + let size = self.place.size(page, tail).map_err(|_| Violation::OwnCursor)?; + let far = |switch: u32| Place::::distance(switch, tail).min(Place::::distance(tail, switch)) > size.cap / 2; + if size.switch.is_some_and(far) { + return Err(Violation::OwnCursor); + } + *self = Self { size, local: tail, published: tail, room: 0, head: tail, ..*self }; Ok(()) } + /// The capacity it holds now: what a span may reach below the base. + pub fn capacity(&self) -> u32 { + self.size.cap + } + + /// The size its spans are laid out by, for an adapter that copies by + /// position ([`Place::spans`]). + pub fn size(&self) -> Size { + self.size + } + + /// The most it holds from the head [`Self::room`] last read: the old + /// capacity while a growth is pending before it. + pub fn holds(&self) -> u32 { + self.size.cap_at(self.head) + } + + /// Whether [`Self::grow`] may double it now: below `C`, and no growth + /// pending. + pub fn can_grow(&self) -> bool { + self.size.cap < C && self.size.switch.is_none() + } + + /// Double the ring, where [`Self::can_grow`] says so, from the old ring + /// past the head [`Self::room`] last read: the caller made the region + /// below twice the capacity memory first. The room grows once the + /// consumer is past the switch. + pub fn grow(&mut self, page: &[W; N]) { + assert!(self.can_grow(), "a ring of {} grown, at most {C} and none pending", self.size.cap); + let switch = self.head.wrapping_add(self.size.cap); + self.size = Size { cap: self.size.cap.wrapping_mul(2), switch: Some(switch) }; + word(page, self.place.size).store(self.size.word(), Ordering::Release); + } + /// How many bytes past the tail may be written, the consumer's head read - /// again. + /// again; a growth the consumer is past is no longer pending. pub fn room(&mut self, page: &[W; N]) -> Result { let head = word(page, self.place.head).load(TAKE_BACK); - let published = self.published; - let unreleased = - Place::::reach(head, |head| Place::::distance(head, published), Violation::HeadPastTail)?; + let unreleased = clamp(Place::::distance(head, self.published), self.size.cap, Violation::HeadPastTail)?; + if self.size.switch.is_some_and(|switch| !before(head, switch)) { + self.size.switch = None; + word(page, self.place.size).store(self.size.word(), Ordering::Release); + } + let cap = if cfg!(feature = "transport-grow-room-now") { self.size.cap } else { self.size.cap_at(head) }; let pending = Place::::distance(self.published, self.local); - self.room = C.saturating_sub(unreleased.saturating_add(pending)); + self.room = cap.saturating_sub(unreleased.saturating_add(pending)); + self.head = head; Ok(self.room) } /// Where to copy the `len` bytes `at` past the tail; `None` past the room /// last seen. - pub fn span(&self, at: u32, len: u32) -> Option<[Span; 2]> { - (at.checked_add(len)? <= self.room).then(|| self.place.spans(Place::::forward(self.local, at), len)) + pub fn span(&self, at: u32, len: u32) -> Option<[Span; 4]> { + if at.checked_add(len)? > self.room { + return None; + } + self.place.spans(self.size, self.local.wrapping_add(at), len) } /// Count `n` more bytes written at the tail; `false`, and nothing counted, @@ -196,7 +340,7 @@ impl Producer { #[must_use] pub fn advance(&mut self, n: u32) -> bool { let Some(room) = self.room.checked_sub(n) else { return false }; - self.local = Place::::forward(self.local, n); + self.local = self.local.wrapping_add(n); self.room = room; true } @@ -220,15 +364,18 @@ impl Producer { self.room(page) } + /// Bytes published that the consumer has not released. + pub fn held(&self, page: &[W; N]) -> Result { + let head = word(page, self.place.head).load(TAKE_BACK); + clamp(Place::::distance(head, self.published), self.size.cap, Violation::HeadPastTail) + } + /// The consumer asked to be rung back: `true`, and its want exchanged for /// 0, if it is set and the consumer has bytes past its head or the ring /// `ended`. With neither, nothing is exchanged and the want waits for the /// next publish. pub fn ring_back(&self, page: &[W; N], ended: bool) -> Result { - let head = word(page, self.place.head).load(TAKE_BACK); - let published = self.published; - let held = - Place::::reach(head, |head| Place::::distance(head, published), Violation::HeadPastTail)?; + let held = self.held(page)?; Ok((held > 0 || ended) && word(page, self.place.data).swap(0, WAKE) != 0) } } @@ -238,6 +385,8 @@ impl Producer { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Consumer { place: Place, + /// The size as last judged. + size: Size, local: u32, released: u32, /// Bytes published past `local`, as last seen. @@ -248,13 +397,13 @@ impl Consumer { /// This end of the ring at `place`, its head stored 0. pub fn new(page: &[W; N], place: Place) -> Self { word(page, place.head).store(0, Ordering::Release); - Self { place, local: 0, released: 0, ready: 0 } + Self::joining(place) } /// The end at `place` as its holders left it, its head resumed before each /// use ([`Self::resume`]). pub const fn joining(place: Place) -> Self { - Self { place, local: 0, released: 0, ready: 0 } + Self { place, size: Size { cap: 1, switch: None }, local: 0, released: 0, ready: 0 } } /// Takes up the head the end's last holder stored, nothing advanced past it. @@ -262,26 +411,54 @@ impl Consumer { if cfg!(feature = "transport-no-resume") { return Ok(()); } - let head = Place::::own(page, self.place.head)?; + let head = word(page, self.place.head).load(Ordering::Acquire); *self = Self { local: head, released: head, ready: 0, ..*self }; Ok(()) } - /// How many bytes past the head are published, the producer's tail read - /// again. - pub fn ready(&mut self, page: &[W; N]) -> Result { + /// The capacity last judged: what a span may reach below the base, which + /// the caller makes sure is memory before it copies. + pub fn capacity(&self) -> u32 { + self.size.cap + } + + /// The size last judged, as [`Producer::size`]. + pub fn size(&self) -> Size { + self.size + } + + /// The producer's tail and size, judged: the bytes published past what + /// this end released, and the size they are laid out by. + fn judged(&self, page: &[W; N]) -> Result<(u32, Size), Violation> { + let early = if SIZE_AFTER_TAIL { None } else { Some(self.place.size(page, self.released)) }; let tail = word(page, self.place.tail).load(Ordering::Acquire); - let released = self.released; - let published = - Place::::reach(tail, |tail| Place::::distance(released, tail), Violation::TailPastDepth)?; + let size = match early { + Some(size) => size?, + None => self.place.size(page, self.released)?, + }; + if size.switch.is_some_and(|switch| before(self.released, switch) && Place::::distance(self.released, switch) > size.cap / 2) { + return Err(Violation::Size); + } + let published = clamp(Place::::distance(self.released, tail), size.cap, Violation::TailPastDepth)?; + Ok((published, size)) + } + + /// How many bytes past the head are published, the producer's tail and + /// size read again. + pub fn ready(&mut self, page: &[W; N]) -> Result { + let (published, size) = self.judged(page)?; + self.size = size; self.ready = published.saturating_sub(Place::::distance(self.released, self.local)); Ok(self.ready) } /// Where to copy the `len` bytes `at` past the head from; `None` past what /// was last seen ready. - pub fn span(&self, at: u32, len: u32) -> Option<[Span; 2]> { - (at.checked_add(len)? <= self.ready).then(|| self.place.spans(Place::::forward(self.local, at), len)) + pub fn span(&self, at: u32, len: u32) -> Option<[Span; 4]> { + if at.checked_add(len)? > self.ready { + return None; + } + self.place.spans(self.size, self.local.wrapping_add(at), len) } /// Count `n` more bytes read at the head; `false`, and nothing counted, @@ -290,7 +467,7 @@ impl Consumer { #[must_use] pub fn advance(&mut self, n: u32) -> bool { let Some(ready) = self.ready.checked_sub(n) else { return false }; - self.local = Place::::forward(self.local, n); + self.local = self.local.wrapping_add(n); self.ready = ready; true } @@ -319,11 +496,8 @@ impl Consumer { /// or its direction `answers` without room. With neither, nothing is /// exchanged and the want waits for the next release. pub fn ring_back(&self, page: &[W; N], answers: bool) -> Result { - let tail = word(page, self.place.tail).load(Ordering::Acquire); - let released = self.released; - let unreleased = - Place::::reach(tail, |tail| Place::::distance(released, tail), Violation::TailPastDepth)?; - Ok((unreleased < C || answers) && word(page, self.place.room).swap(0, WAKE) != 0) + let (unreleased, size) = self.judged(page)?; + Ok((unreleased < size.cap_at(self.released) || answers) && word(page, self.place.room).swap(0, WAKE) != 0) } } @@ -332,7 +506,8 @@ mod tests { use super::*; use core::sync::atomic::AtomicU32; - const WORDS: usize = 4; + const WORDS: usize = 5; + const SIZE: usize = 4; /// Off any alignment, so a span that forgets the base lands outside it. const BASE: usize = 3; /// Bytes on either side of the ring that no span may reach. @@ -343,10 +518,10 @@ mod tests { } fn place() -> Place { - Place::new::<0, 1, 2, 3, BASE>() + Place::new::<0, 1, 2, 3, SIZE, BASE>() } - /// A region with the ring between two guards. + /// A region with the ring, at its most, between two guards. fn region() -> Vec { vec![0xA5; BASE + C as usize + GUARD] } @@ -373,14 +548,15 @@ mod tests { } } - fn inside(spans: [Span; 2], len: u32) { - assert_eq!(spans[0].len + spans[1].len, len as usize, "{spans:?} is not {len} bytes"); - for span in spans { - assert!(span.offset >= BASE && span.offset + span.len <= BASE + C as usize, "{span:?} leaves the ring"); + /// The spans hold `len` bytes and lie below `cap` past the base. + fn inside(spans: [Span; 4], len: u32, cap: u32) { + assert_eq!(spans.iter().map(|span| span.len).sum::(), len as usize, "{spans:?} is not {len} bytes"); + for span in spans.iter().filter(|span| span.len > 0) { + assert!(span.offset >= BASE && span.offset + span.len <= BASE + cap as usize, "{span:?} leaves a ring of {cap}"); } } - fn put(region: &mut [u8], spans: [Span; 2], from: u64) { + fn put(region: &mut [u8], spans: [Span; 4], from: u64) { let mut at = from; for span in spans { for b in &mut region[span.offset..span.offset + span.len] { @@ -390,7 +566,7 @@ mod tests { } } - fn get(region: &[u8], spans: [Span; 2]) -> Vec { + fn get(region: &[u8], spans: [Span; 4]) -> Vec { spans.iter().flat_map(|span| region[span.offset..span.offset + span.len].iter().copied()).collect() } @@ -398,26 +574,54 @@ mod tests { (from..from + u64::from(len)).map(byte).collect() } - /// The oracle is a stream that never wraps: four counts of bytes since the - /// start. The producer places each batch out of order, the consumer reads - /// at offsets as well as at its head, and every room, every count ready and - /// every byte read is what the counts say, round the lap many times. - fn agrees(seed: u64) { + /// What the producer's room is, by the module's rule: the old capacity + /// until the head is past a pending switch. + struct Model { + cap: u32, + switch: Option, + } + + impl Model { + fn room(&mut self, written: u64, released: u64) -> u64 { + match self.switch { + Some(switch) if released < switch => u64::from(self.cap / 2) - (written - released), + _ => { + self.switch = None; + u64::from(self.cap) - (written - released) + } + } + } + } + + /// The oracle is a stream that never wraps: four counts of bytes since + /// the start, and the rule the room follows. The producer starts at + /// `start`, places each batch out of order, and doubles the ring at + /// random to `C`; the consumer reads at offsets as well as at its head, + /// and every room, every count ready and every byte read is what the + /// counts say, round the lap many times. A byte the producer placed and + /// had not published when it grew is read as it was written. + fn agrees(start: u32, seed: u64) { let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), start), Consumer::new(&page, place::())); let mut region = region::(); let (mut written, mut published, mut read, mut released) = (0u64, 0u64, 0u64, 0u64); + let mut model = Model { cap: start, switch: None }; let mut rng = Rng(seed); for _ in 0..400 * C + 400 { - match rng.below(4) { + match rng.below(5) { 0 => { let room = tx.room(&page).unwrap(); - assert_eq!(u64::from(room), u64::from(C) - (written - released), "the room"); + assert_eq!(u64::from(room), model.room(written, released), "the room"); let n = rng.below(room + 1); let split = rng.below(n + 1); - for (at, len) in [(split, n - split), (0, split)] { - put(&mut region, tx.span(at, len).unwrap(), written + u64::from(at)); + // The later part is placed first, and a growth may come + // between it and the rest. + put(&mut region, tx.span(split, n - split).unwrap(), written + u64::from(split)); + if tx.can_grow() && rng.below(8) == 0 { + tx.grow(&page); + model = Model { cap: model.cap * 2, switch: Some(released + u64::from(model.cap)) }; } + put(&mut region, tx.span(0, split).unwrap(), written); assert!(tx.advance(n)); written += u64::from(n); } @@ -432,33 +636,131 @@ mod tests { let len = rng.below(ready + 1); (len, rng.below(ready - len + 1)) }; - assert_eq!(get(®ion, rx.span(at, len).unwrap()), expected(read + u64::from(at), len)); + let spans = rx.span(at, len).unwrap(); + inside(spans, len, rx.capacity()); + assert_eq!(get(®ion, spans), expected(read + u64::from(at), len)); let n = rng.below(ready + 1); assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n)); assert!(rx.advance(n)); read += u64::from(n); } - _ => { + 3 => { let _ = rx.release(&page); released = read; } + _ => {} } } - assert!(read > 4 * u64::from(C), "{read} bytes did not go round the lap twice"); + assert!(read > 4 * u64::from(model.cap), "{read} bytes did not go round the lap twice"); assert!(region[..BASE].iter().chain(®ion[BASE + C as usize..]).all(|&b| b == 0xA5), "a guard was written"); } #[test] fn the_ring_agrees_with_a_stream_that_never_wraps() { for seed in 1..=16 { - agrees::<1>(seed); - agrees::<2>(seed); - agrees::<5>(seed); - agrees::<8>(seed); - agrees::<61>(seed); + agrees::<1>(1, seed); + agrees::<2>(2, seed); + agrees::<8>(8, seed); + agrees::<64>(64, seed); + } + } + + /// Both ends resumed from a position just short of 2^32, so every + /// growth's switch and its 24-bit reconstruction cross the wrap. + fn agrees_across_the_wrap(start: u32, seed: u64) { + let page = page(); + let from = u32::MAX - 3 * C; + let (mut tx, mut rx) = (Producer::new(&page, place::(), start), Consumer::new(&page, place::())); + page[0].store(from, Ordering::Release); + page[1].store(from, Ordering::Release); + assert_eq!((tx.resume(&page), rx.resume(&page)), (Ok(()), Ok(()))); + let mut region = region::(); + let (mut written, mut published, mut read, mut released) = (0u64, 0u64, 0u64, 0u64); + let mut model = Model { cap: start, switch: None }; + let mut rng = Rng(seed); + for _ in 0..200 * C { + match rng.below(4) { + 0 => { + let room = tx.room(&page).unwrap(); + assert_eq!(u64::from(room), model.room(written, released), "the room"); + if tx.can_grow() && rng.below(4) == 0 { + tx.grow(&page); + model = Model { cap: model.cap * 2, switch: Some(released + u64::from(model.cap)) }; + } + let n = rng.below(room + 1); + put(&mut region, tx.span(0, n).unwrap(), written); + assert!(tx.advance(n)); + written += u64::from(n); + let _ = tx.publish(&page); + published = written; + } + 1 | 2 => { + let ready = rx.ready(&page).unwrap(); + assert_eq!(u64::from(ready), published - read, "the bytes ready"); + let n = rng.below(ready + 1); + assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n), "bytes {read}.. past the wrap"); + assert!(rx.advance(n)); + read += u64::from(n); + } + _ => { + let _ = rx.release(&page); + released = read; + } + } + } + assert!(u64::from(from) + read > 1 << 32, "the stream did not cross 2^32"); + } + + #[test] + fn a_ring_that_grows_across_the_wrap_agrees_with_the_stream() { + for seed in 1..=64 { + agrees_across_the_wrap::<64>(1, seed); + } + } + + #[test] + fn a_ring_that_grows_agrees_with_the_stream_too() { + for seed in 1..=64 { + agrees::<64>(1, seed); + agrees::<256>(4, seed); } } + /// The ring never empties: the producer keeps it at least half full and + /// the consumer reads only what leaves that much behind, so a growth that + /// waited for an empty ring never comes. It reaches its place, each + /// doubling within a few old rings of bytes, and every byte is read as + /// written. + #[test] + fn a_ring_kept_full_grows_to_its_place() { + const C: u32 = 1 << 12; + let page = page(); + let (mut tx, mut rx) = (Producer::new(&page, place::(), 16), Consumer::new(&page, place::())); + let mut region = region::(); + let (mut written, mut read) = (0u64, 0u64); + let mut grew_at = Vec::new(); + while tx.capacity() < C || tx.holds() < C { + assert!(written < 64 * u64::from(C), "the ring held {} after {written} bytes", tx.holds()); + let room = tx.room(&page).unwrap(); + if tx.can_grow() { + tx.grow(&page); + grew_at.push(written); + } + put(&mut region, tx.span(0, room).unwrap(), written); + assert!(tx.advance(room)); + written += u64::from(room); + let _ = tx.publish(&page); + let ready = rx.ready(&page).unwrap(); + let n = ready - ready.min(tx.holds() / 2); + assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n)); + assert!(rx.advance(n)); + read += u64::from(n); + let _ = rx.release(&page); + assert!(written > read, "the ring emptied"); + } + assert_eq!(grew_at.len(), 8, "16 bytes doubled to {C}: {grew_at:?}"); + } + /// A word a peer might write: anything, a position near the lap, or an edge. fn hostile(rng: &mut Rng) -> u32 { match rng.below(3) { @@ -468,24 +770,32 @@ mod tests { } } + /// A size word a peer might write: any capacity log, pending or not, a + /// switch anywhere. + fn hostile_size(rng: &mut Rng) -> u32 { + match rng.below(2) { + 0 => rng.next() as u32, + _ => rng.below(32) | (rng.below(2) << 5) | ((rng.next() as u32) << 8), + } + } + /// Whatever a producer, or another holder of this end, stores in any of - /// the four words, the consumer holds at most the ring past what it - /// released, every span it is handed lies in the ring, a tail it does not - /// believe is `TailPastDepth` and a head it resumes past the lap is - /// `OwnCursor`. + /// the five words, the consumer holds at most its capacity past what it + /// released, which is at most the place's, every span it is handed lies + /// below it, and what it does not believe is `TailPastDepth` or `Size`. fn hostile_producer(seed: u64) { let page = page(); let mut rx = Consumer::new(&page, place::()); let mut rng = Rng(seed); let mut held = 0; for _ in 0..2000 { - page[rng.below(4) as usize].store(hostile::(&mut rng), Ordering::Release); + let at = rng.below(5) as usize; + let value = if at == SIZE { hostile_size(&mut rng) } else { hostile::(&mut rng) }; + page[at].store(value, Ordering::Release); let seen = match rng.below(5) { 4 => { - match rx.resume(&page) { - Ok(()) => held = 0, - Err(violation) => assert_eq!(violation, Violation::OwnCursor), - } + assert_eq!(rx.resume(&page), Ok(())); + held = 0; continue; } 0 => rx.ready(&page), @@ -494,7 +804,7 @@ mod tests { let n = rng.below(C + 1); match rx.span(0, n) { Some(spans) => { - inside::(spans, n); + inside(spans, n, rx.capacity()); assert!(rx.advance(n)); held += n; } @@ -508,12 +818,14 @@ mod tests { continue; } }; + assert!(rx.capacity() <= C); match seen { Ok(ready) => assert!( - u64::from(held) + u64::from(ready) <= u64::from(C), - "holds {held} and was told of {ready} more in a ring of {C}" + u64::from(held) + u64::from(ready) <= u64::from(rx.capacity()), + "holds {held} and was told of {ready} more in a ring of {}", + rx.capacity() ), - Err(violation) => assert_eq!(violation, Violation::TailPastDepth), + Err(violation) => assert!(matches!(violation, Violation::TailPastDepth | Violation::Size), "{violation:?}"), } } } @@ -522,23 +834,25 @@ mod tests { fn a_hostile_producer_hands_the_consumer_at_most_the_ring() { for seed in 1..=64 { hostile_producer::<1>(seed); - hostile_producer::<5>(seed); hostile_producer::<8>(seed); + hostile_producer::<64>(seed); } } /// Whatever a consumer, or another holder of this end, stores in any of - /// the four words, the producer writes at most the ring past the head it - /// was shown, every span it is handed lies in the ring, a head it does not - /// believe is `HeadPastTail` and a tail it resumes past the lap is + /// the five words, the producer writes at most its capacity past the head + /// it was shown, every span it is handed lies below it, a head it does not + /// believe is `HeadPastTail` and a size it resumes that it never stored is /// `OwnCursor`. fn hostile_consumer(seed: u64) { let page = page(); - let mut tx = Producer::new(&page, place::()); + let mut tx = Producer::new(&page, place::(), C); let mut rng = Rng(seed); let mut pending = 0; for _ in 0..2000 { - page[rng.below(4) as usize].store(hostile::(&mut rng), Ordering::Release); + let at = rng.below(5) as usize; + let value = if at == SIZE { hostile_size(&mut rng) } else { hostile::(&mut rng) }; + page[at].store(value, Ordering::Release); let seen = match rng.below(5) { 4 => { match tx.resume(&page) { @@ -553,7 +867,7 @@ mod tests { let n = rng.below(C + 1); match tx.span(0, n) { Some(spans) => { - inside::(spans, n); + inside(spans, n, tx.capacity()); assert!(tx.advance(n)); pending += n; } @@ -569,8 +883,9 @@ mod tests { }; match seen { Ok(room) => assert!( - u64::from(pending) + u64::from(room) <= u64::from(C), - "{pending} unpublished and {room} more in a ring of {C}" + u64::from(pending) + u64::from(room) <= u64::from(tx.capacity()), + "{pending} unpublished and {room} more in a ring of {}", + tx.capacity() ), Err(violation) => assert_eq!(violation, Violation::HeadPastTail), } @@ -581,39 +896,31 @@ mod tests { fn a_hostile_consumer_gives_the_producer_at_most_the_ring() { for seed in 1..=64 { hostile_consumer::<1>(seed); - hostile_consumer::<5>(seed); hostile_consumer::<8>(seed); + hostile_consumer::<64>(seed); } } - /// From every position round the lap, twice: a tail the ring ahead is - /// believed, and one a byte further, a word past the lap or the last word - /// is not; a head the ring behind is believed, and one a byte past what was - /// published or a word past the lap is not. + /// From positions on both sides of the u32 wrap: a tail the ring ahead is + /// believed and one a byte further is not; a head the ring behind is + /// believed, and one a byte past what was published is not. #[test] fn a_cursor_is_believed_up_to_the_ring_and_no_further() { - const C: u32 = 5; - const LAP: u32 = 2 * C; + const C: u32 = 4; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); - for at in 0..2 * LAP { - let here = at % LAP; - let tails = [ - ((here + C) % LAP, Ok(C)), - ((here + C + 1) % LAP, Err(Violation::TailPastDepth)), - (LAP, Err(Violation::TailPastDepth)), - (u32::MAX, Err(Violation::TailPastDepth)), - ]; - for (tail, believed) in tails { + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); + let from = u32::MAX - 2 * C; + page[0].store(from, Ordering::Release); + page[1].store(from, Ordering::Release); + assert_eq!((rx.resume(&page), tx.resume(&page)), (Ok(()), Ok(()))); + for _ in 0..4 * C { + let here = rx.released; + for (tail, believed) in [(here.wrapping_add(C), Ok(C)), (here.wrapping_add(C + 1), Err(Violation::TailPastDepth))] { page[1].store(tail, Ordering::Release); assert_eq!(rx.ready(&page), believed, "tail {tail} with {here} released"); } - let heads = [ - ((here + LAP - C) % LAP, Ok(0)), - ((here + 1) % LAP, Err(Violation::HeadPastTail)), - (LAP, Err(Violation::HeadPastTail)), - ]; - for (head, believed) in heads { + page[1].store(here, Ordering::Release); + for (head, believed) in [(here.wrapping_sub(C), Ok(0)), (here.wrapping_add(1), Err(Violation::HeadPastTail))] { page[0].store(head, Ordering::Release); assert_eq!(tx.room(&page), believed, "head {head} with {here} published"); } @@ -627,25 +934,21 @@ mod tests { } } - /// A holder resumes its own cursor from any position below the lap, and - /// refuses the lap and past it. + /// A producer resumes the size its end's last holder stored, and refuses + /// one whose switch is further than the old ring from its tail. #[test] - fn a_resumed_cursor_is_believed_below_the_lap_and_no_further() { - const C: u32 = 5; + fn a_resumed_size_is_believed_only_near_the_tail() { + const C: u32 = 16; let page = page(); - let (mut tx, mut rx) = (Producer::joining(place::()), Consumer::joining(place::())); - for at in [0, 1, C, 2 * C - 1] { - page[0].store(at, Ordering::Release); - page[1].store(at, Ordering::Release); - assert_eq!((rx.resume(&page), tx.resume(&page)), (Ok(()), Ok(())), "a cursor at {at}"); - assert_eq!((rx.ready(&page), tx.room(&page)), (Ok(0), Ok(C)), "the ring at {at} is empty"); - } - for at in [2 * C, 2 * C + 1, u32::MAX] { - page[0].store(at, Ordering::Release); - page[1].store(at, Ordering::Release); - assert_eq!(rx.resume(&page), Err(Violation::OwnCursor), "a head at {at}"); - assert_eq!(tx.resume(&page), Err(Violation::OwnCursor), "a tail at {at}"); - } + let mut first = Producer::new(&page, place::(), 4); + assert_eq!(first.room(&page), Ok(4)); + first.grow(&page); + let mut next = Producer::joining(place::()); + assert_eq!(next.resume(&page), Ok(())); + assert_eq!((next.capacity(), next.can_grow()), (8, false), "the growth the first holder made"); + let far = Size { cap: 8, switch: Some(64) }.word(); + page[SIZE].store(far, Ordering::Release); + assert_eq!(next.resume(&page), Err(Violation::OwnCursor)); } /// A ring back exchanges a want only of a peer whose direction is ready, @@ -654,7 +957,7 @@ mod tests { fn a_ring_back_answers_only_a_peer_that_armed_and_is_ready() { const C: u32 = 2; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); assert_eq!(rx.arm(&page), Ok(0)); assert_eq!(tx.ring_back(&page, false), Ok(false), "nothing to read: the want stays"); assert_eq!(tx.ring_back(&page, true), Ok(true), "the ring ended"); @@ -680,7 +983,7 @@ mod tests { fn an_end_rings_only_a_peer_that_armed() { const C: u32 = 4; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); assert!(tx.advance(1)); assert!(!tx.publish(&page), "nobody armed"); assert_eq!(rx.arm(&page), Ok(1), "the look after arming finds the byte"); diff --git a/sdk/toyos/tests/transport_loom.rs b/sdk/toyos/tests/transport_loom.rs index ae82d0d8e71..e9c6c0555c7 100644 --- a/sdk/toyos/tests/transport_loom.rs +++ b/sdk/toyos/tests/transport_loom.rs @@ -3,6 +3,8 @@ //! //! - **Publication**: a consumer that sees a tail sees every word of the //! entries below it. `transport-publish-relaxed` takes the edge away. +//! - **Growth**: a ring doubled while it holds unread bytes reads each byte as +//! written. `transport-grow-room-now` and `transport-size-before-tail` red it. //! - **A hostile peer** leaves the honest end with entries, nothing, or a //! named violation, and never more entries than the ring holds. //! `transport-no-clamp` believes the peer. @@ -162,15 +164,15 @@ fn a_hostile_consumer_yields_room_or_a_violation() { /// failure unwinds the model, which aborts the run before it says which test /// failed (`issues/loom-arc-panics-in-its-drop-while-a-failed-model-unwinds.md`). struct Stream { - page: [Shared; 4], + page: [Shared; 5], bytes: [Cell; RING as usize], to_consumer: Notify, to_producer: Notify, } const RING: u32 = 1; -const STREAM: stream::Place = stream::Place::new::<0, 1, 2, 3, 0>(); -type StreamEnds = (stream::Producer, stream::Consumer); +const STREAM: stream::Place = stream::Place::new::<0, 1, 2, 3, 4, 0>(); +type StreamEnds = (stream::Producer, stream::Consumer); fn stream() -> (Rc, StreamEnds) { let s = Rc::new(Stream { @@ -179,13 +181,13 @@ fn stream() -> (Rc, StreamEnds) { to_consumer: Notify::new(), to_producer: Notify::new(), }); - let ends = (stream::Producer::new(&s.page, STREAM), stream::Consumer::new(&s.page, STREAM)); + let ends = (stream::Producer::new(&s.page, STREAM, RING), stream::Consumer::new(&s.page, STREAM)); (s, ends) } /// Writes `byte` into the room the producer was last shown and publishes it, /// ringing a consumer that armed. -fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { +fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { for span in tx.span(0, 1).expect("a byte of room") { for at in span.offset..span.offset + span.len { s.bytes[at].set(byte); @@ -199,7 +201,7 @@ fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { /// Reads the byte the consumer was last shown and releases it, ringing a /// producer that armed. -fn receive(s: &Stream, rx: &mut stream::Consumer) -> u32 { +fn receive(s: &Stream, rx: &mut stream::Consumer) -> u32 { let mut read = Vec::new(); for span in rx.span(0, 1).expect("a byte ready") { read.extend((span.offset..span.offset + span.len).map(|at| s.bytes[at].get())); @@ -284,6 +286,78 @@ fn a_producer_that_looked_again_is_rung_for_room_it_missed() { }); } +/// A ring of at most four bytes, grown from two while the producer keeps +/// writing: no byte waits for it to empty, and every byte is read as written. +/// The consumer may hold two unread bytes at offsets 0 and 1 when the producer +/// grows, so the first byte past the switch, at offset 0 again, lands on an +/// unread one, a write beside the consumer's read, where the room is counted +/// in the new capacity at once (`transport-grow-room-now`); and a consumer +/// that loads the size before the tail meets a tail past what that size holds +/// (`transport-size-before-tail`). +#[test] +fn a_ring_grown_while_it_holds_bytes_reads_each_as_written() { + const MOST: u32 = 4; + const GROWN: stream::Place = stream::Place::new::<0, 1, 2, 3, 4, 0>(); + bounded(|| { + let page: Rc<[Shared; 5]> = Rc::new(core::array::from_fn(|_| Shared(AtomicU32::new(0)))); + let bytes: Rc<[Cell; MOST as usize]> = Rc::new(core::array::from_fn(|_| Cell::new(0))); + let (mut tx, mut rx) = (stream::Producer::new(&page, GROWN, 2), stream::Consumer::new(&page, GROWN)); + let put = |tx: &mut stream::Producer, bytes: &[Cell], at: u32| { + for span in tx.span(0, 1).expect("a byte of room") { + (span.offset..span.offset + span.len).for_each(|i| bytes[i].set(at)); + } + assert!(tx.advance(1)); + let _ = tx.publish(&page); + }; + // Positions 0 and 1 go round once and are read, then 2 and 3 are + // written and left unread. + for at in 0..2 { + assert_eq!(tx.room(&page), Ok(2 - at)); + put(&mut tx, &bytes[..], at); + } + assert_eq!(rx.ready(&page), Ok(2)); + assert!(rx.advance(2)); + let _ = rx.release(&page); + for at in 2..4 { + assert!(tx.room(&page).expect("the consumer keeps the protocol") > 0); + put(&mut tx, &bytes[..], at); + } + let (producer_page, producer_bytes) = (Rc::clone(&page), Rc::clone(&bytes)); + let producer = loom::thread::spawn(move || { + // The consumer may already be reading: the switch is the old + // ring past whatever head this look finds. + tx.room(&producer_page).expect("the consumer keeps the protocol"); + tx.grow(&producer_page); + for at in 4..7 { + while tx.room(&producer_page).expect("the consumer keeps the protocol") == 0 { + loom::thread::yield_now(); + } + for span in tx.span(0, 1).expect("a byte of room") { + (span.offset..span.offset + span.len).for_each(|i| producer_bytes[i].set(at)); + } + assert!(tx.advance(1)); + let _ = tx.publish(&producer_page); + } + tx.capacity() + }); + let mut read = Vec::new(); + while read.len() < 5 { + match rx.ready(&page).expect("the producer keeps the protocol") { + 0 => loom::thread::yield_now(), + _ => { + for span in rx.span(0, 1).expect("a byte ready") { + read.extend((span.offset..span.offset + span.len).map(|i| bytes[i].get())); + } + assert!(rx.advance(1)); + let _ = rx.release(&page); + } + } + } + assert_eq!(producer.join().expect("the producer thread"), MOST); + assert_eq!(read, [2, 3, 4, 5, 6], "a byte was read other than as written"); + }); +} + // ---- a session: the client's sequences and the server's rings ---- // // The models below drive `session`'s own code: a holder's `read`, `write`, @@ -328,10 +402,10 @@ fn bounded_by(preemptions: usize, model: impl Fn() + Sync + Send + 'static) { } const BYTES: u32 = 2; -const SESSION_WORDS: usize = 14; +const SESSION_WORDS: usize = 16; const SESSION: Layout = Layout::new::<8, 9, 10, 11, 12, 13>( - stream::Place::new::<0, 1, 2, 3, 0>(), - stream::Place::new::<4, 5, 6, 7, { BYTES as usize }>(), + stream::Place::new::<0, 1, 2, 3, 14, 0>(), + stream::Place::new::<4, 5, 6, 7, 15, { BYTES as usize }>(), ); /// A failure the server stores in the receive ring's `end`. const RESET: u32 = 2; @@ -563,9 +637,9 @@ impl Session { /// A session whose end has `holders` holders, and its server end. fn new(holders: u16) -> (Rc, End) { let page: [Shared; SESSION_WORDS] = core::array::from_fn(|_| Shared(AtomicU32::new(0))); - let mut all = vec![Holding::new(session::client(&page, SESSION))]; + let mut all = vec![Holding::new(session::client(&page, SESSION, BYTES))]; all.extend((2..=holders).map(|n| Holding::new(session::holder(SESSION, Holder::new(n).unwrap())))); - let server = Server::new(&page, SESSION); + let server = Server::new(&page, SESSION, BYTES); let s = Session { page, bytes: core::array::from_fn(|_| Cell::new(0)), @@ -602,7 +676,7 @@ impl Session { /// A holder reads at most `max` bytes over `page`: the bytes it read. fn read_by(&self, page: &[W; SESSION_WORDS], by: &By, mode: Mode<()>, max: u32) -> Result, Refused> { let mut read = Vec::new(); - session::read(page, &self.holders[by.at].rx, by, mode, max, |span, _| { + session::read(page, &self.holders[by.at].rx, by, mode, max, |_| true, |span, _| { read.extend((span.offset..span.offset + span.len).map(|at| self.bytes[at].get())); })?; Ok(read) @@ -610,7 +684,7 @@ impl Session { /// A holder writes `byte` over `page`, once at most. fn write_by(&self, page: &[W; SESSION_WORDS], by: &By, mode: Mode<()>, byte: u32) -> Result { - session::write(page, &self.holders[by.at].tx, by, mode, 1, |span, _| { + session::write(page, &self.holders[by.at].tx, by, mode, 1, |_| true, |span, _| { for at in span.offset..span.offset + span.len { if lives(by.fate) { self.bytes[at].set(byte); diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index 95a73008b9e..48e4c6ce212 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -49,7 +49,7 @@ mod streams; pub use holders::{Bell, HolderId}; pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId}; -pub use streams::{ConnectRefused, ExportRefused, Session, StreamEvent, StreamId}; +pub use streams::{ConnectRefused, ExportRefused, Memory, Session, StreamEvent, StreamId}; use alloc::vec; use alloc::vec::Vec; diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index 57c316ef634..7d48234b777 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -10,8 +10,15 @@ //! [tcp] has room, and `rx_room` only while the window last offered is below the update //! threshold. //! -//! **Every word read from the region is the client's, and judged**: a cursor past its bound, or -//! text published after its own shutdown, is a [`Violation`] that resets that stream alone. +//! **Every word read from the region is the client's, and judged**: a cursor past its bound, +//! text published after its own shutdown, or a send ring larger than the region can be made +//! memory for, is a [`Violation`] that resets that stream alone. +//! +//! **The rings grow under their producers** (`toyos::transport::stream`): the node doubles the +//! receive ring when [tcp] wants a larger receive buffer than it holds, after making the region +//! below the new capacity memory, and makes the send ring's memory before [tcp] reads by a size +//! the client grew it to. [tcp] addresses each ring by stream position, laid out by the size the +//! pass last judged. //! //! **A stream has holders**: the client that connected or accepted it, and one more for each //! [`Node::export`], each a process of its own over the one region, each taking a place past @@ -26,15 +33,18 @@ use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; +use alloc::rc::Rc; +use core::cell::Cell; use core::sync::atomic::Ordering; use core::time::Duration; -use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; +use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_INITIAL, RECEIVE_RING, RESET, SEND_INITIAL, SEND_RING, TIMED_OUT, UNREACHABLE, WORDS}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Lent, Options, State, Status}; use toyos_net_wire::Instant; use toyos::transport::session::{Holder, Rings}; -use toyos::transport::Violation; +use toyos::transport::stream::{Place, Size}; +use toyos::transport::{Span, Violation}; use crate::lease::Stack; use crate::Node; @@ -44,14 +54,96 @@ use crate::Node; pub trait Session { /// The region's header. fn words(&self) -> &Words; - /// The region's receive ring, as [tcp]'s receive buffer. - fn receive(&self) -> Box; - /// The region's send ring, as [tcp]'s send buffer. - fn send(&self) -> Box; + /// The region's bytes, which the node lays its two rings out in: its header and each + /// ring's first size are memory already (`RECEIVE_INITIAL`, `SEND_INITIAL`). + fn memory(&self) -> Rc; /// Rings the doorbells `rings` names, of every holder, never waiting. fn ring(&self, rings: Rings); } +/// A region's bytes by offset, reached only by copies. +pub trait Memory: core::fmt::Debug { + /// Copies `data` in at `at`, inside memory [`Self::commit`] made. + fn put(&self, at: usize, data: &[u8]); + /// Copies `out.len()` bytes out from `at`, inside memory [`Self::commit`] made. + fn take(&self, at: usize, out: &mut [u8]); + /// Makes `[at, at + len)` memory where it is not yet; `false` where it cannot be. + fn commit(&self, at: usize, len: usize) -> bool; +} + +/// What a ring is laid out by and holds now, which the pass writes and [tcp]'s storage reads. +#[derive(Clone, Copy, Debug)] +struct Shape { + size: Size, + /// [tcp]'s storage length: the receive ring's capacity from its oldest byte unread, and the + /// most of the send ring netstack committed. + holds: u32, + /// What [tcp] last asked it to hold. + wanted: usize, +} + +/// A ring's shape, shared by the pass and [tcp]'s storage. +type Shared = Rc>; + +/// One of the region's rings, as [tcp]'s storage. +#[derive(Debug)] +struct LentRing { + memory: Rc, + place: Place, + shape: Shared, +} + +impl LentRing { + /// The region's bytes at positions `[at, at + len)`, each with where it starts among them. + fn spans(&self, at: usize, len: usize) -> [(Span, usize); 4] { + let none = (Span { offset: 0, len: 0 }, 0); + let mut out = [none; 4]; + let (Ok(at), Ok(len)) = (u32::try_from(at & 0xFFFF_FFFF), u32::try_from(len)) else { + unreachable!("[tcp] hands a ring at most its capacity, {len} bytes at {at}") + }; + let Some(spans) = self.place.spans(self.shape.get().size, at, len) else { + unreachable!("{len} bytes at {at} lie in at most four spans of a size the pass judged") + }; + let mut from = 0usize; + for (slot, span) in out.iter_mut().zip(spans) { + *slot = (span, from); + from = from.saturating_add(span.len); + } + out + } +} + +impl Lent for LentRing { + fn len(&self) -> usize { + usize::try_from(self.shape.get().holds).unwrap_or(0) + } + + fn most(&self) -> usize { + usize::try_from(C).unwrap_or(0) + } + + fn want(&mut self, bytes: usize) { + let shape = self.shape.get(); + self.shape.set(Shape { wanted: shape.wanted.max(bytes), ..shape }); + } + + fn put(&mut self, at: usize, data: &[u8]) { + for (span, from) in self.spans(at, data.len()) { + if let Some(part) = data.get(from..from.saturating_add(span.len)) { + self.memory.put(span.offset, part); + } + } + } + + fn take(&self, at: usize, out: &mut [u8]) { + for (span, from) in self.spans(at, out.len()) { + if let Some(part) = out.get_mut(from..from.saturating_add(span.len)) { + self.memory.take(span.offset, part); + } + } + } +} + /// Why a connect made no stream. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ConnectRefused { @@ -112,6 +204,8 @@ fn count(n: usize) -> u32 { struct Stream { conn: ConnId, session: Box, + /// The receive ring's and the send ring's shapes, as [tcp]'s storage reads them. + shapes: Option<(Shared, Shared)>, /// The session's server end, once the connection is established and its buffers are the /// region's; `None` while a connect is not answered. server: Option, @@ -135,10 +229,17 @@ impl Stream { /// A connection whose buffers are the region's from here on, and its server end. fn lent(&mut self, stack: &mut Stack) { let session = &self.session; - if stack.tcp_lend(self.conn, session.receive(), session.send()).is_err() { + let memory = session.memory(); + let server = Server::new(session.words(), LAYOUT, RECEIVE_INITIAL); + let rx = Rc::new(Cell::new(Shape { size: server.receive_size(), holds: RECEIVE_INITIAL, wanted: 0 })); + let tx = Rc::new(Cell::new(Shape { size: server.send_size(), holds: SEND_INITIAL, wanted: 0 })); + let receive = LentRing { memory: memory.clone(), place: LAYOUT.receive_place(), shape: rx.clone() }; + let send = LentRing { memory, place: LAYOUT.send_place(), shape: tx.clone() }; + if stack.tcp_lend(self.conn, Box::new(receive), Box::new(send)).is_err() { unreachable!("[tcp] would not lend an established connection its client's buffers"); } - self.server = Some(Server::new(session.words(), LAYOUT)); + self.server = Some(server); + self.shapes = Some((rx, tx)); } /// One pass. `false` lets the stream go: its connection is [tcp]'s alone or gone. @@ -182,10 +283,19 @@ impl Stream { fn moved(&mut self, now: Instant, stack: &mut Stack) -> Result { let conn = self.conn; let Some(server) = self.server.as_mut() else { unreachable!("a stream moves only once established") }; + let Some((rx_shape, tx_shape)) = self.shapes.as_ref() else { unreachable!("a lent stream has its shapes") }; let words = self.session.words(); let mut rings = Rings::default(); let keep = loop { let held = server.held(words)?; + let rx = rx_shape.get(); + if rx.wanted > usize::try_from(server.receive_size_cap()).unwrap_or(usize::MAX) && server.can_grow_receive() { + let more = server.receive_size_cap().saturating_mul(2); + if self.session.memory().commit(RECEIVE_RING, usize::try_from(more).unwrap_or(usize::MAX)) { + server.grow_receive(words); + } + } + rx_shape.set(Shape { size: server.receive_size(), holds: server.receive_holds(), ..rx_shape.get() }); let released = self.held.checked_sub(held).ok_or(Violation::HeadPastTail)?; if released > 0 { stack.tcp_consume(now, conn, released); @@ -206,7 +316,14 @@ impl Stream { } self.queued = queued; let shut = server.shut(words); - let fresh = server.sent(words)?.checked_sub(queued).ok_or(Violation::TailPastDepth)?; + let sent = server.sent(words)?; + let tx = tx_shape.get(); + let cap = server.send_size_cap(); + if cap > tx.holds && !self.session.memory().commit(SEND_RING, usize::try_from(cap).unwrap_or(usize::MAX)) { + return Err(Violation::Size); + } + tx_shape.set(Shape { size: server.send_size(), holds: cap.max(tx.holds), ..tx }); + let fresh = sent.checked_sub(queued).ok_or(Violation::TailPastDepth)?; if fresh > 0 && status.failure.is_none() { match stack.tcp_queue(now, conn, fresh) { // [tcp]'s send buffer is the whole ring, so what the ring held fits it. @@ -242,7 +359,7 @@ impl Stream { more |= server.want_data(words)? > self.queued; } if status.starved { - more |= RECEIVE_BYTES.saturating_sub(server.want_room(words)?) < self.held; + more |= server.receive_holds().saturating_sub(server.want_room(words)?) < self.held; } if !more { break true; @@ -275,7 +392,7 @@ impl Streams { /// has, `nodelay` among them written into the region's header. pub(crate) fn accepted(&mut self, stack: &mut Stack, conn: ConnId, options: Options, session: Box) -> StreamId { session.words()[NODELAY].store(u32::from(options.nodelay), Ordering::Relaxed); - let mut stream = Stream { conn, session, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; + let mut stream = Stream { conn, session, shapes: None, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; stream.lent(stack); self.hold(stream) } @@ -302,7 +419,7 @@ impl Node { let options = Options { nodelay: session.words()[NODELAY].load(Ordering::Relaxed) != 0, ..Options::default() }; self.stack.tcp_set_options(now, conn, options); let deadline = timeout.map(|within| now.after(within)); - Ok(self.streams.hold(Stream { conn, session, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) + Ok(self.streams.hold(Stream { conn, session, shapes: None, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) } /// One pass over every stream: netstack calls it once the kernel answered the connections it diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs index 75d983548df..559b6544518 100644 --- a/system/netstack/node/tests/common/client.rs +++ b/system/netstack/node/tests/common/client.rs @@ -8,15 +8,21 @@ use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::sync::atomic::{AtomicU32, Ordering}; -use toyos::net::stream::{Words, LAYOUT, NODELAY, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; -use toyos_net_node::Session; -use toyos_net_tcp::Lent; +use toyos::net::stream::{Words, LAYOUT, NODELAY, RECEIVE_BYTES, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_INITIAL, SEND_RING}; +use toyos_net_node::{Memory, Session}; use toyos::transport::session::{self, Bells, Doorbell, Holder, HungUp, Lock, Mode, Receiving, Refused, Rings, Sending, Took, Waited}; /// The region and its holders' doorbells, which the holders and the node share. pub struct Region { words: Box, bytes: RefCell>, + /// Each commit's range, the client's and the node's. + committed: RefCell>, + /// Whether the client's and the node's commits make memory: one that makes none leaves + /// every page the other's to commit, so a test shows the other's commits alone suffice. + commits: Cell<(bool, bool)>, + /// The node's commits are refused, as the kernel refuses one past the job's memory. + refused: Cell, holders: RefCell>>, /// A holder rang netstack since the test last asked. rang: Cell, @@ -78,33 +84,66 @@ pub struct Client { tx: Held>, } -/// One ring of the region, as [tcp]'s storage. -#[derive(Debug)] -struct Ring { - region: Rc, - base: usize, - len: usize, -} - impl std::fmt::Debug for Region { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("Region") } } -impl Lent for Ring { - fn len(&self) -> usize { - self.len +impl Memory for Region { + fn put(&self, at: usize, data: &[u8]) { + assert!(self.reaches(at, data.len()), "a put of {} bytes at {at} past what was committed", data.len()); + self.bytes.borrow_mut()[at..][..data.len()].copy_from_slice(data); } - fn put(&mut self, at: usize, data: &[u8]) { - assert!(at + data.len() <= self.len, "a put past the ring"); - self.region.bytes.borrow_mut()[self.base + at..][..data.len()].copy_from_slice(data); + fn take(&self, at: usize, out: &mut [u8]) { + assert!(self.reaches(at, out.len()), "a take of {} bytes at {at} past what was committed", out.len()); + out.copy_from_slice(&self.bytes.borrow()[at..][..out.len()]); } - fn take(&self, at: usize, out: &mut [u8]) { - assert!(at + out.len() <= self.len, "a take past the ring"); - out.copy_from_slice(&self.region.bytes.borrow()[self.base + at..][..out.len()]); + fn commit(&self, at: usize, len: usize) -> bool { + if self.refused.get() { + return false; + } + if self.commits.get().1 { + self.committed.borrow_mut().push((at, at + len)); + } + true + } +} + +impl Region { + /// The client's commit of `[at, at + len)`. + fn client_commit(&self, at: usize, len: usize) -> bool { + if self.commits.get().0 { + self.committed.borrow_mut().push((at, at + len)); + } + true + } + + /// From here on the client's commits make no memory. + pub fn client_commits_nothing(&self) { + self.commits.set((false, self.commits.get().1)); + } + + /// From here on the node's commits make no memory. + pub fn netstack_commits_nothing(&self) { + self.commits.set((self.commits.get().0, false)); + } + + /// From here on the node's commits are refused. + pub fn refuse_netstacks_commits(&self) { + self.refused.set(true); + } + + /// Whether `[at, at + len)` lies in what a commit made memory, as a fault would ask. + fn reaches(&self, at: usize, len: usize) -> bool { + self.committed.borrow().iter().any(|&(start, end)| start <= at && at + len <= end) + } + + /// The most of the ring at `ring` committed from its start. + pub fn committed(&self, ring: usize) -> usize { + self.committed.borrow().iter().filter(|&&(start, _)| start == ring).map(|&(_, end)| end - ring).max().unwrap_or(0) } } @@ -116,12 +155,8 @@ impl Session for Lend { &self.0.words } - fn receive(&self) -> Box { - Box::new(Ring { region: self.0.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) - } - - fn send(&self) -> Box { - Box::new(Ring { region: self.0.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + fn memory(&self) -> Rc { + self.0.clone() } fn ring(&self, rings: Rings) { @@ -155,11 +190,14 @@ impl Client { let region = Rc::new(Region { words: Box::new(std::array::from_fn(|_| AtomicU32::new(0))), bytes: RefCell::new(vec![0; REGION_BYTES]), + committed: RefCell::new(vec![(0, RECEIVE_RING + RECEIVE_INITIAL as usize), (SEND_RING, SEND_RING + SEND_INITIAL as usize)]), + commits: Cell::new((true, true)), + refused: Cell::new(false), holders: RefCell::new(Vec::new()), rang: Cell::new(false), dropped: Cell::new(false), }); - let halves = session::client(®ion.words, LAYOUT); + let halves = session::client(®ion.words, LAYOUT, SEND_INITIAL); (Self::holding(®ion, halves), Box::new(Lend(region))) } @@ -190,7 +228,9 @@ impl Client { pub fn read(&self, max: usize) -> Result, Refused> { let mut out = vec![0; max]; let region = &self.region; - let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), |span, at| { + let reach = |cap: u32| region.client_commit(RECEIVE_RING, cap as usize); + let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), reach, |span, at| { + assert!(region.reaches(span.offset, span.len), "a read of {span:?} past what was committed"); out[at..at + span.len].copy_from_slice(®ion.bytes.borrow()[span.offset..][..span.len]); })?; out.truncate(n as usize); @@ -212,7 +252,9 @@ impl Client { /// A write that does not wait: how much of `data` the send ring took. pub fn write(&self, data: &[u8]) -> Result { let region = &self.region; - let n = session::write(®ion.words, &self.tx, &self.ringing, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), |span, at| { + let reach = |cap: u32| region.client_commit(SEND_RING, cap as usize); + let n = session::write(®ion.words, &self.tx, &self.ringing, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), reach, |span, at| { + assert!(region.reaches(span.offset, span.len), "a write of {span:?} past what was committed"); region.bytes.borrow_mut()[span.offset..][..span.len].copy_from_slice(&data[at..at + span.len]); })?; Ok(n as usize) diff --git a/system/netstack/node/tests/common/mod.rs b/system/netstack/node/tests/common/mod.rs index 36f54fad9dd..7d0e7b5f3cd 100644 --- a/system/netstack/node/tests/common/mod.rs +++ b/system/netstack/node/tests/common/mod.rs @@ -360,7 +360,8 @@ impl Wire { tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, }; let now = Instant::from_millis(3_600_000); - let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; + // netstack's own buffers: each a stream's ring at its most. + let config = Config { receive_buffer: toyos::net::stream::RECEIVE_BYTES, send_buffer: toyos::net::stream::SEND_BYTES, secrets }; let mut draws = 0x5a00_0000; let mut node = Node::new(now, config, HostName::new("toyos")).unwrap(); let link = node.add_link(now, IndividualMac::new(MacAddr(mac)).unwrap(), Rank(0), draw(&mut draws)); diff --git a/system/netstack/node/tests/listeners.rs b/system/netstack/node/tests/listeners.rs index 6d7a749606d..7de7e74bb8d 100644 --- a/system/netstack/node/tests/listeners.rs +++ b/system/netstack/node/tests/listeners.rs @@ -373,10 +373,11 @@ impl Net { self.pump(); } - /// The stream's client writes 100,000 bytes and leaves: [tcp] queues them all, its send + /// The stream's client fills its send ring and leaves: [tcp] queues it all, its send /// buffer being the client's ring, and finishes the connection alone. fn departs(&mut self, id: StreamId, client: &Client) { - assert_eq!(client.write(&[7u8; 100_000]), Ok(100_000)); + let ring = toyos::net::stream::SEND_INITIAL as usize; + assert_eq!(client.write(&vec![7u8; ring]), Ok(ring)); self.node.left(self.now, id); self.pump(); assert!(client.dropped()); diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index ad11de887b8..781d805f9f3 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -90,9 +90,13 @@ struct Far { snd_nxt: u32, /// The node's next sequence number the peer has everything before. rcv_nxt: u32, - /// The node's last acknowledgment, and the window it came with. + /// The node's last acknowledgment, and the window it came with, scaled. acked: u32, - window: u16, + window: u32, + /// The peer offers window scaling in its SYN-ACK (RFC 7323 §2.2). + scales: bool, + /// The shift the node's SYN asked for, where both scale. + shift: u8, received: Vec, /// The node's FIN arrived, after everything before it. fin: bool, @@ -101,6 +105,18 @@ struct Far { segments: Vec, } +/// The window scale a SYN the node emitted asks for, or 0 where it asks none. +fn window_shift(frame: &[u8]) -> u8 { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + let Some(TransportSlice::Tcp(tcp)) = &packet.transport else { panic!("a TCP segment") }; + tcp.options_iterator() + .find_map(|option| match option { + Ok(TcpOptionElement::WindowScale(shift)) => Some(shift), + _ => None, + }) + .unwrap_or(0) +} + fn v4(bytes: &[u8]) -> Ipv4Addr { Ipv4Addr::from(<[u8; 4]>::try_from(bytes).expect("four bytes")) } @@ -123,6 +139,8 @@ impl Far { rcv_nxt: 0, acked: 0, window: 0, + scales: false, + shift: 0, received: Vec::new(), fin: false, resets: Vec::new(), @@ -133,7 +151,9 @@ impl Far { /// A segment from the peer in its frame, built by `etherparse`: PSH with text. fn frame(&self, seq: u32, ack: Option, syn: bool, fin: bool, rst: bool, text: &[u8]) -> Vec { let mut step = PacketBuilder::ethernet2(self.mac, MAC).ipv4(self.addr.octets(), A.octets(), 64).tcp(PORT, self.port, seq, 65_535); - if syn { + if syn && self.scales { + step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460), TcpOptionElement::WindowScale(7)]).unwrap(); + } else if syn { step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(); } if let Some(ack) = ack { @@ -174,7 +194,7 @@ impl Far { /// How many bytes the node's window still lets the peer send. fn open(&self) -> usize { - self.acked.wrapping_add(u32::from(self.window)).wrapping_sub(self.snd_nxt) as usize + self.acked.wrapping_add(self.window).wrapping_sub(self.snd_nxt) as usize } /// The outside reading of a frame the node emitted, and the frames it is answered with. @@ -193,6 +213,7 @@ impl Far { self.port = segment.from_port; self.received.clear(); self.fin = false; + self.shift = if self.scales { window_shift(frame) } else { 0 }; } else if segment.from_port != self.port { return Vec::new(); } @@ -221,7 +242,9 @@ impl Far { return Vec::new(); } if let Some(ack) = segment.ack { - (self.acked, self.window) = (ack, segment.window); + // A SYN's window is never scaled (RFC 7323 §2.2). + let shift = if segment.syn { 0 } else { self.shift }; + (self.acked, self.window) = (ack, u32::from(segment.window) << shift); } match self.manner { Manner::Deaf => return Vec::new(), @@ -596,7 +619,7 @@ fn a_deadline_moves_an_established_stream_with_no_opportunity_after_it() { #[test] fn a_write_larger_than_the_window_is_queued_whole_and_arrives_whole() { let (mut net, id, client) = established(); - let sent = text(100_000); + let sent = text(toyos::net::stream::SEND_INITIAL as usize); assert_eq!(client.write(&sent), Ok(sent.len()), "the send ring holds all of it"); net.node.bridge(net.now, [id]); net.pump(); @@ -729,7 +752,8 @@ fn room_released_rings_a_writer_over_a_full_send_ring_on_its_bell() { } net.bridge(); } - assert_eq!(written, toyos::net::stream::SEND_BYTES as usize, "the ring is full"); + // A deaf peer acknowledges nothing, so no byte is released and the ring stays as made. + assert_eq!(written, toyos::net::stream::SEND_INITIAL as usize, "the ring is full"); let bell = client.bell(); // The peer acknowledges every byte it heard of the node's. let heard = u32::try_from(net.far.segments.iter().map(|segment| segment.text.len()).sum::()).unwrap(); @@ -978,6 +1002,68 @@ fn a_tail_past_the_send_rings_room_resets_that_stream() { assert_eq!((client.dropped(), net.node.streams(), net.far.resets.len()), (true, 0, 1)); } +/// The send ring's size word, which only the client writes. +const TX_SIZE: usize = 16 * 16; + +/// A client that claims its send ring grown to its place without committing a page of it, and +/// publishes `len` bytes, past the ring's first size. +fn grown_without_a_commit(client: &Client, len: usize) { + client.region.client_commits_nothing(); + client.word(TX_SIZE).store(toyos::net::stream::SEND_BYTES.trailing_zeros(), Ordering::Release); + client.word(TX_TAIL).store(u32::try_from(len).unwrap(), Ordering::Release); + client.region_ring(); +} + +// netstack commits what a client's size claims before [tcp] reads by it: every read stays inside +// netstack's own commits, and the peer hears the bytes, a fresh commit's zeros, past the first +// size. +#[test] +fn a_send_ring_a_client_grew_without_committing_is_read_through_netstacks_own_commit() { + let (mut net, id, client) = established(); + const LEN: usize = 100_000; + grown_without_a_commit(&client, LEN); + for _ in 0..1_000 { + if net.far.received.len() == LEN { + break; + } + let before = net.far.received.len(); + net.bridge(); + if net.far.received.len() == before { + let at = net.node.next_deadline().expect("a timer, if nothing else"); + net.fire(at); + } + } + assert!(net.far.received == vec![0; LEN], "the peer heard {} bytes", net.far.received.len()); + assert!(net.events().iter().all(|event| !matches!(event, StreamEvent::Violated { id: v, .. } if *v == id))); +} + +// A size netstack cannot make memory, the kernel refusing the commit, is a violation of that +// stream alone, before [tcp] reads a byte by it. +#[test] +fn a_send_ring_size_netstack_cannot_commit_resets_that_stream() { + let (mut net, id, client) = established(); + client.region.refuse_netstacks_commits(); + grown_without_a_commit(&client, 100_000); + net.bridge(); + assert_eq!(net.events(), [StreamEvent::Violated { id, violation: Violation::Size }]); + assert_eq!((client.dropped(), net.node.streams(), net.far.received.len()), (true, 0, 0)); +} + +// The client, as the receive ring's consumer, commits what a size claims before it reads by it: +// a size netstack stored without committing, here written in its place, is read only through +// the client's own commit. +#[test] +fn a_receive_ring_grown_without_netstacks_commit_is_read_through_the_clients_own() { + const RX_TAIL: usize = 16; + const RX_SIZE: usize = 15 * 16; + const LEN: usize = 100_000; + let (_net, _, client) = established(); + client.region.netstack_commits_nothing(); + client.word(RX_SIZE).store(toyos::net::stream::RECEIVE_BYTES.trailing_zeros(), Ordering::Release); + client.word(RX_TAIL).store(u32::try_from(LEN).unwrap(), Ordering::Release); + assert!(inbox(&client) == vec![0; LEN]); +} + // Text published after the client's own shutdown is past the FIN [tcp] queued: a violation. #[test] fn text_published_after_the_shutdown_resets_that_stream() { @@ -1131,3 +1217,73 @@ fn the_last_holder_leaving_is_the_streams_close() { assert!(net.far.fin, "the last holder left"); assert_eq!((first.dropped(), net.node.streams()), (true, 0)); } + +// ---- the rings grow under load ---- + +// A client that writes more than its send ring holds doubles the ring at each write the room +// cannot take whole, to its place, and netstack reads the grown ring by the size the client +// stored: the peer's stream is the client's, byte for byte. Netstack's commits make no memory +// here, so every page either end touches is one the client committed. +#[test] +fn an_upload_larger_than_its_send_ring_grows_it_to_its_place() { + use toyos::net::stream::{SEND_BYTES, SEND_RING}; + const PERIOD: usize = 251; + let byte = |i: usize| (i % PERIOD) as u8; + const WRITE: usize = 2 * SEND_BYTES as usize; + let pattern: Vec = (0..WRITE + PERIOD).map(byte).collect(); + let (mut net, _, client) = established(); + client.region.netstack_commits_nothing(); + let mut offset = 0; + for _ in 0..1_000 { + if client.region.committed(SEND_RING) == SEND_BYTES as usize { + break; + } + match client.write(&pattern[offset % PERIOD..][..WRITE]) { + Ok(n) => offset += n, + Err(Refused::WouldBlock) => {} + Err(other) => panic!("{other:?}"), + } + net.bridge(); + } + assert_eq!(client.region.committed(SEND_RING), SEND_BYTES as usize, "the send ring after {offset} bytes"); + let heard = &net.far.received; + assert!(!heard.is_empty() && heard.iter().enumerate().all(|(i, &b)| b == byte(i)), "the peer's stream is the client's"); +} + +// RFC 7323 §2: a peer that scales windows lets [tcp] want a receive buffer past the receive +// ring's first size as the client keeps reading a round trip's worth; the node commits the +// region below twice the ring and doubles it, and the client reads every byte in place across +// each switch. Each round trip the peer fills the window it was offered over a millisecond, the +// round trip [tcp] measures, and the client reads all of it. The client commits nothing itself, +// so every page either end touches is one the node committed. +#[test] +fn a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte() { + use toyos::net::stream::{RECEIVE_INITIAL, RECEIVE_RING}; + const GROWN: usize = 4 * RECEIVE_INITIAL as usize; + let mut net = Net::new(); + net.far.scales = true; + let (id, client) = net.connect(None); + client.region.client_commits_nothing(); + let local = Endpoint { addr: A, port: Port::new(net.far.port).expect("the SYN's source port") }; + assert_eq!(net.events(), [StreamEvent::Connected { id, local }]); + assert!(net.far.shift > 0, "the node's SYN asked to scale its window"); + let sent = text(16 << 20); + let (mut offset, mut got) = (0, Vec::new()); + for _ in 0..1_000 { + if client.region.committed(RECEIVE_RING) >= GROWN { + break; + } + while let len @ 1.. = net.far.open().min(1_460).min(sent.len() - offset) { + net.now = net.now.after(Duration::from_micros(20)); + let frame = net.far.text(&sent[offset..offset + len]); + offset += len; + net.deliver(&frame); + } + net.now = net.now.after(Duration::from_millis(1)); + got.extend(inbox(&client)); + net.bridge(); + } + assert!(client.region.committed(RECEIVE_RING) >= GROWN, "the receive ring after {offset} bytes: {}", client.region.committed(RECEIVE_RING)); + got.extend(inbox(&client)); + assert!(got.len() > GROWN && got[..] == sent[..got.len()], "the client's stream is the peer's: {} bytes", got.len()); +} diff --git a/system/netstack/shard/tcp/src/conn.rs b/system/netstack/shard/tcp/src/conn.rs index f73b25bc400..2da30e7165a 100644 --- a/system/netstack/shard/tcp/src/conn.rs +++ b/system/netstack/shard/tcp/src/conn.rs @@ -1068,7 +1068,7 @@ impl Sync { if !(self.rx.lendable(rx.len()) && self.tx.buf.lendable(tx.len())) { return false; } - let ring = tx.len(); + let ring = tx.most(); self.rx.lend(rx); self.tx.buf.lend(tx); self.tx.buf.grow(ring); diff --git a/system/netstack/shard/tcp/src/ring.rs b/system/netstack/shard/tcp/src/ring.rs index 59accbd06d6..d834b1dc94e 100644 --- a/system/netstack/shard/tcp/src/ring.rs +++ b/system/netstack/shard/tcp/src/ring.rs @@ -1,21 +1,27 @@ //! A byte ring over storage of its own or a lent one. Its own grows, by doubling, with the //! furthest offset written and never past the capacity, so a connection holds memory for what it -//! holds and not for what it may. A lent one is a fixed length another party holds, bytes are -//! copied into and out of it and never borrowed from it, and a ring is lent at most once. The -//! capacity only rises, and never past a lent ring's length. Offsets count from the oldest byte -//! held. +//! holds and not for what it may. A lent one another party holds and lays out: bytes are copied +//! into and out of it by stream position, counted from the first byte it was lent, never +//! borrowed from it, and a ring is lent at most once. The capacity only rises, and holds no more +//! than a lent ring holds now. Offsets count from the oldest byte held. use alloc::boxed::Box; use alloc::vec::Vec; -/// Storage another party holds: a ring's bytes, reached only by copies. +/// Storage another party holds: a ring's bytes, reached only by copies, at stream positions +/// counted from the first byte it was lent, which wrap with `usize`. #[allow(clippy::len_without_is_empty, reason = "storage is never empty, and nothing asks")] pub trait Lent: core::fmt::Debug { - /// The bytes it holds. + /// The bytes it holds now from the oldest one held on. fn len(&self) -> usize; - /// Copies `data` in at `at`, which with `data`'s length lies within [`Self::len`]. + /// The most [`Self::len`] ever rises to. + fn most(&self) -> usize; + /// Its holder is asked to let it hold `bytes`; [`Self::len`] rises once it does. + fn want(&mut self, bytes: usize); + /// Copies `data` in at position `at`; the ring holds no more than [`Self::len`] from its + /// oldest byte. fn put(&mut self, at: usize, data: &[u8]); - /// Copies `out.len()` bytes out from `at`, which with that length lies within [`Self::len`]. + /// Copies `out.len()` bytes out from position `at`, as [`Self::put`] places them. fn take(&self, at: usize, out: &mut [u8]); } @@ -38,6 +44,7 @@ enum Storage { pub struct Ring { bytes: Storage, capacity: usize, + /// The oldest byte's index in its own storage, or its position in a lent one. head: usize, len: usize, } @@ -51,38 +58,43 @@ impl Ring { self.len } - pub const fn capacity(&self) -> usize { - self.capacity + /// The capacity, held to what a lent ring holds now. + pub fn capacity(&self) -> usize { + match &self.bytes { + Storage::Own(_) => self.capacity, + Storage::Lent(lent) => self.capacity.min(lent.len()), + } } - pub const fn room(&self) -> usize { - self.capacity.saturating_sub(self.len) + pub fn room(&self) -> usize { + self.capacity().saturating_sub(self.len) } - /// Raises the capacity to `capacity`, never past a lent ring's length; a lower one is no - /// change. + /// Raises the capacity to `capacity`, never past the most a lent ring holds, whose holder is + /// asked for it; a lower one is no change. pub fn grow(&mut self, capacity: usize) { - let capacity = match &self.bytes { + let capacity = match &mut self.bytes { Storage::Own(_) => capacity, - Storage::Lent(lent) => capacity.min(lent.len()), + Storage::Lent(lent) => { + let capacity = capacity.min(lent.most()); + lent.want(capacity); + capacity + } }; self.capacity = self.capacity.max(capacity); } - /// How many bytes the storage has. - fn stored(&self) -> usize { + /// The storage index of `offset`: below its own storage's length, or a lent ring's position. + fn at(&self, offset: usize) -> usize { match &self.bytes { - Storage::Own(bytes) => bytes.len(), - Storage::Lent(lent) => lent.len(), + Storage::Own(bytes) => { + let index = self.head.saturating_add(offset); + index.checked_sub(bytes.len()).unwrap_or(index) + } + Storage::Lent(_) => self.head.wrapping_add(offset), } } - /// The storage index of `offset`, which is below the storage's length. - fn at(&self, offset: usize) -> usize { - let index = self.head.saturating_add(offset); - index.checked_sub(self.stored()).unwrap_or(index) - } - /// Storage for every offset below `end`, at most the capacity: each byte stored keeps its /// offset, laid out again from the head. A lent ring has it already. fn reserve(&mut self, end: usize) { @@ -102,14 +114,14 @@ impl Ring { } /// The ring's bytes from here on are `lent`'s, at the offsets they had: everything stored is - /// copied in, the oldest byte held at its start, and the capacity is held to its length. - /// Only where [`Self::lendable`] says so. + /// copied in, the oldest byte held at position 0, and the capacity is held to the most it + /// holds. Only where [`Self::lendable`] says so. pub fn lend(&mut self, mut lent: Box) { let Storage::Own(bytes) = &self.bytes else { return }; let (newer, older) = bytes.split_at(self.head.min(bytes.len())); lent.put(0, older); lent.put(older.len(), newer); - self.capacity = self.capacity.min(lent.len()); + self.capacity = self.capacity.min(lent.most()); self.bytes = Storage::Lent(lent); self.head = 0; } @@ -123,9 +135,7 @@ impl Ring { first.saturating_add(copy(bytes, rest)) } Storage::Lent(lent) => { - let (first, rest) = data.split_at(lent.len().saturating_sub(start).min(data.len())); - lent.put(start, first); - lent.put(0, rest); + lent.put(start, data); data.len() } } @@ -133,7 +143,7 @@ impl Ring { /// Stores `data` at `offset` without counting it held; what lies past the capacity is not stored. pub fn write_at(&mut self, offset: usize, data: &[u8]) -> usize { - let fits = self.capacity.saturating_sub(offset).min(data.len()); + let fits = self.capacity().saturating_sub(offset).min(data.len()); if fits == 0 { return 0; } @@ -171,10 +181,10 @@ impl Ring { let offset = offset.min(self.len); let len = len.min(self.len.saturating_sub(offset)); let start = self.at(offset); - let first = len.min(self.stored().saturating_sub(start)); match &self.bytes { - Storage::Own(bytes) if first == len => bytes.get(start..start.saturating_add(len)).unwrap_or_default(), + Storage::Own(bytes) if len <= bytes.len().saturating_sub(start) => bytes.get(start..start.saturating_add(len)).unwrap_or_default(), Storage::Own(bytes) => { + let first = bytes.len().saturating_sub(start); scratch.clear(); scratch.extend_from_slice(bytes.get(start..start.saturating_add(first)).unwrap_or_default()); scratch.extend_from_slice(bytes.get(..len.saturating_sub(first)).unwrap_or_default()); @@ -183,9 +193,7 @@ impl Ring { Storage::Lent(lent) => { scratch.clear(); scratch.resize(len, 0); - let (to_first, to_rest) = scratch.split_at_mut(first); - lent.take(start, to_first); - lent.take(0, to_rest); + lent.take(start, scratch); scratch } } diff --git a/system/netstack/shard/tcp/src/rx.rs b/system/netstack/shard/tcp/src/rx.rs index cea9cfd41d1..a7f8b9cc8e4 100644 --- a/system/netstack/shard/tcp/src/rx.rs +++ b/system/netstack/shard/tcp/src/rx.rs @@ -461,9 +461,9 @@ impl Rx { /// Received text goes into `lent` from here on, everything held moved there at its offset: /// in-order text and out-of-order ranges, the FIN staying this side's state. The capacity - /// grows no further than its length. Only where [`Self::lendable`] says so. + /// grows no further than the most it holds, and holds what it holds now. Only where [`Self::lendable`] says so. pub fn lend(&mut self, lent: Box) { - self.max = self.max.min(lent.len()); + self.max = self.max.min(lent.most()); self.buf.lend(lent); } diff --git a/system/netstack/shard/tcp/tests/common/net.rs b/system/netstack/shard/tcp/tests/common/net.rs index 119340ea15b..30655689c8f 100644 --- a/system/netstack/shard/tcp/tests/common/net.rs +++ b/system/netstack/shard/tcp/tests/common/net.rs @@ -89,12 +89,26 @@ impl toyos_net_tcp::Lent for Shared { self.0.borrow().len() } + fn most(&self) -> usize { + self.0.borrow().len() + } + + fn want(&mut self, _: usize) {} + + /// A fixed ring: a position is its offset below the length. fn put(&mut self, at: usize, data: &[u8]) { - self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + let mut bytes = self.0.borrow_mut(); + let len = bytes.len(); + for (i, b) in data.iter().enumerate() { + bytes[(at + i) % len] = *b; + } } fn take(&self, at: usize, out: &mut [u8]) { - out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + let bytes = self.0.borrow(); + for (i, b) in out.iter_mut().enumerate() { + *b = bytes[(at + i) % bytes.len()]; + } } } diff --git a/system/netstack/shard/tcp/tests/lent.rs b/system/netstack/shard/tcp/tests/lent.rs index bac42bfc257..aeb806782ca 100644 --- a/system/netstack/shard/tcp/tests/lent.rs +++ b/system/netstack/shard/tcp/tests/lent.rs @@ -42,12 +42,26 @@ impl Lent for Shared { self.0.borrow().len() } + fn most(&self) -> usize { + self.0.borrow().len() + } + + fn want(&mut self, _: usize) {} + + /// A fixed ring: a position is its offset below the length. fn put(&mut self, at: usize, data: &[u8]) { - self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + let mut bytes = self.0.borrow_mut(); + let len = bytes.len(); + for (i, b) in data.iter().enumerate() { + bytes[(at + i) % len] = *b; + } } fn take(&self, at: usize, out: &mut [u8]) { - out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + let bytes = self.0.borrow(); + for (i, b) in out.iter_mut().enumerate() { + *b = bytes[(at + i) % bytes.len()]; + } } } diff --git a/system/netstack/src/lent.rs b/system/netstack/src/lent.rs index d3c35e3fb55..002a2914f22 100644 --- a/system/netstack/src/lent.rs +++ b/system/netstack/src/lent.rs @@ -1,5 +1,5 @@ -//! A region a client lent, as the node's [`Session`]: its header as words, its two rings as -//! [tcp]'s buffers, and the stream's [`Holders`], every one of whose two bells a ring reaches. +//! A region a client lent, as the node's [`Session`]: its header as words, its bytes as the +//! node's [`Memory`], and the stream's [`Holders`], every one of whose two bells a ring reaches. //! //! **A region is judged before it is mapped** ([`SharedMemory::lent`]): a handle that is no //! region, one that is not memory the kernel allocated, and one that is not exactly @@ -19,47 +19,35 @@ use std::collections::BTreeMap; use std::rc::Rc; use toyos::ipc::FrameRx; -use toyos::net::stream::{Words, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; +use toyos::net::stream::{Words, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_INITIAL, SEND_RING}; use toyos::shm::{self, SharedMemory}; use toyos::transport::session::{Holder, Rings}; use toyos::wake::{Bell, Waker}; use toyos::{Connection, OwnedHandle}; use toyos_abi::syscall::SyscallError; -use toyos_net_node::Session; -use toyos_net_tcp::Lent; +use toyos_net_node::{Memory, Session}; -/// The region's mapping. +/// The region's mapping, as the node lays its rings out in it: a page is touched only once +/// [`Memory::commit`] made it memory, since a region is made with only its first rings'. struct Mapping(SharedMemory); -/// One of the region's rings, as [tcp]'s storage. -struct Ring { - mapping: Rc, - base: usize, - len: usize, -} - -impl std::fmt::Debug for Ring { +impl std::fmt::Debug for Mapping { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Ring({:#x}, {})", self.base, self.len) + f.write_str("Mapping") } } -impl Ring { - /// The address of `len` bytes at `at`, which [`Lent`]'s contract keeps inside the ring. +impl Mapping { + /// The address of `len` bytes at `at`, inside the region. fn at(&self, at: usize, len: usize) -> *mut u8 { - assert!(at.checked_add(len).is_some_and(|end| end <= self.len), "{len} bytes at {at} run past a ring of {}", self.len); - // SAFETY: the ring lies inside the mapping, which `mapping` keeps alive, and the - // assertion keeps the bytes inside the ring. - unsafe { self.mapping.0.as_ptr().add(self.base + at) } + assert!(at.checked_add(len).is_some_and(|end| end <= REGION_BYTES), "{len} bytes at {at} run past the region"); + // SAFETY: the assertion keeps the bytes inside the mapping, which `self` keeps alive. + unsafe { self.0.as_ptr().add(at) } } } -impl Lent for Ring { - fn len(&self) -> usize { - self.len - } - - fn put(&mut self, at: usize, data: &[u8]) { +impl Memory for Mapping { + fn put(&self, at: usize, data: &[u8]) { let to = self.at(at, data.len()); // SAFETY: `at` bounded the bytes inside the mapping; `data` is netstack's own memory // and never the region. @@ -71,6 +59,10 @@ impl Lent for Ring { // SAFETY: as in `put`, with the two sides swapped. unsafe { core::ptr::copy_nonoverlapping(from, out.as_mut_ptr(), out.len()) } } + + fn commit(&self, at: usize, len: usize) -> bool { + self.0.commit(at, len).is_ok() + } } /// One holder of a stream, as netstack keeps it. @@ -106,10 +98,14 @@ pub struct Region { } impl Region { - /// The region `handle` names, judged and mapped. Refused, nothing is mapped and the handle - /// is closed. + /// The region `handle` names, judged and mapped, and its header and both rings' first + /// sizes made memory, charged to the client as each commit is. Refused, nothing is mapped + /// and the handle is closed. pub fn judged(handle: toyos::OwnedHandle) -> Result { - SharedMemory::lent(handle, REGION_BYTES) + let mapping = SharedMemory::lent(handle, REGION_BYTES)?; + mapping.commit(0, RECEIVE_RING + RECEIVE_INITIAL as usize).map_err(shm::Lent::Kernel)?; + mapping.commit(SEND_RING, SEND_INITIAL as usize).map_err(shm::Lent::Kernel)?; + Ok(mapping) } /// The session over `mapping`, `first` its first holder, and its holders, which `serve` @@ -128,12 +124,8 @@ impl Session for Region { unsafe { &*(self.mapping.0.as_ptr() as *const Words) } } - fn receive(&self) -> Box { - Box::new(Ring { mapping: self.mapping.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) - } - - fn send(&self) -> Box { - Box::new(Ring { mapping: self.mapping.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + fn memory(&self) -> Rc { + self.mapping.clone() } /// A ring into a bell its holder no longer holds is dropped: that diff --git a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs index 075d974a3a0..dfc6a14e667 100644 --- a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs +++ b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs @@ -31,7 +31,7 @@ use std::io::{BufRead, BufReader, Write}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicI32, Ordering}; -use toyos::net::stream::{Server, Words, LAYOUT, RECEIVE_RING, REGION_BYTES}; +use toyos::net::stream::{Server, Words, LAYOUT, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES}; use toyos::wake::{self, Hungup}; use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; use toyos::shm::SharedMemory; @@ -135,7 +135,7 @@ fn test(role: &str) { let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); // SAFETY: the mapping is `REGION_BYTES`, 2 MiB aligned, and its header is reached only as atomics. let words = unsafe { &*(region.as_ptr() as *const Words) }; - let mut server = Server::new(words, LAYOUT); + let mut server = Server::new(words, LAYOUT, RECEIVE_INITIAL); let (receive, receive_bell) = wake::pair().expect("the receive bell"); let (_send, send_bell) = wake::pair().expect("the send bell"); let (rung_waker, rung) = wake::pair().expect("the rung bell"); diff --git a/tests/guest/rust-tests/src/bin/shm_commit.rs b/tests/guest/rust-tests/src/bin/shm_commit.rs new file mode 100644 index 00000000000..f671c4e2e84 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/shm_commit.rs @@ -0,0 +1,148 @@ +//! A region made with only its first page committed is memory there and +//! nowhere else until `shm_commit` makes more of it: a touch of a page past +//! it ends the process, a syscall's output into one is `BadAddress`, a spawn +//! from a region nothing committed is refused and the machine goes on, a +//! commit makes the page zeroed memory every mapping sees, a second commit of +//! it changes nothing, and the job's account rises by each page a commit +//! makes and falls back when the region goes. +//! +//! It needs the guest: a fault that never allocates, and a commit that a +//! mapping made before it sees, are the kernel's page tables and frames. + +use std::process::{Command, Stdio}; + +use toyos_abi::syscall::{self, SpawnArgs, SyscallError, VmProt, PAGE_SIZE}; + +const ME: &str = "/system/bin/test_rs_shm_commit"; +const PAGES: usize = 4; + +fn main() { + if std::env::args().nth(1).as_deref() == Some("touch") { + return touch(); + } + let own = toyos::job::own().expect("a handle to this job"); + let account = || own.account().expect("this job's account").committed; + let before = account(); + let shm = syscall::shm_reserve(PAGES * PAGE_SIZE, PAGE_SIZE).expect("a region of four pages, one committed"); + assert_eq!(account() - before, PAGE_SIZE as u64, "a region made with one page committed counts one"); + // SAFETY: a fresh region, mapped whole, which nothing else names. + let at = unsafe { syscall::shm_map(shm, VmProt::ReadWrite) }.expect("map the region"); + // SAFETY: the first page is committed. + unsafe { + assert_eq!(at.read_volatile(), 0, "a committed page is zeroed"); + at.write_volatile(0x5A); + } + println!(" PASS: a region made with one page committed is memory there, and counted"); + + dies(); + output_is_refused(at); + a_spawn_from_nothing_committed_is_refused(); + + let held = account(); + syscall::shm_commit(shm, PAGE_SIZE + 100, 2 * PAGE_SIZE).expect("commit what pages 1 to 3 are touched by"); + assert_eq!(account() - held, (3 * PAGE_SIZE) as u64, "every page the range touched is counted"); + // SAFETY: pages 1 to 3 are committed now, and the mapping was made before. + unsafe { + for page in 1..PAGES { + let byte = at.add(page * PAGE_SIZE + 7); + assert_eq!(byte.read_volatile(), 0, "page {page} is zeroed memory"); + byte.write_volatile(page as u8); + } + } + let held = account(); + syscall::shm_commit(shm, 0, PAGES * PAGE_SIZE).expect("a commit of what is committed"); + assert_eq!(account(), held, "a second commit counts nothing more"); + // SAFETY: as above. + unsafe { + assert_eq!(at.read_volatile(), 0x5A, "a second commit kept page 0"); + assert_eq!(at.add(3 * PAGE_SIZE + 7).read_volatile(), 3, "a second commit kept page 3"); + } + println!(" PASS: a commit makes zeroed memory a mapping made before sees, counted once, and a second keeps it"); + + assert_eq!(syscall::shm_commit(shm, 0, 0), Err(SyscallError::InvalidArgument), "no bytes"); + assert_eq!(syscall::shm_commit(shm, PAGES * PAGE_SIZE, 1), Err(SyscallError::InvalidArgument), "past the region"); + assert_eq!(syscall::shm_reserve(PAGE_SIZE, 2 * PAGE_SIZE), Err(SyscallError::InvalidArgument), "more committed than made"); + println!(" PASS: a commit of no bytes or past the region, and a reservation committing more than it makes, are refused"); + + let held = account(); + // SAFETY: the mapping is this test's, and nothing reads it after. + unsafe { syscall::vm_release(at, PAGES * PAGE_SIZE) }.expect("unmap the region"); + // SAFETY: this test's own handle, used no more. + unsafe { syscall::close(shm) }; + assert_eq!(held - account(), (PAGES * PAGE_SIZE) as u64, "the region's pages went back with it"); + println!(" PASS: the region's pages leave the account with it"); + println!("shm_commit: committed pages are memory, and only they"); +} + +/// A syscall's output into a page not committed is refused, not dropped: +/// a pipe's bytes read into page 1, and nothing taken off the pipe. +fn output_is_refused(at: *mut u8) { + let ends = syscall::pipe().expect("a pipe"); + assert_eq!(syscall::write(ends.write, b"held"), Ok(4)); + // SAFETY: page 1 of the mapping, never touched: the kernel judges it. + let page = unsafe { core::slice::from_raw_parts_mut(at.add(PAGE_SIZE), 4) }; + assert_eq!(syscall::read(ends.read, page), Err(SyscallError::BadAddress), "a read into a page not committed"); + let mut out = [0u8; 4]; + assert_eq!(syscall::read(ends.read, &mut out), Ok(4), "the refused read took nothing"); + assert_eq!(&out, b"held"); + // SAFETY: this test's own pipe, used no more. + unsafe { + syscall::close(ends.read); + syscall::close(ends.write); + } + println!(" PASS: a syscall's output into a page not committed is BadAddress, and nothing is lost"); +} + +/// A spawn whose image is a region nothing committed: the loader reads its +/// pages as zeros, refuses the image, and the machine goes on. +fn a_spawn_from_nothing_committed_is_refused() { + let image = syscall::shm_reserve(2 * PAGE_SIZE, 0).expect("a region of two pages, none committed"); + let argv = format!("{ME}\0"); + let cwd = "/"; + // SAFETY: every pointer names a live local for the whole call. + let spawned = unsafe { + syscall::spawn(&SpawnArgs { + path_ptr: ME.as_ptr() as u64, + path_len: ME.len() as u64, + argv_ptr: argv.as_ptr() as u64, + argv_len: argv.len() as u64, + env_ptr: 0, + env_len: 0, + endow_ptr: 0, + endow_count: 0, + labels_ptr: 0, + labels_len: 0, + cwd_ptr: cwd.as_ptr() as u64, + cwd_len: cwd.len() as u64, + image: image.0 as u64, + image_len: (2 * PAGE_SIZE) as u64, + place: u64::from(toyos_abi::HANDLE_INVALID.0), + pager: u64::from(toyos_abi::HANDLE_INVALID.0), + }) + }; + assert!(spawned.is_err(), "an image of nothing committed was spawned"); + // SAFETY: this test's own handle, used no more. + unsafe { syscall::close(image) }; + println!(" PASS: a spawn from a region nothing committed is refused: {spawned:?}"); +} + +/// A child touches page 1 of a region committed only on page 0, and dies of it. +fn dies() { + let child = Command::new(ME).arg("touch").stdout(Stdio::piped()).spawn().expect("spawn the toucher"); + let out = child.wait_with_output().expect("wait for the toucher"); + let said = String::from_utf8_lossy(&out.stdout); + assert!(said.contains("armed"), "the child never reached the touch, so it proved nothing:\n{said}"); + assert!(!said.contains("SURVIVED"), "a touch past what was committed was served:\n{said}"); + assert!(!out.status.success(), "a touch past what was committed did not end the process ({:?})", out.status.code()); + println!(" PASS: a touch of a page not committed ends the process"); +} + +fn touch() { + let shm = syscall::shm_reserve(2 * PAGE_SIZE, PAGE_SIZE).expect("a region of two pages, one committed"); + // SAFETY: a fresh region, mapped whole; page 1 is past what is committed, which is the test. + let at = unsafe { syscall::shm_map(shm, VmProt::ReadWrite) }.expect("map the region"); + println!("armed"); + // SAFETY: as above. + let byte = unsafe { at.add(PAGE_SIZE).read_volatile() }; + println!("SURVIVED with {byte}"); +} diff --git a/tests/guest/rust-tests/src/bin/stream_grows.rs b/tests/guest/rust-tests/src/bin/stream_grows.rs new file mode 100644 index 00000000000..3269730cf0e --- /dev/null +++ b/tests/guest/rust-tests/src/bin/stream_grows.rs @@ -0,0 +1,84 @@ +//! A stream carries 4 MiB each way, every byte where it was written, while +//! this end's send ring grows, at the first write its room cannot take, here +//! the third: netstack's mapping of the region read past the ring's first +//! size, over pages the kernel committed after both ends mapped it. A page +//! neither end committed ends this process or netstack; a byte laid out by +//! the wrong size arrives out of place. Which end's commit suffices, and the +//! receive ring's growth, which QEMU's user network's round trips seldom +//! make [tcp] want, are the node's host tests'. +//! +//! It needs the guest: the commits and the mappings are the kernel's, and the +//! region's adapter is netstack's own. +//! ToyOS has no loopback, so the harness is the peer, as for `socket_export`: +//! this job's argument is the port of its datagram socket, which QEMU's user +//! network reaches as `10.0.2.2`. The job listens on the port the boot +//! forwards, says it is ready in a datagram, accepts the harness's +//! connection, and both ends send [`BYTES`] and check what they receive. +//! `netstack_holders_exported_to_children`'s boot runs it, on +//! tests/images/netcase.toml. + +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::thread; + +/// QEMU's user network's address of its host. +const HOST: Ipv4Addr = Ipv4Addr::new(10, 0, 2, 2); +/// The port the boot forwards to (`tests/toyos.rs`'s `SOCKET_EXPORT_PORT`). +const LISTENING: u16 = 7070; +/// What each end sends, as `tests/toyos.rs`'s `STREAM_GROWS_BYTES`. +const BYTES: usize = 4 << 20; +/// Write sizes in turn: a byte, a part of a ring, one ring and more than one, +/// so a write meets a partial room, a full ring and a growth pending. +const WRITES: [usize; 4] = [1, 1000, 64 << 10, 300 << 10]; + +/// The byte at stream position `at`, which repeats at no power of two a ring +/// could be: `tests/toyos.rs`'s `stream_grows_byte`. +fn byte(at: usize) -> u8 { + ((at as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +fn send(mut stream: &TcpStream) { + let all: Vec = (0..BYTES).map(byte).collect(); + let mut at = 0; + for size in WRITES.iter().cycle() { + if at == BYTES { + break; + } + let end = (at + size).min(BYTES); + stream.write_all(&all[at..end]).expect("a write the stream takes"); + at = end; + } + stream.shutdown(Shutdown::Write).expect("this end's FIN"); +} + +fn receive(mut stream: &TcpStream) { + let mut buf = vec![0u8; 200 << 10]; + let mut at = 0; + loop { + // Reads of a varying size: some take less than is ready. + let want = [7usize, 4096, 200 << 10][at % 3]; + let n = stream.read(&mut buf[..want]).expect("a read the stream answers"); + if n == 0 { + break; + } + if let Some(wrong) = (0..n).find(|&i| buf[i] != byte(at + i)) { + panic!("the byte at {} is {:#04x}, sent {:#04x}", at + wrong, buf[wrong], byte(at + wrong)); + } + at += n; + } + assert_eq!(at, BYTES, "the stream ended after {at} bytes"); +} + +fn main() { + let port: u16 = std::env::args().nth(1).expect("the harness's datagram port").parse().expect("a port"); + let listener = TcpListener::bind((Ipv4Addr::UNSPECIFIED, LISTENING)).expect("a listener through netstack"); + let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).expect("a datagram socket through netstack"); + socket.send_to(b"grows ready", SocketAddr::from((HOST, port))).expect("the ready datagram"); + let (stream, _) = listener.accept().expect("the harness's connection"); + thread::scope(|scope| { + scope.spawn(|| send(&stream)); + receive(&stream); + }); + println!(" PASS: 4 MiB each way arrived in order through the send ring's growth"); + println!("stream_grows: a stream's send ring grows under load and keeps every byte in place"); +} diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs index a1b2e46754b..fcadfec7e45 100644 --- a/tests/guest/rust-tests/src/bin/stream_handles.rs +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -52,7 +52,7 @@ use std::process::{Child, ChildStdout, Command, Stdio}; use std::sync::atomic::Ordering; use toyos::ipc::{FrameRx, IpcError, RxStep}; -use toyos::net::stream::{Adopt, Server, Words, ADOPT, EXPORT, EXPORTED, LAYOUT, RECEIVE_RING, REGION_BYTES, SEND_RING}; +use toyos::net::stream::{Adopt, Server, Words, ADOPT, EXPORT, EXPORTED, LAYOUT, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_RING}; use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; use toyos::poller::{Poller, READABLE}; use toyos::shm::SharedMemory; @@ -533,7 +533,7 @@ fn run_answering(role: &str, peer: &[u8], local: [u8; 4]) -> (Child, Netstack) { let _: TcpConnectRequest = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the connect's request"); let [region] = rx.take_handles::<1>().expect("the stream's region"); let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); - let mut server = Server::new(words_of(®ion), LAYOUT); + let mut server = Server::new(words_of(®ion), LAYOUT, RECEIVE_INITIAL); region.bytes().write(RECEIVE_RING, peer); let _ = server.publish(words_of(®ion), peer.len() as u32).expect("the client keeps the protocol"); let (handed, (receive, send, rung)) = bells(); diff --git a/tests/guest/rust-tests/src/bin/stream_region_refused.rs b/tests/guest/rust-tests/src/bin/stream_region_refused.rs new file mode 100644 index 00000000000..d02bb57b26b --- /dev/null +++ b/tests/guest/rust-tests/src/bin/stream_region_refused.rs @@ -0,0 +1,34 @@ +//! A connect whose region has none of its pages committed does not fault +//! netstack, which goes on serving: the header and both rings' first sizes +//! are made memory where netstack judges the region, charged to the client, +//! before anything reads them, or the region is refused there. +//! +//! It needs the guest's real netstack: the judge is netstack's own, and the +//! fault it prevents would end netstack, which only the kernel delivers. +//! `netstack_holders_exported_to_children`'s boot runs it, on +//! tests/images/netcase.toml, whose netstack has its lease. + +use std::net::{Ipv4Addr, SocketAddrV4, TcpListener}; + +use toyos::net::stream::{BELLS, REGION_BYTES}; +use toyos::net::{MsgType, NetstackConn, TcpConnectRequest, TcpConnectResponse}; +use toyos::OwnedHandle; +use toyos_abi::syscall; + +fn main() { + let region = syscall::shm_reserve(REGION_BYTES, 0).expect("a region with nothing committed"); + // SAFETY: a handle the kernel just made, which nothing else answers for. + let region = unsafe { OwnedHandle::from_raw(region) }; + let request = TcpConnectRequest { addr: [192, 0, 2, 9], port: 9, _pad: 0, timeout_ms: 1_000 }; + let answer = NetstackConn::connect() + .and_then(|conn| conn.request_with_handles([region], MsgType::TcpConnect, &request)) + .and_then(|pending| pending.answer_with_handles::()); + match answer { + Err(refused) => println!(" PASS: a connect with no page of its region committed was refused: {refused:?}"), + Ok(_) => println!(" PASS: a connect with no page of its region committed had its first pages made memory, and was answered"), + } + let listener = TcpListener::bind(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)).expect("netstack serves on after the connect"); + drop(listener); + println!(" PASS: netstack serves on"); + println!("stream_region_refused: a client's uncommitted region never faults netstack"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 4f33f1c52b2..208f0580cea 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -254,6 +254,15 @@ const RUST_SKIP: &[&str] = &[ // `netstack_holders_exported_to_children`'s boot sets up. "stream_export", "socket_export", + // A memory object's commit and its faults: `shm_commit_alone` runs it on + // QEMU, which establishes them; nothing in it is hardware. + "shm_commit", + // A connect whose region nothing committed, to real netstack, which goes + // on serving: `netstack_holders_exported_to_children`'s boot runs it. + "stream_region_refused", + // A stream's bytes through its send ring's growth, the harness its peer: + // `netstack_holders_exported_to_children`'s boot runs it. + "stream_grows", // A stream's holders across processes, as their own netstack: it spawns a // child that faults, which a shared boot's other members must not watch, // and `stream_holders` runs it. @@ -757,6 +766,13 @@ const MACHINE_TESTS: &[&str] = &[ // refused meanwhile. QEMU establishes the refusals and that the machine // goes on; nothing in it is timing. "kernel_objects_refused_alone", + // A region committed in parts: a touch past the commit ends the process, a + // syscall's output into it is `BadAddress`, a spawn from nothing committed + // is refused, a commit is zeroed memory an earlier mapping sees, counted + // to the job once and given back with the region. These are the kernel's + // page tables, frames and accounts, which no host model has; order and + // contents only, on the shared boot's q35 guest in a boot of its own. + "shm_commit_alone", // With one ordinary job holding every frame it may, the supervisor answers // a stop's request, a key reaches the owner of input through ps2server, a // system holder kills the greedy job, and a service the exhaustion ended is @@ -4853,6 +4869,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "std_threads_count_the_cpus_and_give_back_ended_threads" => std_threads_count_the_cpus_and_give_back_ended_threads(test_config), "affinity_pins_hold_on_every_cpu" => affinity_pins_hold_on_every_cpu(test_config), "kernel_objects_refused_alone" => kernel_objects_refused_alone(test_config), + "shm_commit_alone" => shm_commit_alone(test_config), "memory_floor_holds_the_controls" => memory_floor_holds_the_controls(), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), "crypto_instrument_prints_its_table" => crypto_instrument_prints_its_table(test_config), @@ -5685,6 +5702,24 @@ fn kernel_objects_refused_alone(test_config: &Path) -> Result<(), String> { Ok(()) } +fn shm_commit_alone(test_config: &Path) -> Result<(), String> { + const JOB: &str = "shm_commit"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], BootOptions::default()); + serial::Serial::boot(&qemu).must_be_clean()?; + let result = qemu.run_test("test_rs_shm_commit", Duration::from_secs(120)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + for line in result.stdout.lines().filter(|l| l.contains("PASS: ")) { + eprintln!(" [commit] {}", line.trim()); + } + Ok(()) +} + /// `memory_floor --input` alone in `tests/images/floorcase.toml`'s boot, a key /// typed once every frame is held: the job holds what the controls did, and the /// supervisor said it was refused the starved service's restart, then started @@ -5781,9 +5816,11 @@ const SOCKET_EXPORT_PORT: u16 = 7070; /// `tests/images/netcase.toml`, the harness every socket's peer: a stream's /// child's line, then its parent's, then the FIN once both holders are gone; /// then a listener's and a datagram socket's child's datagram, its accept and -/// its line, and the parent's after it. Once over virtio, and once over QEMU's -/// `e1000e`, which the image's rows name no machine for: the I219 driver's -/// row matches it, and its link is the one netstack opens. +/// its line, and the parent's after it; then a connect whose region nothing +/// committed, and a stream to the harness through its send ring's growth. Once over +/// virtio, and once over QEMU's `e1000e`, which the image's rows name no +/// machine for: the I219 driver's row matches it, and its link is the one +/// netstack opens. fn netstack_holders_exported_to_children() -> Result<(), String> { for (profile, driver) in [(qemu::Profile::Headless, "virtio-net"), (qemu::Profile::HeadlessE1000e, "82574")] { holders_exported_over(profile, driver)?; @@ -5797,7 +5834,7 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str const LEASED: &str = "netstack: ready, at most"; let tests = compile::repo_root().join("tests/guest/rust-tests"); let bins: Vec<(String, Vec)> = - ["stream_export", "socket_export"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); + ["stream_export", "socket_export", "stream_region_refused", "stream_grows"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); // The host's end of the forward: a port found free and given up, which // QEMU binds as it starts and refuses to start without. Another process // can take it between the two, which fails the boot by name. @@ -5821,7 +5858,12 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str return Err(format!("{driver}: netstack leased without saying {opened:?}\n{said}")); } stream_exported_to_a_child(&mut qemu)?; - sockets_exported_to_a_child(&mut qemu, forwarded) + sockets_exported_to_a_child(&mut qemu, forwarded)?; + let result = qemu.run_test("test_rs_stream_region_refused", Duration::from_secs(60)); + if result.error.is_some() || result.exit_code != Some(0) { + return Err(format!("{driver}: {:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)); + } + stream_grows_with_the_harness(&mut qemu, forwarded) } /// `test_rs_stream_export`: the child's line, then the parent's, then the FIN @@ -5932,6 +5974,56 @@ fn sockets_exported_to_a_child(qemu: &mut QemuInstance, forwarded: u16) -> Resul heard.join().map_err(|_| "the peer's thread panicked".to_string())? } +/// What `test_rs_stream_grows` and the harness each send. +const STREAM_GROWS_BYTES: usize = 4 << 20; + +/// The byte at stream position `at` of either way: `stream_grows.rs`'s `byte`. +fn stream_grows_byte(at: usize) -> u8 { + ((at as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +/// `test_rs_stream_grows`, the harness its peer through the forwarded port: +/// once the job's datagram says it listens, the harness connects, sends +/// [`STREAM_GROWS_BYTES`] and its FIN, and reads as many back, each in place. +fn stream_grows_with_the_harness(qemu: &mut QemuInstance, forwarded: u16) -> Result<(), String> { + use std::io::{Read, Write}; + /// A liveness ceiling on each of the peer's waits, never a timing. + const CEILING: Duration = Duration::from_secs(120); + let datagrams = std::net::UdpSocket::bind("127.0.0.1:0").map_err(|e| format!("no loopback datagram socket: {e}"))?; + datagrams.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let port = datagrams.local_addr().map_err(|e| format!("the datagram socket's port: {e}"))?.port(); + let peer = std::thread::spawn(move || -> Result<(), String> { + let mut ready = [0u8; 64]; + let (n, _) = datagrams.recv_from(&mut ready).map_err(|e| format!("the job's ready datagram: {e}"))?; + if &ready[..n] != b"grows ready" { + return Err(format!("the peer heard {:?}", String::from_utf8_lossy(&ready[..n]))); + } + let conn = std::net::TcpStream::connect(("127.0.0.1", forwarded)).map_err(|e| format!("the connect: {e}"))?; + conn.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let mut sending = conn.try_clone().map_err(|e| format!("a second handle to the connection: {e}"))?; + let sent = std::thread::spawn(move || -> Result<(), String> { + let all: Vec = (0..STREAM_GROWS_BYTES).map(stream_grows_byte).collect(); + sending.write_all(&all).map_err(|e| format!("the peer's send: {e}"))?; + sending.shutdown(std::net::Shutdown::Write).map_err(|e| format!("the peer's FIN: {e}")) + }); + let mut got = Vec::with_capacity(STREAM_GROWS_BYTES); + (&conn).read_to_end(&mut got).map_err(|e| format!("the job's bytes, until its FIN: {e} after {}", got.len()))?; + sent.join().map_err(|_| "the peer's sender panicked".to_string())??; + if got.len() != STREAM_GROWS_BYTES { + return Err(format!("the job sent {} bytes, not {STREAM_GROWS_BYTES}", got.len())); + } + match got.iter().enumerate().find(|&(at, &b)| b != stream_grows_byte(at)) { + Some((at, &b)) => Err(format!("the job's byte at {at} is {b:#04x}, not {:#04x}", stream_grows_byte(at))), + None => Ok(()), + } + }); + let result = qemu.run_test(&format!("test_rs_stream_grows {port}"), Duration::from_secs(300)); + if result.error.is_some() || result.exit_code != Some(0) { + return Err(format!("{:?}, ended {:?}; {}\nthe job said:\n{}", result.error, result.exit_code, peer_verdict(peer), result.stdout)); + } + peer.join().map_err(|_| "the peer's thread panicked".to_string())? +} + /// What the peer's thread had come to when its job failed: a guest that waits /// for a step the peer could not take is no stall of the guest's. fn peer_verdict(heard: std::thread::JoinHandle>) -> String {