From 3aa7c070d2074eed5c0330f0732e17b1d21c432c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:08:22 +0100 Subject: [PATCH 01/16] stream region: a connect commits 132 KiB, and each ring doubles under load A connect made its whole 8 MiB region memory, every frame zeroed before the SYN left. It now commits the header and 64 KiB of each ring, and a ring's producer doubles it, up to its 4 MiB place, as it needs more. Kernel: a memory object's pages are committed at an explicit call, never at a fault. SYS_SHM_CREATE with SHM_PART commits only a first part; SYS_SHM_COMMIT (any holder with MAP) promises and zeroes the rest at the call, charged to the object's own job as its creation is. A fault on a page not committed is refused like an access outside every range, and a copy reads it as zeros. fstat's new `committed` reports what is. Transport: cursors are stream positions and a ring's capacity a power of two, so a byte's offset is its position below the capacity on every lap. A producer grows its ring without waiting for it to empty: the size word names the new capacity and a switch one old ring past the head it last read, past every byte it may have placed; bytes before the switch keep the old mask, bytes after use the new, and the room stays counted in the old capacity until the consumer passes the switch, so a new byte can only land on an unread one a whole old ring behind it, which the old room forbids. A consumer reads the size after the tail and judges it before believing it. The client grows its send ring when a write finds less room than it brings; netstack grows the receive ring when [tcp] asks for more buffer than it holds, each after committing the memory. Each consumer commits what a peer's size claims before it touches it, so a lying peer spends its own region's memory and cannot fault the reader. [tcp] addresses a lent ring by stream position and the node lays it out by the judged size. Co-Authored-By: Claude Opus 5.5 --- kernel/src/inbox/mod.rs | 2 +- kernel/src/mm/pmm.rs | 16 +- kernel/src/mm/space.rs | 6 +- kernel/src/object/ops.rs | 7 +- kernel/src/object/shm.rs | 130 +++- kernel/src/syscall/dispatch.rs | 5 +- kernel/src/syscall/ipc.rs | 22 +- kernel/src/vfs.rs | 2 +- sdk/abi/src/syscall.rs | 33 +- sdk/toyos/Cargo.toml | 6 + sdk/toyos/src/net/stream.rs | 44 +- sdk/toyos/src/shm.rs | 12 + sdk/toyos/src/transport/mod.rs | 3 + sdk/toyos/src/transport/session.rs | 178 +++-- sdk/toyos/src/transport/stream.rs | 610 ++++++++++++------ system/netstack/node/src/lib.rs | 2 +- system/netstack/node/src/streams.rs | 151 ++++- system/netstack/node/tests/common/client.rs | 65 +- system/netstack/node/tests/listeners.rs | 5 +- system/netstack/node/tests/streams.rs | 5 +- system/netstack/shard/tcp/src/conn.rs | 2 +- system/netstack/shard/tcp/src/ring.rs | 84 +-- system/netstack/shard/tcp/src/rx.rs | 4 +- system/netstack/shard/tcp/tests/common/net.rs | 18 +- system/netstack/shard/tcp/tests/lent.rs | 18 +- system/netstack/src/lent.rs | 54 +- 26 files changed, 1062 insertions(+), 422 deletions(-) diff --git a/kernel/src/inbox/mod.rs b/kernel/src/inbox/mod.rs index 6b7e1474b52..ec60c0c3add 100644 --- a/kernel/src/inbox/mod.rs +++ b/kernel/src/inbox/mod.rs @@ -223,7 +223,7 @@ struct RingState { /// One atomic word of one ring header; never `&RingHeader` — see the block above. fn ring_word(ring: &SharedMemObject, ring_off: u64, field_off: usize) -> &core::sync::atomic::AtomicU32 { let at = ring_off + field_off as u64; - let word = DirectMap::from_phys(ring.page(at) + at % crate::mm::PAGE_SIZE); + let word = DirectMap::from_phys(ring.page(at).expect("inbox: its ring is committed whole") + at % crate::mm::PAGE_SIZE); // SAFETY: the offset is in-bounds and 4-aligned, so inside one page of the // object, which outlives the borrow; `AtomicU32` is sound over memory the // process also writes. diff --git a/kernel/src/mm/pmm.rs b/kernel/src/mm/pmm.rs index d32aa3e0921..c5cc679dcbb 100644 --- a/kernel/src/mm/pmm.rs +++ b/kernel/src/mm/pmm.rs @@ -136,21 +136,9 @@ pub fn release_floor(job: &Job) { /// # Safety /// Each is a frame [`Frame::into_raw`] gave up, that no leaf on any CPU maps /// any more and nothing else frees. -pub unsafe fn free_drawn(job: &Job, frames: &[u64]) { +pub unsafe fn free_drawn(job: &Job, frames: impl IntoIterator) { let pins = pins(); - with(|f| f.free_drawn(job.memory(), frames.iter().copied(), &pins)); -} - -/// Give back every frame of `frames`, drawn for pages `job` committed, under -/// one hold of the allocator's lock. -pub fn free_all(job: &Job, frames: alloc::boxed::Box<[Frame]>) { - let pins = pins(); - with(|f| f.free_drawn(job.memory(), frames.iter().map(|frame| frame.phys), &pins)); - // Given up only once the allocator's lock is let go: the list goes back - // to the heap, which may give a page back through it. - for frame in frames.into_vec() { - let _ = frame.into_raw(); - } + with(|f| f.free_drawn(job.memory(), frames, &pins)); } /// The pin counters, published once by [`init`]: where they are, how many, diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 519b10f4cf6..b229a5286fe 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -774,7 +774,7 @@ impl AddressSpace { crate::sched::futex::revoke_frames(frames); // SAFETY: frames owned leaves gave up, which no leaf maps on any // CPU now and no waiter is keyed on. - unsafe { pmm::free_drawn(&self.job, frames) }; + unsafe { pmm::free_drawn(&self.job, frames.iter().copied()) }; freed += frames.len() as u64; }); let inventory = &self.inventory; @@ -822,7 +822,7 @@ impl AddressSpace { Leaf::Empty => match &entry.kind { Kind::Private { committed: false } => unreachable!("space: a reservation at {page:#x} allows an access"), Kind::Object { object, offset, .. } => { - let phys = object.page(offset + (page - start)); + let phys = object.page(offset + (page - start))?; leaves.set(page, Leaf::Mapped(Mapping { phys, prot, owned: false, cache: object.cache() })); phys } @@ -956,7 +956,7 @@ impl Drop for AddressSpace { let mut freed = 0; self.tables.take_owned(FLOOR, TOP, |frames| { // SAFETY: the space's own frames, which nothing maps once it drops. - unsafe { pmm::free_drawn(&self.job, frames) }; + unsafe { pmm::free_drawn(&self.job, frames.iter().copied()) }; freed += frames.len() as u64; }); self.usage.freed(freed); diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 5198dbffdb9..1e74c802f1c 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -645,13 +645,16 @@ toyos_abi::user_safe! { pub mtime: u64, pub volume: u64, pub node: u64, + pub committed: u64, } } +const _: () = assert!(core::mem::size_of::() == core::mem::size_of::()); + /// What kind of thing this is, how big, and which file: a file's node is its /// `FileId`, one per file for as long as it exists (`vfs::FileSystem::open_file`). pub fn fstat(object: &KObjectRef) -> Stat { - let plain = |t: FileType| Stat { file_type: t as u64, size: 0, mtime: 0, volume: 0, node: 0 }; + let plain = |t: FileType| Stat { file_type: t as u64, size: 0, mtime: 0, volume: 0, node: 0, committed: 0 }; match object { KObjectRef::File(f) => f.with(|state| Stat { file_type: FileType::File as u64, @@ -659,6 +662,7 @@ pub fn fstat(object: &KObjectRef) -> Stat { mtime: state.mtime, volume: state.volume, node: state.file_id, + committed: 0, }), KObjectRef::PipeRead(r) => { plain(if r.is_tty() { FileType::Tty } else { FileType::Pipe }) @@ -683,6 +687,7 @@ pub fn fstat(object: &KObjectRef) -> Stat { mtime: 0, volume: 0, node: 0, + committed: m.committed(), }, KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index c501dbcd187..dc33eee4802 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -10,12 +10,18 @@ //! closing its last handle is how a device's window is taken back from every //! process that mapped it. //! -//! Frames are the object's from its creation to its end, so a fault on a -//! mapping of one never allocates. +//! **A fault never allocates.** A region's pages are committed, each a +//! frame of its own, zeroed, counted to the object's job, at the call that +//! commits them: `SYS_SHM_CREATE`, all or a first part of them, then +//! `SYS_SHM_COMMIT`, which any holder that may map it calls. A committed +//! page is the object's to its end. A fault on a page not committed is +//! refused as an access outside every range is, and a copy reads it as +//! zeros. use alloc::boxed::Box; use alloc::sync::{Arc, Weak}; use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, Ordering}; use kernel::sched::job::Job; @@ -23,7 +29,7 @@ use toyos_abi::syscall::SyscallError; use crate::mm::charge::{self, Charge, Exhausted}; use crate::mm::policy::CachePolicy; -use crate::mm::pmm::{self, Frame, Run}; +use crate::mm::pmm::{self, Run}; use crate::mm::space::Space; use crate::mm::{PAGE_2M, PAGE_SIZE}; use crate::{DirectMap, UserAddr}; @@ -40,8 +46,9 @@ const CHUNK: usize = 1 << 16; /// What an object's bytes are. enum Backing { /// Frames of its own, one per page, in no order, [`CHUNK`] to a piece so - /// that no piece is past one heap allocation. - Frames(Vec>), + /// that no piece is past one heap allocation: each word its page's + /// frame, or 0 while the page is not committed, since no frame is at 0. + Frames(Vec>), /// 2 MiB pages of its own in one physical run: what a device may be lent /// or granted, and what a kernel-built image addresses as one slice. Run(Run), @@ -77,27 +84,85 @@ impl SharedMemObject { /// to `job`; `InvalidArgument` for no bytes or a size no page count /// holds, `ResourceExhausted` where frames run out. pub fn create(job: &Arc, size: u64) -> Result, SyscallError> { + Self::reserve(job, size, size) + } + + /// [`Self::create`], with only its first `committed` bytes' pages + /// committed; [`Self::commit`] commits the rest. + pub fn reserve(job: &Arc, size: u64, committed: u64) -> Result, SyscallError> { let pages = pages(size)?; - let (count, full, rest) = (pages.div_ceil(CHUNK as u64), pages / CHUNK as u64, (pages % CHUNK as u64) as usize); - let lists = charge::array::(CHUNK).saturating_mul(full as usize).saturating_add(charge::array::(rest)); - let core = Self::new_core_for(job, charge::array::>(count as usize).saturating_add(lists))?; - // Promised whole before any is taken: frames one at a time until none - // is left would starve every other allocation meanwhile. - if !pmm::promise(job, pages) { - return Err(SyscallError::ResourceExhausted); + if committed > size { + return Err(SyscallError::InvalidArgument); } + let (count, full, rest) = (pages.div_ceil(CHUNK as u64), pages / CHUNK as u64, (pages % CHUNK as u64) as usize); + let lists = charge::array::(CHUNK).saturating_mul(full as usize).saturating_add(charge::array::(rest)); + let core = Self::new_core_for(job, charge::array::>(count as usize).saturating_add(lists))?; let mut pieces = Vec::with_capacity(count as usize); let mut left = pages as usize; while left > 0 { let n = left.min(CHUNK); - let mut piece = Vec::with_capacity(n); - for _ in 0..n { - piece.push(pmm::alloc_promised()); - } - pieces.push(piece.into_boxed_slice()); + pieces.push((0..n).map(|_| AtomicU64::new(0)).collect()); left -= n; } - Ok(Self::new(core, Backing::Frames(pieces), pages * PAGE_SIZE, CachePolicy::Normal)) + let object = Self::new(core, Backing::Frames(pieces), pages * PAGE_SIZE, CachePolicy::Normal); + if committed > 0 { + object.commit(0, committed)?; + } + Ok(object) + } + + /// Commit every page `[offset, offset + len)` touches that is not yet, + /// each a zeroed frame counted to the object's job: refused whole, + /// `ResourceExhausted`, where its frames cannot all be promised. + /// `InvalidArgument` for no bytes, a range past the object, or an + /// aperture; memory with no page to commit answers at once. + pub fn commit(&self, offset: u64, len: u64) -> Result<(), SyscallError> { + if len == 0 || offset.checked_add(len).is_none_or(|end| end > self.size) { + return Err(SyscallError::InvalidArgument); + } + let pieces = match &self.backing { + Backing::Frames(pieces) => pieces, + Backing::Sub { whole, offset: base, .. } => return whole.commit(base + offset, len), + Backing::Run(_) | Backing::Boot { .. } => return Ok(()), + Backing::Aperture { .. } => return Err(SyscallError::InvalidArgument), + }; + let words = || { + let (first, end) = (offset / PAGE_SIZE, (offset + len).div_ceil(PAGE_SIZE)); + (first..end).map(|page| &pieces[page as usize / CHUNK][page as usize % CHUNK]) + }; + let wanted = words().filter(|word| word.load(Ordering::Acquire) == 0).count() as u64; + // Promised whole before any is taken, as a creation's are. + if !pmm::promise(self.core.job(), wanted) { + return Err(SyscallError::ResourceExhausted); + } + let mut drawn = 0; + // A page another commit filled meanwhile was counted here and is + // left; one it fills between the look and the store keeps its frame. + for word in words().filter(|word| word.load(Ordering::Acquire) == 0) { + let frame = pmm::alloc_promised().into_raw(); + drawn += 1; + // Release: a fault that reads the frame finds it zeroed. + if word.compare_exchange(0, frame, Ordering::Release, Ordering::Acquire).is_err() { + // SAFETY: drawn just now and named by nothing. + unsafe { pmm::free_drawn(self.core.job(), [frame]) }; + } + } + pmm::unpromise(self.core.job(), wanted - drawn); + Ok(()) + } + + /// The bytes of it that are committed. + pub fn committed(&self) -> u64 { + match &self.backing { + Backing::Frames(pieces) => { + let pages = pieces.iter().flat_map(|piece| piece.iter()).filter(|word| word.load(Ordering::Relaxed) != 0).count(); + pages as u64 * PAGE_SIZE + } + Backing::Sub { whole, offset, len } => { + (0..len.div_ceil(PAGE_SIZE)).filter(|page| whole.page(offset + page * PAGE_SIZE).is_some()).count() as u64 * PAGE_SIZE + } + Backing::Run(_) | Backing::Boot { .. } | Backing::Aperture { .. } => self.size, + } } /// `size` bytes of zeroed memory in whole pages, over one run of whole @@ -199,24 +264,26 @@ impl SharedMemObject { } } - /// The physical address of the page `offset` bytes in, which lies inside it. - pub fn page(&self, offset: u64) -> u64 { + /// The physical address of the page `offset` bytes in, which lies inside + /// it; `None` while that page is not committed. + pub fn page(&self, offset: u64) -> Option { assert!(offset < self.size, "shm koid {}: page {offset:#x} of {:#x} bytes", self.core.koid().raw(), self.size); let page = offset & !(PAGE_SIZE - 1); match &self.backing { Backing::Frames(pieces) => { let frame = (page / PAGE_SIZE) as usize; - pieces[frame / CHUNK][frame % CHUNK].direct_map().phys() + // Acquire: the commit's zeroing is seen before the frame is. + Some(pieces[frame / CHUNK][frame % CHUNK].load(Ordering::Acquire)).filter(|&phys| phys != 0) } - Backing::Run(run) => run.direct_map().phys() + page, - Backing::Boot { phys, .. } | Backing::Aperture { phys, .. } => phys + page, + Backing::Run(run) => Some(run.direct_map().phys() + page), + Backing::Boot { phys, .. } | Backing::Aperture { phys, .. } => Some(phys + page), Backing::Sub { whole, offset, .. } => whole.page(offset + page), } } /// Copy `bytes` in at `offset`, through the direct map. For memory the /// kernel lays out before a process can see it, and the words it alone - /// writes after; never an aperture. + /// writes after; never an aperture, and only over committed pages. pub fn write(&self, offset: u64, bytes: &[u8]) { assert!(self.is_ram(), "shm koid {}: the kernel writes no aperture", self.core.koid().raw()); self.each_piece(offset, bytes.len(), |phys, at, n| { @@ -251,7 +318,8 @@ impl SharedMemObject { while done < len { let at = offset + done as u64; let n = ((PAGE_SIZE - at % PAGE_SIZE) as usize).min(len - done); - f(self.page(at) + at % PAGE_SIZE, done, n); + let page = self.page(at).unwrap_or_else(|| panic!("shm koid {}: the kernel reached {at:#x}, not committed", self.core.koid().raw())); + f(page + at % PAGE_SIZE, done, n); done += n; } } @@ -288,15 +356,19 @@ fn pages(size: u64) -> Result { /// first every futex wait on a word in them ends, so none is left keyed on a /// frame the allocator hands out again. Nothing is allocated: each piece is /// sorted in place, its waits ended by one walk of the 64 buckets, and its -/// frames freed under one hold of the allocator's lock. +/// committed frames freed under one hold of the allocator's lock. impl Drop for SharedMemObject { fn drop(&mut self) { match &mut self.backing { Backing::Frames(pieces) => { for mut piece in pieces.drain(..) { - piece.sort_unstable_by_key(|frame| frame.direct_map().phys()); - crate::sched::futex::revoke_sorted(&piece, |frame| frame.direct_map().phys()); - pmm::free_all(self.core.job(), piece); + piece.sort_unstable_by_key(|word| word.load(Ordering::Relaxed)); + let committed = piece.partition_point(|word| word.load(Ordering::Relaxed) == 0); + let frames = &piece[committed..]; + crate::sched::futex::revoke_sorted(frames, |word| word.load(Ordering::Relaxed)); + // SAFETY: the last mapping and handle are gone, so no leaf + // maps a frame of it, and nothing else frees one. + unsafe { pmm::free_drawn(self.core.job(), frames.iter().map(|word| word.load(Ordering::Relaxed))) }; } } Backing::Run(run) => { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 575f26ee549..4599fa11608 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -39,7 +39,7 @@ use super::ipc::{ Incoming, Outgoing, sys_accept, sys_inbox_setup, sys_message_recv, sys_message_send, sys_inbox_submit, sys_namespace_build, sys_namespace_open, sys_pipe, - sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, sys_shm_sub, + sys_port_badge, sys_port_create, sys_port_mint, sys_shm_commit, sys_shm_create, sys_shm_map, sys_shm_sub, }; use super::machine::{ sys_boot_sources, sys_counters, sys_device_inventory, sys_log_read, sys_machine_stops, sys_reboot, sys_shutdown, @@ -471,7 +471,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> let Some(mut into) = into else { return bad_addr }; sys_message_recv(RawHandle(a1 as u32), &mut into, args.flags & MESSAGE_NONBLOCK != 0) } - SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32)), + SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32), a4), + SYS_SHM_COMMIT => sys_shm_commit(RawHandle(a1 as u32), a2, a3), SYS_SHM_MAP => sys_shm_map(RawHandle(a1 as u32), a2), SYS_SHM_SUB => sys_shm_sub(RawHandle(a1 as u32), a2, a3), SYS_CONSOLE_CARRIED => sys_console_carried(RawHandle(a1 as u32), a2), diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 90dfe083089..3fd1f91fae0 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -325,10 +325,10 @@ pub(super) fn sys_port_badge( } /// Make a shared region and hand back the one handle to it, carrying MAP, WRITE, DUP and TRANSFER — naming it is what authorizes mapping it. -/// `SHM_DEVICE` asks for memory a device can be lent, and `SHM_FOR_JOB` -/// counts it to the job `job` names rather than the caller's; any other -/// bit is refused. -pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { +/// `SHM_DEVICE` asks for memory a device can be lent, `SHM_PART` commits +/// only its first `committed` bytes, and `SHM_FOR_JOB` counts it to the job +/// `job` names rather than the caller's; any other bit is refused. +pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle, committed: u64) -> u64 { let charged = match flags & SHM_FOR_JOB { 0 => crate::mm::charge::current(), _ => match process::with_process_data(|data| data.handles.get::(job, Rights::WRITE)) { @@ -338,6 +338,7 @@ pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { }; let created = match flags & !SHM_FOR_JOB { 0 => crate::object::shm::SharedMemObject::create(&charged, size), + SHM_PART => crate::object::shm::SharedMemObject::reserve(&charged, size, committed), SHM_DEVICE => crate::object::shm::SharedMemObject::create_run(&charged, size), _ => return SyscallError::InvalidArgument.to_u64(), }; @@ -348,6 +349,19 @@ pub(super) fn sys_shm_create(size: u64, flags: u64, job: RawHandle) -> u64 { process::with_process_data(|data| handle_result(ops::install(&mut data.handles, object))) } +/// `toyos_abi::syscall::shm_commit`, charged to the region's own job. +pub(super) fn sys_shm_commit(h: RawHandle, offset: u64, len: u64) -> u64 { + // A refusal is made with nothing held: a bad handle's ends the process. + let object = match process::with_process_data(|data| data.handles.get::(h, Rights::MAP)) { + Ok(object) => object, + Err(e) => return e.refuse(), + }; + match object.commit(offset, len) { + Ok(()) => 0, + Err(e) => e.to_u64(), + } +} + /// `toyos_abi::syscall::shm_sub`: the new object's handle carries `h`'s rights. pub(super) fn sys_shm_sub(h: RawHandle, offset: u64, len: u64) -> u64 { // A refusal is made with nothing held: a bad handle's ends the process. diff --git a/kernel/src/vfs.rs b/kernel/src/vfs.rs index 8f98cf5c2db..6b0bbac59c3 100644 --- a/kernel/src/vfs.rs +++ b/kernel/src/vfs.rs @@ -488,7 +488,7 @@ impl Vfs { pub fn stat(&mut self, path: &str, follow: bool) -> Result { let (mount, file) = if follow { self.followed(path)? } else { self.resolve_path("/", path) }; let named = |file_type: FileType, size, mtime, volume, node| { - Stat { file_type: file_type as u64, size, mtime, volume, node } + Stat { file_type: file_type as u64, size, mtime, volume, node, committed: 0 } }; if mount.is_empty() { return Ok(named(FileType::Directory, 0, 0, NAMESPACE_VOLUME, ROOT_NODE)); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 390a8bca26b..c2fd05c2299 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -183,6 +183,10 @@ pub const SHM_DEVICE: u64 = 1; /// authority to place a child in a job is the authority to make memory for it. pub const SHM_FOR_JOB: u64 = 1 << 1; +/// [`shm_reserve`]: only the region's first bytes, the call's fourth +/// argument, are committed; [`shm_commit`] commits more. +pub const SHM_PART: u64 = 1 << 2; + /// What a [`SHM_DEVICE`] region is a whole number of. pub const SHM_DEVICE_PAGE: u64 = 2 * 1024 * 1024; @@ -373,6 +377,8 @@ pub const SYS_CURRENT_CPU: u64 = 163; /// Map a firmware window a claim was granted at its mint. See /// [`device_firmware_window`]. pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; +/// Commit pages of a region made with fewer. See [`shm_commit`]. +pub const SYS_SHM_COMMIT: u64 = 165; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -380,13 +386,13 @@ pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 166; +pub const SYSCALL_PROFILE_BINS: usize = 167; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_DEVICE_FIRMWARE_WINDOW < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_SHM_COMMIT < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -828,6 +834,9 @@ pub struct Stat { /// reaches it, for as long as it exists and its volume stays mounted, and /// no other file's while both exist. 0 exactly when `volume` is. pub node: u64, + /// A memory object's bytes that are committed ([`shm_commit`]): its + /// `size` but for a region made with fewer; 0 for anything else. + pub committed: u64, } #[cfg(target_arch = "x86_64")] @@ -2306,6 +2315,26 @@ pub fn shm_create_flags(size: usize, flags: u64) -> Result Result { + check(syscall(SYS_SHM_CREATE, size as u64, SHM_PART, 0, committed as u64)).map(|v| RawHandle(v as u32)) +} + +/// Commit every page `[offset, offset + len)` of the region `shm` names +/// touches that is not yet: zeroed, counted to the job the region is +/// counted to, and refused whole, `ResourceExhausted`, where they cannot all +/// be. A page committed already is left as it is, so any holder may commit +/// what it is about to touch. Needs [`Rights::MAP`]. `InvalidArgument` for +/// no bytes, a range past the region, or a device's window. +/// +/// [`Rights::MAP`]: crate::handle::Rights::MAP +pub fn shm_commit(shm: RawHandle, offset: usize, len: usize) -> Result<(), SyscallError> { + check(syscall(SYS_SHM_COMMIT, shm.0 as u64, offset as u64, len as u64, 0)).map(|_| ()) +} + /// [`shm_create`], counted to the job `job` names (carrying `WRITE`) rather /// than the caller's: memory a launcher lays out for a child it starts in /// that job, such as its log ring. diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index ac4288d067e..ee3ddbcf1f7 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -95,6 +95,12 @@ transport-no-resume = [] # A look without the gate is believed when it names a violation, so a cursor # another holder moved under it ends the call. transport-ungated-violation = [] +# A stream's consumer loads the ring's size before its tail, so it can read a +# byte written after a growth by the size before it. +transport-size-before-tail = [] +# A producer that grew its ring counts its room in the new capacity at once, +# so it writes over bytes before the switch the consumer has not read. +transport-grow-room-now = [] [lints.rust] # The guest's target: its stream sinks are compiled only for it. diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 7a8880aa05d..f3bb517ed40 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -65,17 +65,26 @@ pub use crate::transport::session::Way; // ---- the region ---- -/// The region a stream is: four 2 MiB pages. -pub const REGION_BYTES: usize = 8 << 20; +/// The region a stream is at most: its header and its two rings each grown +/// whole. A connect makes only the header and each ring's first +/// [`RECEIVE_INITIAL`] and [`SEND_INITIAL`] bytes memory; each ring's producer +/// makes more as it grows it (`crate::transport::stream`). +pub const REGION_BYTES: usize = SEND_RING + SEND_BYTES as usize; /// The header's words: its first 4 KiB. pub const WORDS: usize = 1024; const HEADER_BYTES: usize = WORDS * 4; -/// The receive ring, after the header. +/// The receive ring, after the header, at most. pub const RECEIVE_BYTES: u32 = 4 << 20; pub const RECEIVE_RING: usize = HEADER_BYTES; -/// The send ring, the rest. -pub const SEND_BYTES: u32 = (4 << 20) - HEADER_BYTES as u32; +/// The send ring, after it, at most. +pub const SEND_BYTES: u32 = 4 << 20; pub const SEND_RING: usize = RECEIVE_RING + RECEIVE_BYTES as usize; +/// The receive ring as a connect makes it: [tcp]'s first receive buffer +/// (`toyos_net_tcp::limits::RECEIVE_BUFFER_INITIAL`), rounded up to the power +/// of two a ring is. +pub const RECEIVE_INITIAL: u32 = 64 << 10; +/// The send ring as a connect makes it. +pub const SEND_INITIAL: u32 = 64 << 10; /// Word `n` of the protocol's, on a 64-byte line of its own. const fn line(n: usize) -> usize { @@ -93,11 +102,11 @@ pub const LAYOUT: Layout = Layout::new::< { line(13) }, { line(14) }, >( - Place::new::<{ line(0) }, { line(1) }, { line(2) }, { line(3) }, RECEIVE_RING>(), - Place::new::<{ line(4) }, { line(5) }, { line(6) }, { line(7) }, SEND_RING>(), + Place::new::<{ line(0) }, { line(1) }, { line(2) }, { line(3) }, { line(15) }, RECEIVE_RING>(), + Place::new::<{ line(4) }, { line(5) }, { line(6) }, { line(7) }, { line(16) }, SEND_RING>(), ); -const _: () = assert!(SEND_RING + SEND_BYTES as usize == REGION_BYTES && NODELAY < WORDS); +const _: () = assert!(line(16) < WORDS && NODELAY < WORDS); /// The header, as the one type that may alias words a peer writes. pub type Words = [AtomicU32; WORDS]; @@ -347,6 +356,11 @@ struct Call { } impl Call { + /// Make a ring's first `cap` bytes, at `ring`, memory. + fn commit(&self, ring: usize, cap: u32) -> bool { + self.end.region.is_some_and(|(region, _)| syscall::shm_commit(region, ring, cap as usize).is_ok()) + } + fn words(&self) -> &Words { // SAFETY: the region is mapped at `base`, 2 MiB aligned, for as long as // a call is counted, and its header is `WORDS` words reached only as @@ -521,7 +535,8 @@ pub fn read(handle: BorrowedHandle<'_>, buf: &mut [u8], wait: Wait) -> Result, buf: &[u8], wait: Wait) -> Result( ) -> Result, NetError> { let slot = TABLE.lock().reserve().ok_or(NetError::ResourceExhausted)?; let made = (|| { - let region = SharedMemory::create(REGION_BYTES).map_err(|e| match e { + let exhausted = |e| match e { SyscallError::ResourceExhausted => NetError::ResourceExhausted, _ => NetError::Io, - })?; + }; + let region = SharedMemory::reserve(REGION_BYTES, RECEIVE_RING + RECEIVE_INITIAL as usize).map_err(exhausted)?; + region.commit(SEND_RING, SEND_INITIAL as usize).map_err(exhausted)?; // SAFETY: a fresh region of `REGION_BYTES`, 2 MiB aligned, whose header // is reached only as atomics. let words = unsafe { &*(region.as_ptr() as *const Words) }; words[NODELAY].store(u32::from(nodelay), Ordering::Relaxed); - let (rx, tx) = session::client(words, LAYOUT); + let (rx, tx) = session::client(words, LAYOUT, SEND_INITIAL); let lent = region.share().map_err(|_| NetError::Io)?; let (answer, bells, connection) = ask(lent)?; let (addr, port) = answer.local(); diff --git a/sdk/toyos/src/shm.rs b/sdk/toyos/src/shm.rs index c681793b6b3..1b5fac358f3 100644 --- a/sdk/toyos/src/shm.rs +++ b/sdk/toyos/src/shm.rs @@ -62,6 +62,18 @@ impl SharedMemory { Self::adopt(OwnedHandle(syscall::shm_create(len)?), len) } + /// [`create`](Self::create), only the pages of its first `committed` + /// bytes memory until [`Self::commit`] makes more of it. + pub fn reserve(size: usize, committed: usize) -> Result { + Self::adopt(syscall::shm_reserve(size, committed)?, size) + } + + /// Make every page `[offset, offset + len)` of the region touches memory + /// that is not yet ([`syscall::shm_commit`]). + pub fn commit(&self, offset: usize, len: usize) -> Result<(), SyscallError> { + syscall::shm_commit(self.handle.raw(), offset, len) + } + /// [`create`](Self::create), counted to the job `job` is a handle to, /// carrying `WRITE`, rather than this process's own: memory laid out /// for a child started in it. diff --git a/sdk/toyos/src/transport/mod.rs b/sdk/toyos/src/transport/mod.rs index 92e94f5a7fe..e5ca57f328a 100644 --- a/sdk/toyos/src/transport/mod.rs +++ b/sdk/toyos/src/transport/mod.rs @@ -102,4 +102,7 @@ pub enum Violation { /// This end's own cursor, as another holder of the end stored it, names no /// position. OwnCursor, + /// A ring's size is past its place, or names a growth further from this + /// end than the old ring. + Size, } diff --git a/sdk/toyos/src/transport/session.rs b/sdk/toyos/src/transport/session.rs index dafe2b8db4f..c268aaa01ce 100644 --- a/sdk/toyos/src/transport/session.rs +++ b/sdk/toyos/src/transport/session.rs @@ -61,7 +61,7 @@ use core::num::NonZeroU16; use core::sync::atomic::Ordering; -use super::stream::{Consumer, Place, Producer, WAKE}; +use super::stream::{Consumer, Place, Producer, Size, WAKE}; use super::{word, Span, Violation, Word}; /// An `end` while its ring runs. @@ -174,7 +174,7 @@ pub struct Layout { impl Layout { /// The receive ring at `rx`, the send ring at `tx`, their `end`s and /// gates, [`shutdown_read`]'s word and the word that says the server hung - /// up. A layout whose words are not fourteen distinct ones below `N` does + /// up. A layout whose words are not sixteen distinct ones below `N` does /// not compile where it is a constant. pub const fn new< const RX_END: usize, @@ -187,14 +187,24 @@ impl Layout { rx: Place, tx: Place, ) -> Self { - let [a, b, c, d] = rx.words(); - let [e, f, g, h] = tx.words(); + let [a, b, c, d, e] = rx.words(); + let [f, g, h, i, j] = tx.words(); let own = [RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]; assert!(below(&own, N), "a word past the words"); - assert!(distinct(&[a, b, c, d, e, f, g, h, RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]), "a session's fourteen words are fourteen"); + assert!(distinct(&[a, b, c, d, e, f, g, h, i, j, RX_END, TX_END, RX_GATE, TX_GATE, READ_SHUT, GONE]), "a session's sixteen words are sixteen"); Self { rx, tx, rx_end: RX_END, tx_end: TX_END, rx_gate: RX_GATE, tx_gate: TX_GATE, read_shut: READ_SHUT, gone: GONE } } + /// Where the receive ring is. + pub const fn receive_place(&self) -> Place { + self.rx + } + + /// Where the send ring is. + pub const fn send_place(&self) -> Place { + self.tx + } + /// The receive ring's `end` as the server last stored it. pub fn received_end(&self, page: &[W; N]) -> u32 { word(page, self.rx_end).load(Ordering::Acquire) @@ -352,6 +362,8 @@ pub trait Half: sealed::Sealed { #[derive(Debug)] pub struct Receiving { ring: Consumer, + /// The ring's capacity this process made sure is memory. + reached: u32, end: usize, read_shut: usize, common: Common, @@ -362,6 +374,8 @@ pub struct Receiving { #[derive(Debug)] pub struct Sending { ring: Producer, + /// The ring's capacity this process made sure is memory. + reached: u32, /// The server's `end`, which a failure makes ready. received: usize, end: usize, @@ -515,14 +529,17 @@ impl Half for Sending { } } -/// The first holder's two halves, the end's cursors stored 0. +/// The first holder's two halves, the end's cursors stored 0 and its send +/// ring `send` bytes, which the region holds as memory. pub fn client( page: &[W; N], layout: Layout, + send: u32, ) -> (Receiving, Sending) { let (mut receiving, mut sending) = holder(layout, Holder::FIRST); receiving.ring = Consumer::new(page, layout.rx); - sending.ring = Producer::new(page, layout.tx); + sending.ring = Producer::new(page, layout.tx, send); + sending.reached = send; (receiving, sending) } @@ -535,6 +552,7 @@ pub fn holder( let common = |gate, shut| Common { gate, shut, gone: layout.gone, holder }; let receiving = Receiving { ring: Consumer::joining(layout.rx), + reached: 0, end: layout.rx_end, read_shut: layout.read_shut, common: common(layout.rx_gate, None), @@ -542,6 +560,7 @@ pub fn holder( }; let sending = Sending { ring: Producer::joining(layout.tx), + reached: 0, received: layout.rx_end, end: layout.tx_end, common: common(layout.tx_gate, Some(layout.tx_end)), @@ -807,13 +826,17 @@ where /// Reads at most `max` bytes, each span copied out by `copy` at its offset /// into the caller's buffer: the count, at least one, or 0 for an orderly end /// with every byte read, or once a holder shut the receiving half down. A read -/// of no bytes answers 0 at once. +/// of no bytes answers 0 at once. Before it copies past a capacity this +/// process has not reached, `reach` makes the ring's first that many bytes +/// memory; one it cannot is a size the server never made, +/// [`Violation::Size`]. pub fn read( page: &[W; N], lock: &L, bells: &B, mode: Mode, max: u32, + mut reach: impl FnMut(u32) -> bool, mut copy: impl FnMut(Span, usize), ) -> Result where @@ -829,8 +852,18 @@ where return (if seen.end == ORDERLY { Ok(0) } else { Err(Refused::Failed(seen.end)) }, false); } let n = seen.ready.min(max); + let cap = half.ring.capacity(); + if cap > half.reached { + if !reach(cap) { + return (Err(Refused::Violation(Violation::Size)), false); + } + half.reached = cap; + } + let Some(spans) = half.ring.span(0, n) else { + return (Err(Refused::Violation(Violation::TailPastDepth)), false); + }; let mut at = 0usize; - for span in half.ring.span(0, n).into_iter().flatten() { + for span in spans { copy(span, at); at = at.saturating_add(span.len); } @@ -844,13 +877,17 @@ where /// Writes at most `len` bytes, each span copied in by `copy` from its offset /// into the caller's buffer: the count, at least one. A shutdown, the server's /// hang-up or a failure it stored is answered, and nothing copied, whatever -/// room there is. A write of no bytes answers 0 at once. +/// room there is. A write of no bytes answers 0 at once. A write the room +/// cannot take whole doubles the ring, up to its place, where `reach` makes +/// the ring's first twice its capacity memory; where it cannot, the ring +/// stays as it is. pub fn write( page: &[W; N], lock: &L, bells: &B, mode: Mode, len: u32, + mut reach: impl FnMut(u32) -> bool, mut copy: impl FnMut(Span, usize), ) -> Result where @@ -867,14 +904,24 @@ where Writable::Failed(end) => (Err(Refused::Failed(end)), false), Writable::Room(room) => { let n = room.min(len); + let Some(spans) = half.ring.span(0, n) else { + return (Err(Refused::Violation(Violation::HeadPastTail)), false); + }; let mut at = 0usize; - for span in half.ring.span(0, n).into_iter().flatten() { + for span in spans { copy(span, at); at = at.saturating_add(span.len); } if !half.ring.advance(n) { return (Err(Refused::Violation(Violation::HeadPastTail)), false); } + if n < len && half.ring.can_grow() { + let cap = half.ring.capacity().saturating_mul(2); + if half.reached >= cap || reach(cap) { + half.reached = half.reached.max(cap); + half.ring.grow(page); + } + } (Ok(n), half.ring.publish(page)) } }) @@ -1107,14 +1154,55 @@ pub struct Server { } impl Server { - /// The server's end, its cursors stored 0. - pub fn new(page: &[W; N], layout: Layout) -> Self { - Self { layout, rx: Producer::new(page, layout.rx), tx: Consumer::new(page, layout.tx), ended: OPEN } + /// The server's end, its cursors stored 0 and its receive ring `receive` + /// bytes, which the region holds as memory. + pub fn new(page: &[W; N], layout: Layout, receive: u32) -> Self { + Self { layout, rx: Producer::new(page, layout.rx, receive), tx: Consumer::new(page, layout.tx), ended: OPEN } } /// Bytes published that the client has not released. pub fn held(&mut self, page: &[W; N]) -> Result { - Ok(RX.saturating_sub(self.rx.room(page)?)) + self.rx.room(page)?; + self.rx.held(page) + } + + /// The most the receive ring holds from the client's head [`Self::held`] + /// last read: what netstack may place in it. + pub fn receive_holds(&self) -> u32 { + self.rx.holds() + } + + /// The receive ring's size, which every byte netstack puts in it is laid + /// out by. + pub fn receive_size(&self) -> Size { + self.rx.size() + } + + /// The send ring's size as [`Self::sent`] last judged it, which the + /// caller makes sure is memory before it copies a byte by it. + pub fn send_size(&self) -> Size { + self.tx.size() + } + + /// The receive ring's capacity. + pub fn receive_size_cap(&self) -> u32 { + self.rx.capacity() + } + + /// The send ring's capacity as [`Self::sent`] last judged it. + pub fn send_size_cap(&self) -> u32 { + self.tx.capacity() + } + + /// Whether [`Self::grow_receive`] may double the receive ring now. + pub fn can_grow_receive(&self) -> bool { + self.rx.can_grow() + } + + /// Double the receive ring, the region below twice its capacity made + /// memory first. + pub fn grow_receive(&mut self, page: &[W; N]) { + self.rx.grow(page); } /// Publishes `n` more bytes, already in the ring behind the tail. @@ -1168,7 +1256,7 @@ impl Server { } self.ended = ORDERLY; word(page, self.layout.rx_end).store(ORDERLY, Ordering::Release); - let [_, _, data, _] = self.layout.rx.words(); + let [_, _, data, _, _] = self.layout.rx.words(); Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0) } @@ -1178,8 +1266,8 @@ impl Server { pub fn fail(&mut self, page: &[W; N], code: u32) -> Rings { self.ended = code; word(page, self.layout.rx_end).store(code, Ordering::Release); - let [_, _, data, _] = self.layout.rx.words(); - let [_, _, _, room] = self.layout.tx.words(); + let [_, _, data, _, _] = self.layout.rx.words(); + let [_, _, _, room, _] = self.layout.tx.words(); let receive = Rings::of(Way::Receive, word(page, data).swap(0, WAKE) != 0); receive.and(Rings::of(Way::Send, word(page, room).swap(0, WAKE) != 0)) } @@ -1227,14 +1315,14 @@ mod tests { use core::cell::{Cell, RefCell}; use core::sync::atomic::AtomicU32; - const N: usize = 14; + const N: usize = 16; const C: u32 = 4; const RX_GATE: usize = 10; const TX_GATE: usize = 11; const TX_END: usize = 9; const LAYOUT: Layout = Layout::new::<8, TX_END, RX_GATE, TX_GATE, 12, 13>( - Place::new::<0, 1, 2, 3, 0>(), - Place::new::<4, 5, 6, 7, { C as usize }>(), + Place::new::<0, 1, 2, 3, 14, 0>(), + Place::new::<4, 5, 6, 7, 15, { C as usize }>(), ); /// The send ring's two wants: the server's and the client's. const TX_DATA: usize = 6; @@ -1298,14 +1386,14 @@ mod tests { /// The first holder's halves and another's, over one page. fn two(page: &[AtomicU32; N]) -> (Halves, Halves) { - let (rx, tx) = client(page, LAYOUT); + let (rx, tx) = client(page, LAYOUT, C); let (other_rx, other_tx) = holder(LAYOUT, second()); ((Held(RefCell::new(rx)), Held(RefCell::new(tx))), (Held(RefCell::new(other_rx)), Held(RefCell::new(other_tx)))) } fn write_one(page: &[AtomicU32; N], tx: &Held>, bells: &B, mode: Mode) -> (Result, u32) { let mut copied = 0; - let wrote = write(page, tx, bells, mode, 1, |span, _| copied += span.len as u32); + let wrote = write(page, tx, bells, mode, 1, |_| true, |span, _| copied += span.len as u32); (wrote, copied) } @@ -1316,8 +1404,8 @@ mod tests { fn a_write_with_room_after_a_reset_answers_it_and_copies_nothing() { const RESET: u32 = 2; let page = page(); - let (_, sending) = client(&page, LAYOUT); - let mut server = Server::new(&page, LAYOUT); + let (_, sending) = client(&page, LAYOUT, C); + let mut server = Server::new(&page, LAYOUT, C); let _ = server.fail(&page, RESET); let (sending, bells) = (Held(RefCell::new(sending)), quiet()); assert_eq!(write_one(&page, &sending, &bells, Mode::Nonblocking), (Err(Refused::Failed(RESET)), 0)); @@ -1328,12 +1416,12 @@ mod tests { #[test] fn a_read_takes_every_byte_before_the_end() { let page = page(); - let (receiving, _) = client(&page, LAYOUT); - let mut server = Server::new(&page, LAYOUT); + let (receiving, _) = client(&page, LAYOUT, C); + let mut server = Server::new(&page, LAYOUT, C); let _ = server.publish(&page, 3); let _ = server.finish(&page); let (receiving, bells) = (Held(RefCell::new(receiving)), quiet()); - let read = |max| read(&page, &receiving, &bells, Mode::Nonblocking, max, |_, _| {}); + let read = |max| read(&page, &receiving, &bells, Mode::Nonblocking, max, |_| true, |_, _| {}); assert_eq!((read(2), read(2), read(2)), (Ok(2), Ok(1), Ok(0))); } @@ -1363,8 +1451,8 @@ mod tests { #[test] fn a_departure_frees_only_the_departed_holders_gates() { let page = page(); - let mut server = Server::new(&page, LAYOUT); - let _ = client(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); + let _ = client(&page, LAYOUT, C); page[RX_GATE].store(2 | WANT, Ordering::Relaxed); page[TX_GATE].store(1, Ordering::Relaxed); assert_eq!(server.departed(&page, second()), Ok(Rings::default()), "nobody armed a want"); @@ -1383,7 +1471,7 @@ mod tests { #[test] fn a_write_after_another_holders_shutdown_answers_shut_and_publishes_nothing() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((_, tx), (_, other_tx)) = two(&page); let bells = quiet(); shutdown(&page, &tx, &bells); @@ -1402,9 +1490,9 @@ mod tests { fn a_shutdown_under_another_holders_gate_is_stored_by_its_release() { let page = page(); let ((_, tx), _) = two(&page); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); // A full send ring: no room rings the shutting holder, only the shut. - let filled = write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_, _| {}); + let filled = write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_| true, |_, _| {}); assert_eq!(filled, Ok(C)); let owner = Common { gate: TX_GATE, shut: Some(TX_END), gone: 13, holder: second() }; assert_eq!(acquire(&page, owner), Ok(())); @@ -1450,7 +1538,7 @@ mod tests { fn a_blocking_write_under_another_holders_gate_waits_for_that_holders_gate() { let page = page(); let ((_, tx), _) = two(&page); - let server = RefCell::new(Server::new(&page, LAYOUT)); + let server = RefCell::new(Server::new(&page, LAYOUT, C)); let owner = Common { gate: TX_GATE, shut: Some(TX_END), gone: 13, holder: second() }; assert_eq!(acquire(&page, owner), Ok(())); let gated = Quiet { @@ -1463,7 +1551,7 @@ mod tests { }; assert_eq!(write_one(&page, &tx, &gated, Mode::Blocking(None)), (Ok(1), 1)); assert_eq!(*gated.gates.borrow(), [Some(2)]); - assert_eq!(write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_, _| {}), Ok(C - 1), "the ring filled"); + assert_eq!(write(&page, &tx, &quiet(), Mode::Nonblocking, C, |_| true, |_, _| {}), Ok(C - 1), "the ring filled"); let full = Quiet { rang: Cell::new(0), gone: false, @@ -1490,7 +1578,7 @@ mod tests { assert_eq!(gated.gate(), Some(2)); let _ = unwatch(&page, &tx, &bells, gated); let _ = release(&page, owner); - assert_eq!(write(&page, &tx, &bells, Mode::Nonblocking, C, |_, _| {}), Ok(C), "the ring filled"); + assert_eq!(write(&page, &tx, &bells, Mode::Nonblocking, C, |_| true, |_, _| {}), Ok(C), "the ring filled"); let Polled::Watching(full) = watch(&page, &tx, &bells) else { panic!("room in a full ring") }; assert_eq!(full.gate(), None, "a watch for room"); let _ = unwatch(&page, &tx, &bells, full); @@ -1501,15 +1589,15 @@ mod tests { #[test] fn a_hang_up_one_holder_saw_is_every_holders() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((rx, _), (other_rx, other_tx)) = two(&page); let _ = server.publish(&page, 1); let hung = Quiet:: { rang: Cell::new(0), gone: true, waited: None, gates: RefCell::new(Vec::new()) }; - assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_, _| {}), Ok(1), "the byte before the hang-up"); - assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::Gone)); + assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_| true, |_, _| {}), Ok(1), "the byte before the hang-up"); + assert_eq!(read(&page, &rx, &hung, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::Gone)); let quiet = quiet(); assert_eq!(write_one(&page, &other_tx, &quiet, Mode::Nonblocking), (Err(Refused::Gone), 0)); - assert_eq!(read(&page, &other_rx, &quiet, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::Gone)); + assert_eq!(read(&page, &other_rx, &quiet, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::Gone)); assert_eq!(quiet.rang.get(), 0, "nothing was rung"); } @@ -1548,15 +1636,15 @@ mod tests { #[test] fn a_read_shutdown_is_every_holders() { let page = page(); - let mut server = Server::new(&page, LAYOUT); + let mut server = Server::new(&page, LAYOUT, C); let ((rx, _), (other_rx, _)) = two(&page); let bells = quiet(); - assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_, _| {}), Err(Refused::WouldBlock), "armed"); + assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| {}), Err(Refused::WouldBlock), "armed"); shutdown_read(&page, &rx, &bells); assert_eq!(bells.rang.get(), 1); assert_eq!(server.rung(&page), Ok(Rings { receive: true, send: false }), "the reader that armed"); let _ = server.publish(&page, 2); - assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_, _| {}), Ok(0)); + assert_eq!(read(&page, &other_rx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| {}), Ok(0)); assert_eq!(server.held(&page), Ok(2), "a byte was read after the shutdown"); } @@ -1565,10 +1653,10 @@ mod tests { #[test] fn a_copy_that_unwinds_lets_the_gate_go_and_rings() { let page = page(); - let (_, tx) = client(&page, LAYOUT); + let (_, tx) = client(&page, LAYOUT, C); let (tx, bells) = (Held(RefCell::new(tx)), quiet()); let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - write(&page, &tx, &bells, Mode::Nonblocking, 1, |_, _| panic!("the copy faulted")) + write(&page, &tx, &bells, Mode::Nonblocking, 1, |_| true, |_, _| panic!("the copy faulted")) })); assert!(unwound.is_err()); assert_eq!(page[TX_GATE].load(Ordering::Relaxed), 0, "the gate is still held"); diff --git a/sdk/toyos/src/transport/stream.rs b/sdk/toyos/src/transport/stream.rs index 7e1a00c0464..1b976f450db 100644 --- a/sdk/toyos/src/transport/stream.rs +++ b/sdk/toyos/src/transport/stream.rs @@ -1,4 +1,5 @@ -//! A single-producer ring of `C` bytes, its cursors and wants among `N` words. +//! A single-producer ring of at most `C` bytes, its cursors, wants and size +//! among `N` words. //! //! **A producer copies bytes in through the [`Span`]s it is handed, then //! publishes its tail with `Release`; a consumer loads the tail with `Acquire`, @@ -8,11 +9,27 @@ //! peer's cursor only when it asks for its room or what is ready, and stores //! its own once a batch. //! -//! **A cursor is a position modulo twice the ring**, so a full ring and an empty -//! one differ and the ring may be any size. The peer's is believed only as a -//! position at most the ring away from what this end last stored; otherwise -//! [`Violation::TailPastDepth`] or [`Violation::HeadPastTail`] names it. Every -//! span lies in the ring whatever the peer wrote. +//! **A cursor is a stream position, counting every byte and wrapping at +//! 2^32**, and the ring's capacity is a power of two, so a byte's offset is +//! its position below the capacity whichever lap the cursor is on. The peer's +//! cursor is believed only as a position at most the ring away from what this +//! end last stored; otherwise [`Violation::TailPastDepth`] or +//! [`Violation::HeadPastTail`] names it. Every span lies in the ring whatever +//! the peer wrote. +//! +//! **The producer alone grows the ring, by doubling, whatever it holds.** The +//! size word holds the capacity and, while a growth is pending, the position +//! it switches at: the old ring past the head the producer last read, past +//! every byte it may have placed. A byte before it keeps its offset below the +//! old capacity and a byte from it on lies below the new one, so no byte +//! moves. Until the consumer's head passes the switch the producer's room is +//! still counted in the old capacity, so a byte from the switch on lands +//! where no unread byte lies: it would have to be a whole old ring past one. +//! The consumer loads the size after the tail, so it never reads a byte by a +//! size older than the byte. A consumer only touches the ring below the +//! capacity it was told, and judges a size before it believes it: a capacity +//! past `C`, or a switch more than the old ring past what it released, is +//! [`Violation::Size`]. //! //! **A sleeper never misses a wake.** An end about to sleep [`arm`]s its want, //! which exchanges 1 into it and only then looks at the peer's cursor again; it @@ -27,7 +44,7 @@ //! //! **An end may have several holders**, which take turns under a gate of the //! session's. Such a holder keeps no cursor between turns: it [`resume`]s its -//! own from the word the last holder stored, and refuses one past the lap. +//! own, and a producer its size, from the words the last holder stored. //! //! [`arm`]: Consumer::arm //! [`resume`]: Consumer::resume @@ -47,6 +64,9 @@ pub(super) const WAKE: Ordering = Ordering::Relaxed; /// after the consumer's copy out of it, which the consumer's store carries. const GIVE_BACK: Ordering = if cfg!(feature = "transport-head-relaxed") { Ordering::Relaxed } else { Ordering::Release }; const TAKE_BACK: Ordering = if cfg!(feature = "transport-head-relaxed") { Ordering::Relaxed } else { Ordering::Acquire }; +/// A consumer loads the size after the tail it reads by, so it never maps a +/// byte by an older size than the one it was written by. +const SIZE_AFTER_TAIL: bool = !cfg!(feature = "transport-size-before-tail"); /// `n` as an offset into the region; `Place::new` holds `usize` to every `u32`. #[allow(clippy::as_conversions)] @@ -54,19 +74,76 @@ const fn bytes(n: u32) -> usize { n as usize } -/// Where a ring of `C` bytes is: its four words among `N` and the region offset -/// of its first byte. +/// Whether position `at` comes before `switch`: they are within 2^31. +#[allow(clippy::as_conversions, clippy::cast_possible_wrap)] +const fn before(at: u32, switch: u32) -> bool { + (switch.wrapping_sub(at) as i32) > 0 +} + +/// A ring's capacity and the growth to it that may be pending. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Size { + /// Bytes, a power of two. + cap: u32, + /// The position the ring doubled to `cap` at, while a byte before it may + /// be unread. + switch: Option, +} + +/// The size word: log2 of the capacity in its low five bits, bit 5 for a +/// growth pending, and the switch's low 24 bits from bit 8. +const LOG: u32 = 0x1F; +const PENDING: u32 = 1 << 5; +const SWITCH: u32 = 8; + +impl Size { + /// The capacity the byte at `at` lies under. + fn cap_at(&self, at: u32) -> u32 { + match self.switch { + Some(switch) if before(at, switch) => self.cap / 2, + _ => self.cap, + } + } + + fn word(&self) -> u32 { + let log = self.cap.trailing_zeros(); + match self.switch { + Some(switch) => log | PENDING | (switch << SWITCH), + None => log, + } + } + + /// The size `word` says, its switch read as the position within 2^23 of + /// `near`; `None` for a capacity that is no power of two to `max`, or a + /// growth to a capacity of one. + #[allow(clippy::as_conversions, clippy::cast_possible_wrap, clippy::cast_sign_loss)] + fn of(word: u32, near: u32, max: u32) -> Option { + let log = word & LOG; + let cap = 1u32.checked_shl(log).filter(|&cap| cap <= max)?; + if word & PENDING == 0 { + return Some(Self { cap, switch: None }); + } + if cap < 2 { + return None; + } + let apart = (((word >> SWITCH).wrapping_sub(near) << SWITCH) as i32) >> SWITCH; + Some(Self { cap, switch: Some(near.wrapping_add(apart as u32)) }) + } +} + +/// Where a ring of at most `C` bytes is: its five words among `N` and the +/// region offset of its first byte. /// /// ``` /// use toyos::transport::stream::Place; -/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 48, 4096>(); +/// const RING: Place<4096, 80> = Place::new::<0, 16, 32, 48, 64, 4096>(); /// ``` /// /// A place with a word at `N` or past it does not compile: /// /// ```compile_fail,E0080 /// use toyos::transport::stream::Place; -/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 64, 4096>(); +/// const RING: Place<4096, 64> = Place::new::<0, 16, 32, 48, 64, 4096>(); /// ``` #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Place { @@ -76,65 +153,77 @@ pub struct Place { data: usize, /// Set by a producer about to sleep on a full ring. room: usize, + /// The producer's [`Size`]. + size: usize, base: usize, } impl Place { - /// What a cursor counts modulo. - const LAP: u32 = C.wrapping_mul(2); - - /// The cursor at word `at`, as an end's own last stored it. - fn own(page: &[W; N], at: usize) -> Result { - let cursor = word(page, at).load(Ordering::Acquire); - if cursor < Self::LAP { Ok(cursor) } else { Err(Violation::OwnCursor) } - } - - pub const fn new( - ) -> Self { + pub const fn new< + const HEAD: usize, + const TAIL: usize, + const DATA: usize, + const ROOM: usize, + const SIZE: usize, + const BASE: usize, + >() -> Self { const { - assert!(C > 0 && C <= 1 << 30, "a ring holds a byte and at most a gibibyte"); + assert!(C.is_power_of_two() && C <= 1 << 22, "a ring holds a power of two bytes, at most 4 MiB"); assert!(usize::BITS >= u32::BITS, "a ring offset is a u32"); - assert!(HEAD < N && TAIL < N && DATA < N && ROOM < N, "a place names a word past its words"); + assert!(HEAD < N && TAIL < N && DATA < N && ROOM < N && SIZE < N, "a place names a word past its words"); assert!( - HEAD != TAIL && HEAD != DATA && HEAD != ROOM && TAIL != DATA && TAIL != ROOM && DATA != ROOM, - "a ring's four words are four" + HEAD != TAIL && HEAD != DATA && HEAD != ROOM && HEAD != SIZE && TAIL != DATA && TAIL != ROOM + && TAIL != SIZE && DATA != ROOM && DATA != SIZE && ROOM != SIZE, + "a ring's five words are five" ); assert!(BASE.checked_add(bytes(C)).is_some(), "a ring ends inside the address space"); } - Self { head: HEAD, tail: TAIL, data: DATA, room: ROOM, base: BASE } + Self { head: HEAD, tail: TAIL, data: DATA, room: ROOM, size: SIZE, base: BASE } } - /// Its four words: head, tail and the two wants. - pub(super) const fn words(&self) -> [usize; 4] { - [self.head, self.tail, self.data, self.room] + /// Its five words: head, tail, the two wants and the size. + pub(super) const fn words(&self) -> [usize; 5] { + [self.head, self.tail, self.data, self.room, self.size] } - /// `at` moved `n` on; both are below the lap. - fn forward(at: u32, n: u32) -> u32 { - let to = at.wrapping_add(n); - if to >= Self::LAP { to.wrapping_sub(Self::LAP) } else { to } + /// The region offset of its first byte. + pub const fn base(&self) -> usize { + self.base } - /// How far `to` is past `from`; both are below the lap. + /// How far `to` is past `from`. fn distance(from: u32, to: u32) -> u32 { - let apart = to.wrapping_sub(from); - if to >= from { apart } else { apart.wrapping_add(Self::LAP) } - } - - /// How far the peer's `claimed` position is from this end's, by `apart`: - /// at most the ring. A word past the lap names no position, so it is - /// farther than any ring. - fn reach(claimed: u32, apart: impl FnOnce(u32) -> u32, broken: Violation) -> Result { - clamp(if claimed < Self::LAP { apart(claimed) } else { u32::MAX }, C, broken) + to.wrapping_sub(from) + } + + /// The region's bytes `len` on from position `at` by `size`, in ring + /// order: at most two before the switch and two after, since what lies + /// before it is at most the old ring and the rest at most the new. + /// `None` where they would be more, which no size this end believed makes. + pub fn spans(&self, size: Size, at: u32, len: u32) -> Option<[Span; 4]> { + let mut spans = [Span { offset: self.base, len: 0 }; 4]; + let (mut at, mut left) = (at, len); + for span in &mut spans { + if left == 0 { + break; + } + let cap = size.cap_at(at); + let to_switch = match size.switch { + Some(switch) if before(at, switch) => switch.wrapping_sub(at), + _ => u32::MAX, + }; + let slot = at & cap.wrapping_sub(1); + let run = left.min(to_switch).min(cap.wrapping_sub(slot)); + *span = Span { offset: self.base.wrapping_add(bytes(slot)), len: bytes(run) }; + at = at.wrapping_add(run); + left = left.wrapping_sub(run); + } + (left == 0).then_some(spans) } - /// The region's bytes `len` on from position `at`, in ring order: the - /// second span is empty unless they wrap. - fn spans(&self, at: u32, len: u32) -> [Span; 2] { - let slot = if at >= C { at.wrapping_sub(C) } else { at }; - let first = len.min(C.wrapping_sub(slot)); - let start = self.base.wrapping_add(bytes(slot)); - [Span { offset: start, len: bytes(first) }, Span { offset: self.base, len: bytes(len.wrapping_sub(first)) }] + /// The size the producer stored, judged by an end at `near`. + fn size(&self, page: &[W; N], near: u32) -> Result { + Size::of(word(page, self.size).load(Ordering::Acquire), near, C).ok_or(Violation::Size) } } @@ -143,51 +232,106 @@ impl Place { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Producer { place: Place, + size: Size, local: u32, published: u32, /// Bytes past `local` that may be written, as last seen. room: u32, + /// The head the room was last counted from. + head: u32, } impl Producer { - /// This end of the ring at `place`, its tail stored 0. - pub fn new(page: &[W; N], place: Place) -> Self { + /// This end of the ring at `place`, its tail stored 0 and its capacity + /// `cap`, a power of two to `C`. + pub fn new(page: &[W; N], place: Place, cap: u32) -> Self { + assert!(cap.is_power_of_two() && cap <= C, "a ring of {cap} bytes, at most {C}"); + let size = Size { cap, switch: None }; + word(page, place.size).store(size.word(), Ordering::Release); word(page, place.tail).store(0, Ordering::Release); - Self { place, local: 0, published: 0, room: C } + Self { place, size, local: 0, published: 0, room: cap, head: 0 } } - /// The end at `place` as its holders left it, its tail resumed before each - /// use ([`Self::resume`]). + /// The end at `place` as its holders left it, its tail and size resumed + /// before each use ([`Self::resume`]). pub const fn joining(place: Place) -> Self { - Self { place, local: 0, published: 0, room: 0 } + Self { place, size: Size { cap: 1, switch: None }, local: 0, published: 0, room: 0, head: 0 } } - /// Takes up the tail the end's last holder stored, nothing advanced past it. + /// Takes up the tail and the size the end's last holder stored, nothing + /// advanced past it. A switch more than the old ring from the tail is no + /// size a holder stored: `OwnCursor`. pub fn resume(&mut self, page: &[W; N]) -> Result<(), Violation> { if cfg!(feature = "transport-no-resume") { return Ok(()); } - let tail = Place::::own(page, self.place.tail)?; - *self = Self { local: tail, published: tail, room: 0, ..*self }; + let tail = word(page, self.place.tail).load(Ordering::Acquire); + let size = self.place.size(page, tail).map_err(|_| Violation::OwnCursor)?; + let far = |switch: u32| Place::::distance(switch, tail).min(Place::::distance(tail, switch)) > size.cap / 2; + if size.switch.is_some_and(far) { + return Err(Violation::OwnCursor); + } + *self = Self { size, local: tail, published: tail, room: 0, head: tail, ..*self }; Ok(()) } + /// The capacity it holds now: what a span may reach below the base. + pub fn capacity(&self) -> u32 { + self.size.cap + } + + /// The size its spans are laid out by, for an adapter that copies by + /// position ([`Place::spans`]). + pub fn size(&self) -> Size { + self.size + } + + /// The most it holds from the head [`Self::room`] last read: the old + /// capacity while a growth is pending before it. + pub fn holds(&self) -> u32 { + self.size.cap_at(self.head) + } + + /// Whether [`Self::grow`] may double it now: below `C`, and no growth + /// pending. + pub fn can_grow(&self) -> bool { + self.size.cap < C && self.size.switch.is_none() + } + + /// Double the ring, where [`Self::can_grow`] says so, from the old ring + /// past the head [`Self::room`] last read: the caller made the region + /// below twice the capacity memory first. The room grows once the + /// consumer is past the switch. + pub fn grow(&mut self, page: &[W; N]) { + assert!(self.can_grow(), "a ring of {} grown, at most {C} and none pending", self.size.cap); + let switch = self.head.wrapping_add(self.size.cap); + self.size = Size { cap: self.size.cap.wrapping_mul(2), switch: Some(switch) }; + word(page, self.place.size).store(self.size.word(), Ordering::Release); + } + /// How many bytes past the tail may be written, the consumer's head read - /// again. + /// again; a growth the consumer is past is no longer pending. pub fn room(&mut self, page: &[W; N]) -> Result { let head = word(page, self.place.head).load(TAKE_BACK); - let published = self.published; - let unreleased = - Place::::reach(head, |head| Place::::distance(head, published), Violation::HeadPastTail)?; + let unreleased = clamp(Place::::distance(head, self.published), self.size.cap, Violation::HeadPastTail)?; + if self.size.switch.is_some_and(|switch| !before(head, switch)) { + self.size.switch = None; + word(page, self.place.size).store(self.size.word(), Ordering::Release); + } + let cap = if cfg!(feature = "transport-grow-room-now") { self.size.cap } else { self.size.cap_at(head) }; let pending = Place::::distance(self.published, self.local); - self.room = C.saturating_sub(unreleased.saturating_add(pending)); + self.room = cap.saturating_sub(unreleased.saturating_add(pending)); + self.head = head; Ok(self.room) } /// Where to copy the `len` bytes `at` past the tail; `None` past the room /// last seen. - pub fn span(&self, at: u32, len: u32) -> Option<[Span; 2]> { - (at.checked_add(len)? <= self.room).then(|| self.place.spans(Place::::forward(self.local, at), len)) + pub fn span(&self, at: u32, len: u32) -> Option<[Span; 4]> { + if at.checked_add(len)? > self.room { + return None; + } + self.place.spans(self.size, self.local.wrapping_add(at), len) } /// Count `n` more bytes written at the tail; `false`, and nothing counted, @@ -196,7 +340,7 @@ impl Producer { #[must_use] pub fn advance(&mut self, n: u32) -> bool { let Some(room) = self.room.checked_sub(n) else { return false }; - self.local = Place::::forward(self.local, n); + self.local = self.local.wrapping_add(n); self.room = room; true } @@ -220,15 +364,18 @@ impl Producer { self.room(page) } + /// Bytes published that the consumer has not released. + pub fn held(&self, page: &[W; N]) -> Result { + let head = word(page, self.place.head).load(TAKE_BACK); + clamp(Place::::distance(head, self.published), self.size.cap, Violation::HeadPastTail) + } + /// The consumer asked to be rung back: `true`, and its want exchanged for /// 0, if it is set and the consumer has bytes past its head or the ring /// `ended`. With neither, nothing is exchanged and the want waits for the /// next publish. pub fn ring_back(&self, page: &[W; N], ended: bool) -> Result { - let head = word(page, self.place.head).load(TAKE_BACK); - let published = self.published; - let held = - Place::::reach(head, |head| Place::::distance(head, published), Violation::HeadPastTail)?; + let held = self.held(page)?; Ok((held > 0 || ended) && word(page, self.place.data).swap(0, WAKE) != 0) } } @@ -238,6 +385,8 @@ impl Producer { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Consumer { place: Place, + /// The size as last judged. + size: Size, local: u32, released: u32, /// Bytes published past `local`, as last seen. @@ -248,13 +397,13 @@ impl Consumer { /// This end of the ring at `place`, its head stored 0. pub fn new(page: &[W; N], place: Place) -> Self { word(page, place.head).store(0, Ordering::Release); - Self { place, local: 0, released: 0, ready: 0 } + Self::joining(place) } /// The end at `place` as its holders left it, its head resumed before each /// use ([`Self::resume`]). pub const fn joining(place: Place) -> Self { - Self { place, local: 0, released: 0, ready: 0 } + Self { place, size: Size { cap: 1, switch: None }, local: 0, released: 0, ready: 0 } } /// Takes up the head the end's last holder stored, nothing advanced past it. @@ -262,26 +411,54 @@ impl Consumer { if cfg!(feature = "transport-no-resume") { return Ok(()); } - let head = Place::::own(page, self.place.head)?; + let head = word(page, self.place.head).load(Ordering::Acquire); *self = Self { local: head, released: head, ready: 0, ..*self }; Ok(()) } - /// How many bytes past the head are published, the producer's tail read - /// again. - pub fn ready(&mut self, page: &[W; N]) -> Result { + /// The capacity last judged: what a span may reach below the base, which + /// the caller makes sure is memory before it copies. + pub fn capacity(&self) -> u32 { + self.size.cap + } + + /// The size last judged, as [`Producer::size`]. + pub fn size(&self) -> Size { + self.size + } + + /// The producer's tail and size, judged: the bytes published past what + /// this end released, and the size they are laid out by. + fn judged(&self, page: &[W; N]) -> Result<(u32, Size), Violation> { + let early = if SIZE_AFTER_TAIL { None } else { Some(self.place.size(page, self.released)) }; let tail = word(page, self.place.tail).load(Ordering::Acquire); - let released = self.released; - let published = - Place::::reach(tail, |tail| Place::::distance(released, tail), Violation::TailPastDepth)?; + let size = match early { + Some(size) => size?, + None => self.place.size(page, self.released)?, + }; + if size.switch.is_some_and(|switch| before(self.released, switch) && Place::::distance(self.released, switch) > size.cap / 2) { + return Err(Violation::Size); + } + let published = clamp(Place::::distance(self.released, tail), size.cap, Violation::TailPastDepth)?; + Ok((published, size)) + } + + /// How many bytes past the head are published, the producer's tail and + /// size read again. + pub fn ready(&mut self, page: &[W; N]) -> Result { + let (published, size) = self.judged(page)?; + self.size = size; self.ready = published.saturating_sub(Place::::distance(self.released, self.local)); Ok(self.ready) } /// Where to copy the `len` bytes `at` past the head from; `None` past what /// was last seen ready. - pub fn span(&self, at: u32, len: u32) -> Option<[Span; 2]> { - (at.checked_add(len)? <= self.ready).then(|| self.place.spans(Place::::forward(self.local, at), len)) + pub fn span(&self, at: u32, len: u32) -> Option<[Span; 4]> { + if at.checked_add(len)? > self.ready { + return None; + } + self.place.spans(self.size, self.local.wrapping_add(at), len) } /// Count `n` more bytes read at the head; `false`, and nothing counted, @@ -290,7 +467,7 @@ impl Consumer { #[must_use] pub fn advance(&mut self, n: u32) -> bool { let Some(ready) = self.ready.checked_sub(n) else { return false }; - self.local = Place::::forward(self.local, n); + self.local = self.local.wrapping_add(n); self.ready = ready; true } @@ -319,11 +496,8 @@ impl Consumer { /// or its direction `answers` without room. With neither, nothing is /// exchanged and the want waits for the next release. pub fn ring_back(&self, page: &[W; N], answers: bool) -> Result { - let tail = word(page, self.place.tail).load(Ordering::Acquire); - let released = self.released; - let unreleased = - Place::::reach(tail, |tail| Place::::distance(released, tail), Violation::TailPastDepth)?; - Ok((unreleased < C || answers) && word(page, self.place.room).swap(0, WAKE) != 0) + let (unreleased, size) = self.judged(page)?; + Ok((unreleased < size.cap_at(self.released) || answers) && word(page, self.place.room).swap(0, WAKE) != 0) } } @@ -332,7 +506,8 @@ mod tests { use super::*; use core::sync::atomic::AtomicU32; - const WORDS: usize = 4; + const WORDS: usize = 5; + const SIZE: usize = 4; /// Off any alignment, so a span that forgets the base lands outside it. const BASE: usize = 3; /// Bytes on either side of the ring that no span may reach. @@ -343,10 +518,10 @@ mod tests { } fn place() -> Place { - Place::new::<0, 1, 2, 3, BASE>() + Place::new::<0, 1, 2, 3, SIZE, BASE>() } - /// A region with the ring between two guards. + /// A region with the ring, at its most, between two guards. fn region() -> Vec { vec![0xA5; BASE + C as usize + GUARD] } @@ -373,14 +548,15 @@ mod tests { } } - fn inside(spans: [Span; 2], len: u32) { - assert_eq!(spans[0].len + spans[1].len, len as usize, "{spans:?} is not {len} bytes"); - for span in spans { - assert!(span.offset >= BASE && span.offset + span.len <= BASE + C as usize, "{span:?} leaves the ring"); + /// The spans hold `len` bytes and lie below `cap` past the base. + fn inside(spans: [Span; 4], len: u32, cap: u32) { + assert_eq!(spans.iter().map(|span| span.len).sum::(), len as usize, "{spans:?} is not {len} bytes"); + for span in spans.iter().filter(|span| span.len > 0) { + assert!(span.offset >= BASE && span.offset + span.len <= BASE + cap as usize, "{span:?} leaves a ring of {cap}"); } } - fn put(region: &mut [u8], spans: [Span; 2], from: u64) { + fn put(region: &mut [u8], spans: [Span; 4], from: u64) { let mut at = from; for span in spans { for b in &mut region[span.offset..span.offset + span.len] { @@ -390,7 +566,7 @@ mod tests { } } - fn get(region: &[u8], spans: [Span; 2]) -> Vec { + fn get(region: &[u8], spans: [Span; 4]) -> Vec { spans.iter().flat_map(|span| region[span.offset..span.offset + span.len].iter().copied()).collect() } @@ -398,26 +574,54 @@ mod tests { (from..from + u64::from(len)).map(byte).collect() } - /// The oracle is a stream that never wraps: four counts of bytes since the - /// start. The producer places each batch out of order, the consumer reads - /// at offsets as well as at its head, and every room, every count ready and - /// every byte read is what the counts say, round the lap many times. - fn agrees(seed: u64) { + /// What the producer's room is, by the module's rule: the old capacity + /// until the head is past a pending switch. + struct Model { + cap: u32, + switch: Option, + } + + impl Model { + fn room(&mut self, written: u64, released: u64) -> u64 { + match self.switch { + Some(switch) if released < switch => u64::from(self.cap / 2) - (written - released), + _ => { + self.switch = None; + u64::from(self.cap) - (written - released) + } + } + } + } + + /// The oracle is a stream that never wraps: four counts of bytes since + /// the start, and the rule the room follows. The producer starts at + /// `start`, places each batch out of order, and doubles the ring at + /// random to `C`; the consumer reads at offsets as well as at its head, + /// and every room, every count ready and every byte read is what the + /// counts say, round the lap many times. A byte the producer placed and + /// had not published when it grew is read as it was written. + fn agrees(start: u32, seed: u64) { let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), start), Consumer::new(&page, place::())); let mut region = region::(); let (mut written, mut published, mut read, mut released) = (0u64, 0u64, 0u64, 0u64); + let mut model = Model { cap: start, switch: None }; let mut rng = Rng(seed); for _ in 0..400 * C + 400 { - match rng.below(4) { + match rng.below(5) { 0 => { let room = tx.room(&page).unwrap(); - assert_eq!(u64::from(room), u64::from(C) - (written - released), "the room"); + assert_eq!(u64::from(room), model.room(written, released), "the room"); let n = rng.below(room + 1); let split = rng.below(n + 1); - for (at, len) in [(split, n - split), (0, split)] { - put(&mut region, tx.span(at, len).unwrap(), written + u64::from(at)); + // The later part is placed first, and a growth may come + // between it and the rest. + put(&mut region, tx.span(split, n - split).unwrap(), written + u64::from(split)); + if tx.can_grow() && rng.below(8) == 0 { + tx.grow(&page); + model = Model { cap: model.cap * 2, switch: Some(released + u64::from(model.cap)) }; } + put(&mut region, tx.span(0, split).unwrap(), written); assert!(tx.advance(n)); written += u64::from(n); } @@ -432,33 +636,78 @@ mod tests { let len = rng.below(ready + 1); (len, rng.below(ready - len + 1)) }; - assert_eq!(get(®ion, rx.span(at, len).unwrap()), expected(read + u64::from(at), len)); + let spans = rx.span(at, len).unwrap(); + inside(spans, len, rx.capacity()); + assert_eq!(get(®ion, spans), expected(read + u64::from(at), len)); let n = rng.below(ready + 1); assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n)); assert!(rx.advance(n)); read += u64::from(n); } - _ => { + 3 => { let _ = rx.release(&page); released = read; } + _ => {} } } - assert!(read > 4 * u64::from(C), "{read} bytes did not go round the lap twice"); + assert!(read > 4 * u64::from(model.cap), "{read} bytes did not go round the lap twice"); assert!(region[..BASE].iter().chain(®ion[BASE + C as usize..]).all(|&b| b == 0xA5), "a guard was written"); } #[test] fn the_ring_agrees_with_a_stream_that_never_wraps() { for seed in 1..=16 { - agrees::<1>(seed); - agrees::<2>(seed); - agrees::<5>(seed); - agrees::<8>(seed); - agrees::<61>(seed); + agrees::<1>(1, seed); + agrees::<2>(2, seed); + agrees::<8>(8, seed); + agrees::<64>(64, seed); } } + #[test] + fn a_ring_that_grows_agrees_with_the_stream_too() { + for seed in 1..=64 { + agrees::<64>(1, seed); + agrees::<256>(4, seed); + } + } + + /// The ring never empties: the producer keeps it at least half full and + /// the consumer reads only what leaves that much behind, so a growth that + /// waited for an empty ring never comes. It reaches its place, each + /// doubling within a few old rings of bytes, and every byte is read as + /// written. + #[test] + fn a_ring_kept_full_grows_to_its_place() { + const C: u32 = 1 << 12; + let page = page(); + let (mut tx, mut rx) = (Producer::new(&page, place::(), 16), Consumer::new(&page, place::())); + let mut region = region::(); + let (mut written, mut read) = (0u64, 0u64); + let mut grew_at = Vec::new(); + while tx.capacity() < C || tx.holds() < C { + assert!(written < 64 * u64::from(C), "the ring held {} after {written} bytes", tx.holds()); + let room = tx.room(&page).unwrap(); + if tx.can_grow() { + tx.grow(&page); + grew_at.push(written); + } + put(&mut region, tx.span(0, room).unwrap(), written); + assert!(tx.advance(room)); + written += u64::from(room); + let _ = tx.publish(&page); + let ready = rx.ready(&page).unwrap(); + let n = ready - ready.min(tx.holds() / 2); + assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n)); + assert!(rx.advance(n)); + read += u64::from(n); + let _ = rx.release(&page); + assert!(written > read, "the ring emptied"); + } + assert_eq!(grew_at.len(), 8, "16 bytes doubled to {C}: {grew_at:?}"); + } + /// A word a peer might write: anything, a position near the lap, or an edge. fn hostile(rng: &mut Rng) -> u32 { match rng.below(3) { @@ -468,24 +717,32 @@ mod tests { } } + /// A size word a peer might write: any capacity log, pending or not, a + /// switch anywhere. + fn hostile_size(rng: &mut Rng) -> u32 { + match rng.below(2) { + 0 => rng.next() as u32, + _ => rng.below(32) | (rng.below(2) << 5) | ((rng.next() as u32) << 8), + } + } + /// Whatever a producer, or another holder of this end, stores in any of - /// the four words, the consumer holds at most the ring past what it - /// released, every span it is handed lies in the ring, a tail it does not - /// believe is `TailPastDepth` and a head it resumes past the lap is - /// `OwnCursor`. + /// the five words, the consumer holds at most its capacity past what it + /// released, which is at most the place's, every span it is handed lies + /// below it, and what it does not believe is `TailPastDepth` or `Size`. fn hostile_producer(seed: u64) { let page = page(); let mut rx = Consumer::new(&page, place::()); let mut rng = Rng(seed); let mut held = 0; for _ in 0..2000 { - page[rng.below(4) as usize].store(hostile::(&mut rng), Ordering::Release); + let at = rng.below(5) as usize; + let value = if at == SIZE { hostile_size(&mut rng) } else { hostile::(&mut rng) }; + page[at].store(value, Ordering::Release); let seen = match rng.below(5) { 4 => { - match rx.resume(&page) { - Ok(()) => held = 0, - Err(violation) => assert_eq!(violation, Violation::OwnCursor), - } + assert_eq!(rx.resume(&page), Ok(())); + held = 0; continue; } 0 => rx.ready(&page), @@ -494,7 +751,7 @@ mod tests { let n = rng.below(C + 1); match rx.span(0, n) { Some(spans) => { - inside::(spans, n); + inside(spans, n, rx.capacity()); assert!(rx.advance(n)); held += n; } @@ -508,12 +765,14 @@ mod tests { continue; } }; + assert!(rx.capacity() <= C); match seen { Ok(ready) => assert!( - u64::from(held) + u64::from(ready) <= u64::from(C), - "holds {held} and was told of {ready} more in a ring of {C}" + u64::from(held) + u64::from(ready) <= u64::from(rx.capacity()), + "holds {held} and was told of {ready} more in a ring of {}", + rx.capacity() ), - Err(violation) => assert_eq!(violation, Violation::TailPastDepth), + Err(violation) => assert!(matches!(violation, Violation::TailPastDepth | Violation::Size), "{violation:?}"), } } } @@ -522,23 +781,25 @@ mod tests { fn a_hostile_producer_hands_the_consumer_at_most_the_ring() { for seed in 1..=64 { hostile_producer::<1>(seed); - hostile_producer::<5>(seed); hostile_producer::<8>(seed); + hostile_producer::<64>(seed); } } /// Whatever a consumer, or another holder of this end, stores in any of - /// the four words, the producer writes at most the ring past the head it - /// was shown, every span it is handed lies in the ring, a head it does not - /// believe is `HeadPastTail` and a tail it resumes past the lap is + /// the five words, the producer writes at most its capacity past the head + /// it was shown, every span it is handed lies below it, a head it does not + /// believe is `HeadPastTail` and a size it resumes that it never stored is /// `OwnCursor`. fn hostile_consumer(seed: u64) { let page = page(); - let mut tx = Producer::new(&page, place::()); + let mut tx = Producer::new(&page, place::(), C); let mut rng = Rng(seed); let mut pending = 0; for _ in 0..2000 { - page[rng.below(4) as usize].store(hostile::(&mut rng), Ordering::Release); + let at = rng.below(5) as usize; + let value = if at == SIZE { hostile_size(&mut rng) } else { hostile::(&mut rng) }; + page[at].store(value, Ordering::Release); let seen = match rng.below(5) { 4 => { match tx.resume(&page) { @@ -553,7 +814,7 @@ mod tests { let n = rng.below(C + 1); match tx.span(0, n) { Some(spans) => { - inside::(spans, n); + inside(spans, n, tx.capacity()); assert!(tx.advance(n)); pending += n; } @@ -569,8 +830,9 @@ mod tests { }; match seen { Ok(room) => assert!( - u64::from(pending) + u64::from(room) <= u64::from(C), - "{pending} unpublished and {room} more in a ring of {C}" + u64::from(pending) + u64::from(room) <= u64::from(tx.capacity()), + "{pending} unpublished and {room} more in a ring of {}", + tx.capacity() ), Err(violation) => assert_eq!(violation, Violation::HeadPastTail), } @@ -581,39 +843,31 @@ mod tests { fn a_hostile_consumer_gives_the_producer_at_most_the_ring() { for seed in 1..=64 { hostile_consumer::<1>(seed); - hostile_consumer::<5>(seed); hostile_consumer::<8>(seed); + hostile_consumer::<64>(seed); } } - /// From every position round the lap, twice: a tail the ring ahead is - /// believed, and one a byte further, a word past the lap or the last word - /// is not; a head the ring behind is believed, and one a byte past what was - /// published or a word past the lap is not. + /// From positions on both sides of the u32 wrap: a tail the ring ahead is + /// believed and one a byte further is not; a head the ring behind is + /// believed, and one a byte past what was published is not. #[test] fn a_cursor_is_believed_up_to_the_ring_and_no_further() { - const C: u32 = 5; - const LAP: u32 = 2 * C; + const C: u32 = 4; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); - for at in 0..2 * LAP { - let here = at % LAP; - let tails = [ - ((here + C) % LAP, Ok(C)), - ((here + C + 1) % LAP, Err(Violation::TailPastDepth)), - (LAP, Err(Violation::TailPastDepth)), - (u32::MAX, Err(Violation::TailPastDepth)), - ]; - for (tail, believed) in tails { + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); + let from = u32::MAX - 2 * C; + page[0].store(from, Ordering::Release); + page[1].store(from, Ordering::Release); + assert_eq!((rx.resume(&page), tx.resume(&page)), (Ok(()), Ok(()))); + for _ in 0..4 * C { + let here = rx.released; + for (tail, believed) in [(here.wrapping_add(C), Ok(C)), (here.wrapping_add(C + 1), Err(Violation::TailPastDepth))] { page[1].store(tail, Ordering::Release); assert_eq!(rx.ready(&page), believed, "tail {tail} with {here} released"); } - let heads = [ - ((here + LAP - C) % LAP, Ok(0)), - ((here + 1) % LAP, Err(Violation::HeadPastTail)), - (LAP, Err(Violation::HeadPastTail)), - ]; - for (head, believed) in heads { + page[1].store(here, Ordering::Release); + for (head, believed) in [(here.wrapping_sub(C), Ok(0)), (here.wrapping_add(1), Err(Violation::HeadPastTail))] { page[0].store(head, Ordering::Release); assert_eq!(tx.room(&page), believed, "head {head} with {here} published"); } @@ -627,25 +881,21 @@ mod tests { } } - /// A holder resumes its own cursor from any position below the lap, and - /// refuses the lap and past it. + /// A producer resumes the size its end's last holder stored, and refuses + /// one whose switch is further than the old ring from its tail. #[test] - fn a_resumed_cursor_is_believed_below_the_lap_and_no_further() { - const C: u32 = 5; + fn a_resumed_size_is_believed_only_near_the_tail() { + const C: u32 = 16; let page = page(); - let (mut tx, mut rx) = (Producer::joining(place::()), Consumer::joining(place::())); - for at in [0, 1, C, 2 * C - 1] { - page[0].store(at, Ordering::Release); - page[1].store(at, Ordering::Release); - assert_eq!((rx.resume(&page), tx.resume(&page)), (Ok(()), Ok(())), "a cursor at {at}"); - assert_eq!((rx.ready(&page), tx.room(&page)), (Ok(0), Ok(C)), "the ring at {at} is empty"); - } - for at in [2 * C, 2 * C + 1, u32::MAX] { - page[0].store(at, Ordering::Release); - page[1].store(at, Ordering::Release); - assert_eq!(rx.resume(&page), Err(Violation::OwnCursor), "a head at {at}"); - assert_eq!(tx.resume(&page), Err(Violation::OwnCursor), "a tail at {at}"); - } + let mut first = Producer::new(&page, place::(), 4); + assert_eq!(first.room(&page), Ok(4)); + first.grow(&page); + let mut next = Producer::joining(place::()); + assert_eq!(next.resume(&page), Ok(())); + assert_eq!((next.capacity(), next.can_grow()), (8, false), "the growth the first holder made"); + let far = Size { cap: 8, switch: Some(64) }.word(); + page[SIZE].store(far, Ordering::Release); + assert_eq!(next.resume(&page), Err(Violation::OwnCursor)); } /// A ring back exchanges a want only of a peer whose direction is ready, @@ -654,7 +904,7 @@ mod tests { fn a_ring_back_answers_only_a_peer_that_armed_and_is_ready() { const C: u32 = 2; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); assert_eq!(rx.arm(&page), Ok(0)); assert_eq!(tx.ring_back(&page, false), Ok(false), "nothing to read: the want stays"); assert_eq!(tx.ring_back(&page, true), Ok(true), "the ring ended"); @@ -680,7 +930,7 @@ mod tests { fn an_end_rings_only_a_peer_that_armed() { const C: u32 = 4; let page = page(); - let (mut tx, mut rx) = (Producer::new(&page, place::()), Consumer::new(&page, place::())); + let (mut tx, mut rx) = (Producer::new(&page, place::(), C), Consumer::new(&page, place::())); assert!(tx.advance(1)); assert!(!tx.publish(&page), "nobody armed"); assert_eq!(rx.arm(&page), Ok(1), "the look after arming finds the byte"); diff --git a/system/netstack/node/src/lib.rs b/system/netstack/node/src/lib.rs index 95a73008b9e..48e4c6ce212 100644 --- a/system/netstack/node/src/lib.rs +++ b/system/netstack/node/src/lib.rs @@ -49,7 +49,7 @@ mod streams; pub use holders::{Bell, HolderId}; pub use listeners::{AcceptRefused, Accepted, ListenRefused, ListenerId}; -pub use streams::{ConnectRefused, ExportRefused, Session, StreamEvent, StreamId}; +pub use streams::{ConnectRefused, ExportRefused, Memory, Session, StreamEvent, StreamId}; use alloc::vec; use alloc::vec::Vec; diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index 57c316ef634..b8576485449 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -10,8 +10,15 @@ //! [tcp] has room, and `rx_room` only while the window last offered is below the update //! threshold. //! -//! **Every word read from the region is the client's, and judged**: a cursor past its bound, or -//! text published after its own shutdown, is a [`Violation`] that resets that stream alone. +//! **Every word read from the region is the client's, and judged**: a cursor past its bound, +//! text published after its own shutdown, or a send ring larger than the region can be made +//! memory for, is a [`Violation`] that resets that stream alone. +//! +//! **The rings grow under their producers** (`toyos::transport::stream`): the node doubles the +//! receive ring when [tcp] wants a larger receive buffer than it holds, after making the region +//! below the new capacity memory, and makes the send ring's memory before [tcp] reads by a size +//! the client grew it to. [tcp] addresses each ring by stream position, laid out by the size the +//! pass last judged. //! //! **A stream has holders**: the client that connected or accepted it, and one more for each //! [`Node::export`], each a process of its own over the one region, each taking a place past @@ -26,15 +33,18 @@ use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; +use alloc::rc::Rc; +use core::cell::Cell; use core::sync::atomic::Ordering; use core::time::Duration; -use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_BYTES, RESET, TIMED_OUT, UNREACHABLE}; +use toyos::net::stream::{Server, Words, ADDRESS_GONE, LAYOUT, NODELAY, RECEIVE_INITIAL, RECEIVE_RING, RESET, SEND_INITIAL, SEND_RING, TIMED_OUT, UNREACHABLE, WORDS}; use toyos_net_shard::ConnectError; use toyos_net_tcp::{ConnId, Endpoint, Error, Failure, Lent, Options, State, Status}; use toyos_net_wire::Instant; use toyos::transport::session::{Holder, Rings}; -use toyos::transport::Violation; +use toyos::transport::stream::{Place, Size}; +use toyos::transport::{Span, Violation}; use crate::lease::Stack; use crate::Node; @@ -44,14 +54,87 @@ use crate::Node; pub trait Session { /// The region's header. fn words(&self) -> &Words; - /// The region's receive ring, as [tcp]'s receive buffer. - fn receive(&self) -> Box; - /// The region's send ring, as [tcp]'s send buffer. - fn send(&self) -> Box; + /// The region's bytes, which the node lays its two rings out in. + fn memory(&self) -> Rc; /// Rings the doorbells `rings` names, of every holder, never waiting. fn ring(&self, rings: Rings); } +/// A region's bytes by offset, reached only by copies. +pub trait Memory: core::fmt::Debug { + /// Copies `data` in at `at`, inside memory [`Self::commit`] made. + fn put(&self, at: usize, data: &[u8]); + /// Copies `out.len()` bytes out from `at`, inside memory [`Self::commit`] made. + fn take(&self, at: usize, out: &mut [u8]); + /// Makes `[at, at + len)` memory where it is not yet; `false` where it cannot be. + fn commit(&self, at: usize, len: usize) -> bool; +} + +/// What a ring is laid out by and holds now, which the pass writes and [tcp]'s storage reads. +#[derive(Clone, Copy, Debug)] +struct Shape { + size: Size, + /// What it holds from the oldest byte unread. + holds: u32, + /// What [tcp] last asked it to hold. + wanted: usize, +} + +/// One of the region's rings, as [tcp]'s storage. +#[derive(Debug)] +struct LentRing { + memory: Rc, + place: Place, + shape: Rc>, +} + +impl LentRing { + /// The region's bytes at positions `[at, at + len)`, each with where it starts among them. + fn spans(&self, at: usize, len: usize) -> [(Span, usize); 4] { + let none = (Span { offset: 0, len: 0 }, 0); + let mut out = [none; 4]; + let (Ok(at), Ok(len)) = (u32::try_from(at & 0xFFFF_FFFF), u32::try_from(len)) else { return out }; + let Some(spans) = self.place.spans(self.shape.get().size, at, len) else { return out }; + let mut from = 0usize; + for (slot, span) in out.iter_mut().zip(spans) { + *slot = (span, from); + from = from.saturating_add(span.len); + } + out + } +} + +impl Lent for LentRing { + fn len(&self) -> usize { + usize::try_from(self.shape.get().holds).unwrap_or(0) + } + + fn most(&self) -> usize { + usize::try_from(C).unwrap_or(0) + } + + fn want(&mut self, bytes: usize) { + let shape = self.shape.get(); + self.shape.set(Shape { wanted: shape.wanted.max(bytes), ..shape }); + } + + fn put(&mut self, at: usize, data: &[u8]) { + for (span, from) in self.spans(at, data.len()) { + if let Some(part) = data.get(from..from.saturating_add(span.len)) { + self.memory.put(span.offset, part); + } + } + } + + fn take(&self, at: usize, out: &mut [u8]) { + for (span, from) in self.spans(at, out.len()) { + if let Some(part) = out.get_mut(from..from.saturating_add(span.len)) { + self.memory.take(span.offset, part); + } + } + } +} + /// Why a connect made no stream. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ConnectRefused { @@ -112,6 +195,10 @@ fn count(n: usize) -> u32 { struct Stream { conn: ConnId, session: Box, + /// The receive ring's and the send ring's shapes, as [tcp]'s storage reads them. + shapes: Option<(Rc>, Rc>)>, + /// What the next pass ends the stream with. + broken: Option, /// The session's server end, once the connection is established and its buffers are the /// region's; `None` while a connect is not answered. server: Option, @@ -132,13 +219,26 @@ struct Stream { } impl Stream { - /// A connection whose buffers are the region's from here on, and its server end. + /// A connection whose buffers are the region's from here on, and its server end. A region + /// whose first rings cannot be made memory holds nothing, and the next pass ends the stream + /// with [`Violation::Size`]. fn lent(&mut self, stack: &mut Stack) { let session = &self.session; - if stack.tcp_lend(self.conn, session.receive(), session.send()).is_err() { + let memory = session.memory(); + let first = |ring, cap: u32| memory.commit(ring, usize::try_from(cap).unwrap_or(usize::MAX)); + let made = first(RECEIVE_RING, RECEIVE_INITIAL) && first(SEND_RING, SEND_INITIAL); + self.broken = (!made).then_some(Violation::Size); + let server = Server::new(session.words(), LAYOUT, RECEIVE_INITIAL); + let (receive, send) = if made { (RECEIVE_INITIAL, SEND_INITIAL) } else { (0, 0) }; + let rx = Rc::new(Cell::new(Shape { size: server.receive_size(), holds: receive, wanted: 0 })); + let tx = Rc::new(Cell::new(Shape { size: server.send_size(), holds: send, wanted: 0 })); + let receive = LentRing { memory: memory.clone(), place: LAYOUT.receive_place(), shape: rx.clone() }; + let send = LentRing { memory, place: LAYOUT.send_place(), shape: tx.clone() }; + if stack.tcp_lend(self.conn, Box::new(receive), Box::new(send)).is_err() { unreachable!("[tcp] would not lend an established connection its client's buffers"); } - self.server = Some(Server::new(session.words(), LAYOUT)); + self.server = Some(server); + self.shapes = Some((rx, tx)); } /// One pass. `false` lets the stream go: its connection is [tcp]'s alone or gone. @@ -181,11 +281,23 @@ impl Stream { /// more, and rings the client once: `false` once the connection ended. fn moved(&mut self, now: Instant, stack: &mut Stack) -> Result { let conn = self.conn; + if let Some(violation) = self.broken.take() { + return Err(violation); + } let Some(server) = self.server.as_mut() else { unreachable!("a stream moves only once established") }; + let Some((rx_shape, tx_shape)) = self.shapes.as_ref() else { unreachable!("a lent stream has its shapes") }; let words = self.session.words(); let mut rings = Rings::default(); let keep = loop { let held = server.held(words)?; + let rx = rx_shape.get(); + if rx.wanted > usize::try_from(server.receive_size_cap()).unwrap_or(usize::MAX) && server.can_grow_receive() { + let more = server.receive_size_cap().saturating_mul(2); + if self.session.memory().commit(RECEIVE_RING, usize::try_from(more).unwrap_or(usize::MAX)) { + server.grow_receive(words); + } + } + rx_shape.set(Shape { size: server.receive_size(), holds: server.receive_holds(), ..rx_shape.get() }); let released = self.held.checked_sub(held).ok_or(Violation::HeadPastTail)?; if released > 0 { stack.tcp_consume(now, conn, released); @@ -206,7 +318,16 @@ impl Stream { } self.queued = queued; let shut = server.shut(words); - let fresh = server.sent(words)?.checked_sub(queued).ok_or(Violation::TailPastDepth)?; + let sent = server.sent(words)?; + let tx = tx_shape.get(); + let cap = server.send_size_cap(); + if cap > tx.holds { + if !self.session.memory().commit(SEND_RING, usize::try_from(cap).unwrap_or(usize::MAX)) { + return Err(Violation::Size); + } + } + tx_shape.set(Shape { size: server.send_size(), holds: cap.max(tx.holds), ..tx }); + let fresh = sent.checked_sub(queued).ok_or(Violation::TailPastDepth)?; if fresh > 0 && status.failure.is_none() { match stack.tcp_queue(now, conn, fresh) { // [tcp]'s send buffer is the whole ring, so what the ring held fits it. @@ -242,7 +363,7 @@ impl Stream { more |= server.want_data(words)? > self.queued; } if status.starved { - more |= RECEIVE_BYTES.saturating_sub(server.want_room(words)?) < self.held; + more |= server.receive_holds().saturating_sub(server.want_room(words)?) < self.held; } if !more { break true; @@ -275,7 +396,7 @@ impl Streams { /// has, `nodelay` among them written into the region's header. pub(crate) fn accepted(&mut self, stack: &mut Stack, conn: ConnId, options: Options, session: Box) -> StreamId { session.words()[NODELAY].store(u32::from(options.nodelay), Ordering::Relaxed); - let mut stream = Stream { conn, session, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; + let mut stream = Stream { conn, session, shapes: None, broken: None, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; stream.lent(stack); self.hold(stream) } @@ -302,7 +423,7 @@ impl Node { let options = Options { nodelay: session.words()[NODELAY].load(Ordering::Relaxed) != 0, ..Options::default() }; self.stack.tcp_set_options(now, conn, options); let deadline = timeout.map(|within| now.after(within)); - Ok(self.streams.hold(Stream { conn, session, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) + Ok(self.streams.hold(Stream { conn, session, shapes: None, broken: None, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) } /// One pass over every stream: netstack calls it once the kernel answered the connections it diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs index 75d983548df..541ffb32045 100644 --- a/system/netstack/node/tests/common/client.rs +++ b/system/netstack/node/tests/common/client.rs @@ -8,15 +8,16 @@ use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::sync::atomic::{AtomicU32, Ordering}; -use toyos::net::stream::{Words, LAYOUT, NODELAY, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; -use toyos_net_node::Session; -use toyos_net_tcp::Lent; +use toyos::net::stream::{Words, LAYOUT, NODELAY, RECEIVE_BYTES, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_INITIAL, SEND_RING}; +use toyos_net_node::{Memory, Session}; use toyos::transport::session::{self, Bells, Doorbell, Holder, HungUp, Lock, Mode, Receiving, Refused, Rings, Sending, Took, Waited}; /// The region and its holders' doorbells, which the holders and the node share. pub struct Region { words: Box, bytes: RefCell>, + /// Each commit's range, the client's and the node's. + committed: RefCell>, holders: RefCell>>, /// A holder rang netstack since the test last asked. rang: Cell, @@ -78,33 +79,38 @@ pub struct Client { tx: Held>, } -/// One ring of the region, as [tcp]'s storage. -#[derive(Debug)] -struct Ring { - region: Rc, - base: usize, - len: usize, -} - impl std::fmt::Debug for Region { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("Region") } } -impl Lent for Ring { - fn len(&self) -> usize { - self.len +impl Memory for Region { + fn put(&self, at: usize, data: &[u8]) { + assert!(self.reaches(at, data.len()), "a put of {} bytes at {at} past what was committed", data.len()); + self.bytes.borrow_mut()[at..][..data.len()].copy_from_slice(data); } - fn put(&mut self, at: usize, data: &[u8]) { - assert!(at + data.len() <= self.len, "a put past the ring"); - self.region.bytes.borrow_mut()[self.base + at..][..data.len()].copy_from_slice(data); + fn take(&self, at: usize, out: &mut [u8]) { + assert!(self.reaches(at, out.len()), "a take of {} bytes at {at} past what was committed", out.len()); + out.copy_from_slice(&self.bytes.borrow()[at..][..out.len()]); } - fn take(&self, at: usize, out: &mut [u8]) { - assert!(at + out.len() <= self.len, "a take past the ring"); - out.copy_from_slice(&self.region.bytes.borrow()[self.base + at..][..out.len()]); + fn commit(&self, at: usize, len: usize) -> bool { + self.committed.borrow_mut().push((at, at + len)); + true + } +} + +impl Region { + /// Whether `[at, at + len)` lies in what a commit made memory, as a fault would ask. + fn reaches(&self, at: usize, len: usize) -> bool { + self.committed.borrow().iter().any(|&(start, end)| start <= at && at + len <= end) + } + + /// The most of the ring at `ring` committed from its start. + pub fn committed(&self, ring: usize) -> usize { + self.committed.borrow().iter().filter(|&&(start, _)| start == ring).map(|&(_, end)| end - ring).max().unwrap_or(0) } } @@ -116,12 +122,8 @@ impl Session for Lend { &self.0.words } - fn receive(&self) -> Box { - Box::new(Ring { region: self.0.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) - } - - fn send(&self) -> Box { - Box::new(Ring { region: self.0.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + fn memory(&self) -> Rc { + self.0.clone() } fn ring(&self, rings: Rings) { @@ -155,11 +157,12 @@ impl Client { let region = Rc::new(Region { words: Box::new(std::array::from_fn(|_| AtomicU32::new(0))), bytes: RefCell::new(vec![0; REGION_BYTES]), + committed: RefCell::new(vec![(0, RECEIVE_RING + RECEIVE_INITIAL as usize), (SEND_RING, SEND_RING + SEND_INITIAL as usize)]), holders: RefCell::new(Vec::new()), rang: Cell::new(false), dropped: Cell::new(false), }); - let halves = session::client(®ion.words, LAYOUT); + let halves = session::client(®ion.words, LAYOUT, SEND_INITIAL); (Self::holding(®ion, halves), Box::new(Lend(region))) } @@ -190,7 +193,9 @@ impl Client { pub fn read(&self, max: usize) -> Result, Refused> { let mut out = vec![0; max]; let region = &self.region; - let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), |span, at| { + let reach = |cap: u32| region.commit(RECEIVE_RING, cap as usize); + let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), reach, |span, at| { + assert!(region.reaches(span.offset, span.len), "a read of {span:?} past what was committed"); out[at..at + span.len].copy_from_slice(®ion.bytes.borrow()[span.offset..][..span.len]); })?; out.truncate(n as usize); @@ -212,7 +217,9 @@ impl Client { /// A write that does not wait: how much of `data` the send ring took. pub fn write(&self, data: &[u8]) -> Result { let region = &self.region; - let n = session::write(®ion.words, &self.tx, &self.ringing, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), |span, at| { + let reach = |cap: u32| region.commit(SEND_RING, cap as usize); + let n = session::write(®ion.words, &self.tx, &self.ringing, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), reach, |span, at| { + assert!(region.reaches(span.offset, span.len), "a write of {span:?} past what was committed"); region.bytes.borrow_mut()[span.offset..][..span.len].copy_from_slice(&data[at..at + span.len]); })?; Ok(n as usize) diff --git a/system/netstack/node/tests/listeners.rs b/system/netstack/node/tests/listeners.rs index 6d7a749606d..7de7e74bb8d 100644 --- a/system/netstack/node/tests/listeners.rs +++ b/system/netstack/node/tests/listeners.rs @@ -373,10 +373,11 @@ impl Net { self.pump(); } - /// The stream's client writes 100,000 bytes and leaves: [tcp] queues them all, its send + /// The stream's client fills its send ring and leaves: [tcp] queues it all, its send /// buffer being the client's ring, and finishes the connection alone. fn departs(&mut self, id: StreamId, client: &Client) { - assert_eq!(client.write(&[7u8; 100_000]), Ok(100_000)); + let ring = toyos::net::stream::SEND_INITIAL as usize; + assert_eq!(client.write(&vec![7u8; ring]), Ok(ring)); self.node.left(self.now, id); self.pump(); assert!(client.dropped()); diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index ad11de887b8..930cf3f1e56 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -596,7 +596,7 @@ fn a_deadline_moves_an_established_stream_with_no_opportunity_after_it() { #[test] fn a_write_larger_than_the_window_is_queued_whole_and_arrives_whole() { let (mut net, id, client) = established(); - let sent = text(100_000); + let sent = text(toyos::net::stream::SEND_INITIAL as usize); assert_eq!(client.write(&sent), Ok(sent.len()), "the send ring holds all of it"); net.node.bridge(net.now, [id]); net.pump(); @@ -729,7 +729,8 @@ fn room_released_rings_a_writer_over_a_full_send_ring_on_its_bell() { } net.bridge(); } - assert_eq!(written, toyos::net::stream::SEND_BYTES as usize, "the ring is full"); + // A deaf peer acknowledges nothing, so no byte is released and the ring stays as made. + assert_eq!(written, toyos::net::stream::SEND_INITIAL as usize, "the ring is full"); let bell = client.bell(); // The peer acknowledges every byte it heard of the node's. let heard = u32::try_from(net.far.segments.iter().map(|segment| segment.text.len()).sum::()).unwrap(); diff --git a/system/netstack/shard/tcp/src/conn.rs b/system/netstack/shard/tcp/src/conn.rs index f73b25bc400..2da30e7165a 100644 --- a/system/netstack/shard/tcp/src/conn.rs +++ b/system/netstack/shard/tcp/src/conn.rs @@ -1068,7 +1068,7 @@ impl Sync { if !(self.rx.lendable(rx.len()) && self.tx.buf.lendable(tx.len())) { return false; } - let ring = tx.len(); + let ring = tx.most(); self.rx.lend(rx); self.tx.buf.lend(tx); self.tx.buf.grow(ring); diff --git a/system/netstack/shard/tcp/src/ring.rs b/system/netstack/shard/tcp/src/ring.rs index 59accbd06d6..d834b1dc94e 100644 --- a/system/netstack/shard/tcp/src/ring.rs +++ b/system/netstack/shard/tcp/src/ring.rs @@ -1,21 +1,27 @@ //! A byte ring over storage of its own or a lent one. Its own grows, by doubling, with the //! furthest offset written and never past the capacity, so a connection holds memory for what it -//! holds and not for what it may. A lent one is a fixed length another party holds, bytes are -//! copied into and out of it and never borrowed from it, and a ring is lent at most once. The -//! capacity only rises, and never past a lent ring's length. Offsets count from the oldest byte -//! held. +//! holds and not for what it may. A lent one another party holds and lays out: bytes are copied +//! into and out of it by stream position, counted from the first byte it was lent, never +//! borrowed from it, and a ring is lent at most once. The capacity only rises, and holds no more +//! than a lent ring holds now. Offsets count from the oldest byte held. use alloc::boxed::Box; use alloc::vec::Vec; -/// Storage another party holds: a ring's bytes, reached only by copies. +/// Storage another party holds: a ring's bytes, reached only by copies, at stream positions +/// counted from the first byte it was lent, which wrap with `usize`. #[allow(clippy::len_without_is_empty, reason = "storage is never empty, and nothing asks")] pub trait Lent: core::fmt::Debug { - /// The bytes it holds. + /// The bytes it holds now from the oldest one held on. fn len(&self) -> usize; - /// Copies `data` in at `at`, which with `data`'s length lies within [`Self::len`]. + /// The most [`Self::len`] ever rises to. + fn most(&self) -> usize; + /// Its holder is asked to let it hold `bytes`; [`Self::len`] rises once it does. + fn want(&mut self, bytes: usize); + /// Copies `data` in at position `at`; the ring holds no more than [`Self::len`] from its + /// oldest byte. fn put(&mut self, at: usize, data: &[u8]); - /// Copies `out.len()` bytes out from `at`, which with that length lies within [`Self::len`]. + /// Copies `out.len()` bytes out from position `at`, as [`Self::put`] places them. fn take(&self, at: usize, out: &mut [u8]); } @@ -38,6 +44,7 @@ enum Storage { pub struct Ring { bytes: Storage, capacity: usize, + /// The oldest byte's index in its own storage, or its position in a lent one. head: usize, len: usize, } @@ -51,38 +58,43 @@ impl Ring { self.len } - pub const fn capacity(&self) -> usize { - self.capacity + /// The capacity, held to what a lent ring holds now. + pub fn capacity(&self) -> usize { + match &self.bytes { + Storage::Own(_) => self.capacity, + Storage::Lent(lent) => self.capacity.min(lent.len()), + } } - pub const fn room(&self) -> usize { - self.capacity.saturating_sub(self.len) + pub fn room(&self) -> usize { + self.capacity().saturating_sub(self.len) } - /// Raises the capacity to `capacity`, never past a lent ring's length; a lower one is no - /// change. + /// Raises the capacity to `capacity`, never past the most a lent ring holds, whose holder is + /// asked for it; a lower one is no change. pub fn grow(&mut self, capacity: usize) { - let capacity = match &self.bytes { + let capacity = match &mut self.bytes { Storage::Own(_) => capacity, - Storage::Lent(lent) => capacity.min(lent.len()), + Storage::Lent(lent) => { + let capacity = capacity.min(lent.most()); + lent.want(capacity); + capacity + } }; self.capacity = self.capacity.max(capacity); } - /// How many bytes the storage has. - fn stored(&self) -> usize { + /// The storage index of `offset`: below its own storage's length, or a lent ring's position. + fn at(&self, offset: usize) -> usize { match &self.bytes { - Storage::Own(bytes) => bytes.len(), - Storage::Lent(lent) => lent.len(), + Storage::Own(bytes) => { + let index = self.head.saturating_add(offset); + index.checked_sub(bytes.len()).unwrap_or(index) + } + Storage::Lent(_) => self.head.wrapping_add(offset), } } - /// The storage index of `offset`, which is below the storage's length. - fn at(&self, offset: usize) -> usize { - let index = self.head.saturating_add(offset); - index.checked_sub(self.stored()).unwrap_or(index) - } - /// Storage for every offset below `end`, at most the capacity: each byte stored keeps its /// offset, laid out again from the head. A lent ring has it already. fn reserve(&mut self, end: usize) { @@ -102,14 +114,14 @@ impl Ring { } /// The ring's bytes from here on are `lent`'s, at the offsets they had: everything stored is - /// copied in, the oldest byte held at its start, and the capacity is held to its length. - /// Only where [`Self::lendable`] says so. + /// copied in, the oldest byte held at position 0, and the capacity is held to the most it + /// holds. Only where [`Self::lendable`] says so. pub fn lend(&mut self, mut lent: Box) { let Storage::Own(bytes) = &self.bytes else { return }; let (newer, older) = bytes.split_at(self.head.min(bytes.len())); lent.put(0, older); lent.put(older.len(), newer); - self.capacity = self.capacity.min(lent.len()); + self.capacity = self.capacity.min(lent.most()); self.bytes = Storage::Lent(lent); self.head = 0; } @@ -123,9 +135,7 @@ impl Ring { first.saturating_add(copy(bytes, rest)) } Storage::Lent(lent) => { - let (first, rest) = data.split_at(lent.len().saturating_sub(start).min(data.len())); - lent.put(start, first); - lent.put(0, rest); + lent.put(start, data); data.len() } } @@ -133,7 +143,7 @@ impl Ring { /// Stores `data` at `offset` without counting it held; what lies past the capacity is not stored. pub fn write_at(&mut self, offset: usize, data: &[u8]) -> usize { - let fits = self.capacity.saturating_sub(offset).min(data.len()); + let fits = self.capacity().saturating_sub(offset).min(data.len()); if fits == 0 { return 0; } @@ -171,10 +181,10 @@ impl Ring { let offset = offset.min(self.len); let len = len.min(self.len.saturating_sub(offset)); let start = self.at(offset); - let first = len.min(self.stored().saturating_sub(start)); match &self.bytes { - Storage::Own(bytes) if first == len => bytes.get(start..start.saturating_add(len)).unwrap_or_default(), + Storage::Own(bytes) if len <= bytes.len().saturating_sub(start) => bytes.get(start..start.saturating_add(len)).unwrap_or_default(), Storage::Own(bytes) => { + let first = bytes.len().saturating_sub(start); scratch.clear(); scratch.extend_from_slice(bytes.get(start..start.saturating_add(first)).unwrap_or_default()); scratch.extend_from_slice(bytes.get(..len.saturating_sub(first)).unwrap_or_default()); @@ -183,9 +193,7 @@ impl Ring { Storage::Lent(lent) => { scratch.clear(); scratch.resize(len, 0); - let (to_first, to_rest) = scratch.split_at_mut(first); - lent.take(start, to_first); - lent.take(0, to_rest); + lent.take(start, scratch); scratch } } diff --git a/system/netstack/shard/tcp/src/rx.rs b/system/netstack/shard/tcp/src/rx.rs index cea9cfd41d1..a7f8b9cc8e4 100644 --- a/system/netstack/shard/tcp/src/rx.rs +++ b/system/netstack/shard/tcp/src/rx.rs @@ -461,9 +461,9 @@ impl Rx { /// Received text goes into `lent` from here on, everything held moved there at its offset: /// in-order text and out-of-order ranges, the FIN staying this side's state. The capacity - /// grows no further than its length. Only where [`Self::lendable`] says so. + /// grows no further than the most it holds, and holds what it holds now. Only where [`Self::lendable`] says so. pub fn lend(&mut self, lent: Box) { - self.max = self.max.min(lent.len()); + self.max = self.max.min(lent.most()); self.buf.lend(lent); } diff --git a/system/netstack/shard/tcp/tests/common/net.rs b/system/netstack/shard/tcp/tests/common/net.rs index 119340ea15b..30655689c8f 100644 --- a/system/netstack/shard/tcp/tests/common/net.rs +++ b/system/netstack/shard/tcp/tests/common/net.rs @@ -89,12 +89,26 @@ impl toyos_net_tcp::Lent for Shared { self.0.borrow().len() } + fn most(&self) -> usize { + self.0.borrow().len() + } + + fn want(&mut self, _: usize) {} + + /// A fixed ring: a position is its offset below the length. fn put(&mut self, at: usize, data: &[u8]) { - self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + let mut bytes = self.0.borrow_mut(); + let len = bytes.len(); + for (i, b) in data.iter().enumerate() { + bytes[(at + i) % len] = *b; + } } fn take(&self, at: usize, out: &mut [u8]) { - out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + let bytes = self.0.borrow(); + for (i, b) in out.iter_mut().enumerate() { + *b = bytes[(at + i) % bytes.len()]; + } } } diff --git a/system/netstack/shard/tcp/tests/lent.rs b/system/netstack/shard/tcp/tests/lent.rs index bac42bfc257..aeb806782ca 100644 --- a/system/netstack/shard/tcp/tests/lent.rs +++ b/system/netstack/shard/tcp/tests/lent.rs @@ -42,12 +42,26 @@ impl Lent for Shared { self.0.borrow().len() } + fn most(&self) -> usize { + self.0.borrow().len() + } + + fn want(&mut self, _: usize) {} + + /// A fixed ring: a position is its offset below the length. fn put(&mut self, at: usize, data: &[u8]) { - self.0.borrow_mut()[at..at + data.len()].copy_from_slice(data); + let mut bytes = self.0.borrow_mut(); + let len = bytes.len(); + for (i, b) in data.iter().enumerate() { + bytes[(at + i) % len] = *b; + } } fn take(&self, at: usize, out: &mut [u8]) { - out.copy_from_slice(&self.0.borrow()[at..at + out.len()]); + let bytes = self.0.borrow(); + for (i, b) in out.iter_mut().enumerate() { + *b = bytes[(at + i) % bytes.len()]; + } } } diff --git a/system/netstack/src/lent.rs b/system/netstack/src/lent.rs index d3c35e3fb55..ed153882d54 100644 --- a/system/netstack/src/lent.rs +++ b/system/netstack/src/lent.rs @@ -1,5 +1,5 @@ -//! A region a client lent, as the node's [`Session`]: its header as words, its two rings as -//! [tcp]'s buffers, and the stream's [`Holders`], every one of whose two bells a ring reaches. +//! A region a client lent, as the node's [`Session`]: its header as words, its bytes as the +//! node's [`Memory`], and the stream's [`Holders`], every one of whose two bells a ring reaches. //! //! **A region is judged before it is mapped** ([`SharedMemory::lent`]): a handle that is no //! region, one that is not memory the kernel allocated, and one that is not exactly @@ -19,47 +19,35 @@ use std::collections::BTreeMap; use std::rc::Rc; use toyos::ipc::FrameRx; -use toyos::net::stream::{Words, RECEIVE_BYTES, RECEIVE_RING, REGION_BYTES, SEND_BYTES, SEND_RING}; +use toyos::net::stream::{Words, REGION_BYTES}; use toyos::shm::{self, SharedMemory}; use toyos::transport::session::{Holder, Rings}; use toyos::wake::{Bell, Waker}; use toyos::{Connection, OwnedHandle}; use toyos_abi::syscall::SyscallError; -use toyos_net_node::Session; -use toyos_net_tcp::Lent; +use toyos_net_node::{Memory, Session}; -/// The region's mapping. +/// The region's mapping, as the node lays its rings out in it: a page is touched only once +/// [`Memory::commit`] made it memory, since a region is made with only its first rings'. struct Mapping(SharedMemory); -/// One of the region's rings, as [tcp]'s storage. -struct Ring { - mapping: Rc, - base: usize, - len: usize, -} - -impl std::fmt::Debug for Ring { +impl std::fmt::Debug for Mapping { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Ring({:#x}, {})", self.base, self.len) + f.write_str("Mapping") } } -impl Ring { - /// The address of `len` bytes at `at`, which [`Lent`]'s contract keeps inside the ring. +impl Mapping { + /// The address of `len` bytes at `at`, inside the region. fn at(&self, at: usize, len: usize) -> *mut u8 { - assert!(at.checked_add(len).is_some_and(|end| end <= self.len), "{len} bytes at {at} run past a ring of {}", self.len); - // SAFETY: the ring lies inside the mapping, which `mapping` keeps alive, and the - // assertion keeps the bytes inside the ring. - unsafe { self.mapping.0.as_ptr().add(self.base + at) } + assert!(at.checked_add(len).is_some_and(|end| end <= REGION_BYTES), "{len} bytes at {at} run past the region"); + // SAFETY: the assertion keeps the bytes inside the mapping, which `self` keeps alive. + unsafe { self.0.as_ptr().add(at) } } } -impl Lent for Ring { - fn len(&self) -> usize { - self.len - } - - fn put(&mut self, at: usize, data: &[u8]) { +impl Memory for Mapping { + fn put(&self, at: usize, data: &[u8]) { let to = self.at(at, data.len()); // SAFETY: `at` bounded the bytes inside the mapping; `data` is netstack's own memory // and never the region. @@ -71,6 +59,10 @@ impl Lent for Ring { // SAFETY: as in `put`, with the two sides swapped. unsafe { core::ptr::copy_nonoverlapping(from, out.as_mut_ptr(), out.len()) } } + + fn commit(&self, at: usize, len: usize) -> bool { + self.0.commit(at, len).is_ok() + } } /// One holder of a stream, as netstack keeps it. @@ -128,12 +120,8 @@ impl Session for Region { unsafe { &*(self.mapping.0.as_ptr() as *const Words) } } - fn receive(&self) -> Box { - Box::new(Ring { mapping: self.mapping.clone(), base: RECEIVE_RING, len: RECEIVE_BYTES as usize }) - } - - fn send(&self) -> Box { - Box::new(Ring { mapping: self.mapping.clone(), base: SEND_RING, len: SEND_BYTES as usize }) + fn memory(&self) -> Rc { + self.mapping.clone() } /// A ring into a bell its holder no longer holds is dropped: that From e1e016b687b1ee7013fc3e8aceb09fb290708710 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:21:01 +0100 Subject: [PATCH 02/16] stream region: the growth models, controls, tests and the commit guest test Co-Authored-By: Claude Opus 5.5 --- build/ci.rs | 2 + sdk/toyos/tests/transport_loom.rs | 100 ++++++++++++++++--- system/netstack/node/tests/streams.rs | 30 ++++++ tests/guest/rust-tests/src/bin/shm_commit.rs | 83 +++++++++++++++ 4 files changed, 202 insertions(+), 13 deletions(-) create mode 100644 tests/guest/rust-tests/src/bin/shm_commit.rs diff --git a/build/ci.rs b/build/ci.rs index 1e1a8e0343d..926577d6c66 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -832,6 +832,8 @@ pub(crate) const CONTROLS: &[Control] = &[ )]), red(SDK, "transport-no-resume", Some("transport_loom"), &[Fails("two_holders_read_each_byte_once")]), red(SDK, "transport-ungated-violation", Some("transport_loom"), &[Fails("looks_without_the_gate_lose_no_wake")]), + red(SDK, "transport-grow-room-now", Some("transport_loom"), &[Fails("a_ring_grown_while_it_holds_bytes_reads_each_as_written")]), + red(SDK, "transport-size-before-tail", Some("transport_loom"), &[Fails("a_ring_grown_while_it_holds_bytes_reads_each_as_written")]), red(SDK, "transport-end-keeps-inflight", None, &[Fails( "transport::inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing", )]), diff --git a/sdk/toyos/tests/transport_loom.rs b/sdk/toyos/tests/transport_loom.rs index ae82d0d8e71..0f1fd42f797 100644 --- a/sdk/toyos/tests/transport_loom.rs +++ b/sdk/toyos/tests/transport_loom.rs @@ -3,6 +3,8 @@ //! //! - **Publication**: a consumer that sees a tail sees every word of the //! entries below it. `transport-publish-relaxed` takes the edge away. +//! - **Growth**: a ring doubled while it holds unread bytes reads each byte as +//! written. `transport-grow-room-now` and `transport-size-before-tail` red it. //! - **A hostile peer** leaves the honest end with entries, nothing, or a //! named violation, and never more entries than the ring holds. //! `transport-no-clamp` believes the peer. @@ -162,15 +164,15 @@ fn a_hostile_consumer_yields_room_or_a_violation() { /// failure unwinds the model, which aborts the run before it says which test /// failed (`issues/loom-arc-panics-in-its-drop-while-a-failed-model-unwinds.md`). struct Stream { - page: [Shared; 4], + page: [Shared; 5], bytes: [Cell; RING as usize], to_consumer: Notify, to_producer: Notify, } const RING: u32 = 1; -const STREAM: stream::Place = stream::Place::new::<0, 1, 2, 3, 0>(); -type StreamEnds = (stream::Producer, stream::Consumer); +const STREAM: stream::Place = stream::Place::new::<0, 1, 2, 3, 4, 0>(); +type StreamEnds = (stream::Producer, stream::Consumer); fn stream() -> (Rc, StreamEnds) { let s = Rc::new(Stream { @@ -179,13 +181,13 @@ fn stream() -> (Rc, StreamEnds) { to_consumer: Notify::new(), to_producer: Notify::new(), }); - let ends = (stream::Producer::new(&s.page, STREAM), stream::Consumer::new(&s.page, STREAM)); + let ends = (stream::Producer::new(&s.page, STREAM, RING), stream::Consumer::new(&s.page, STREAM)); (s, ends) } /// Writes `byte` into the room the producer was last shown and publishes it, /// ringing a consumer that armed. -fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { +fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { for span in tx.span(0, 1).expect("a byte of room") { for at in span.offset..span.offset + span.len { s.bytes[at].set(byte); @@ -199,7 +201,7 @@ fn send(s: &Stream, tx: &mut stream::Producer, byte: u32) { /// Reads the byte the consumer was last shown and releases it, ringing a /// producer that armed. -fn receive(s: &Stream, rx: &mut stream::Consumer) -> u32 { +fn receive(s: &Stream, rx: &mut stream::Consumer) -> u32 { let mut read = Vec::new(); for span in rx.span(0, 1).expect("a byte ready") { read.extend((span.offset..span.offset + span.len).map(|at| s.bytes[at].get())); @@ -284,6 +286,78 @@ fn a_producer_that_looked_again_is_rung_for_room_it_missed() { }); } +/// A ring of at most four bytes, grown from two while the producer keeps +/// writing: no byte waits for it to empty, and every byte is read as written. +/// The consumer may hold two unread bytes at offsets 0 and 1 when the producer +/// grows, so the first byte past the switch, at offset 0 again, lands on an +/// unread one, a write beside the consumer's read, where the room is counted +/// in the new capacity at once (`transport-grow-room-now`); and a consumer +/// that loads the size before the tail meets a tail past what that size holds +/// (`transport-size-before-tail`). +#[test] +fn a_ring_grown_while_it_holds_bytes_reads_each_as_written() { + const MOST: u32 = 4; + const GROWN: stream::Place = stream::Place::new::<0, 1, 2, 3, 4, 0>(); + bounded(|| { + let page: Rc<[Shared; 5]> = Rc::new(core::array::from_fn(|_| Shared(AtomicU32::new(0)))); + let bytes: Rc<[Cell; MOST as usize]> = Rc::new(core::array::from_fn(|_| Cell::new(0))); + let (mut tx, mut rx) = (stream::Producer::new(&*page, GROWN, 2), stream::Consumer::new(&*page, GROWN)); + let put = |tx: &mut stream::Producer, bytes: &[Cell], at: u32| { + for span in tx.span(0, 1).expect("a byte of room") { + (span.offset..span.offset + span.len).for_each(|i| bytes[i].set(at)); + } + assert!(tx.advance(1)); + let _ = tx.publish(&*page); + }; + // Positions 0 and 1 go round once and are read, then 2 and 3 are + // written and left unread. + for at in 0..2 { + assert_eq!(tx.room(&*page), Ok(2 - at)); + put(&mut tx, &bytes[..], at); + } + assert_eq!(rx.ready(&*page), Ok(2)); + assert!(rx.advance(2)); + let _ = rx.release(&*page); + for at in 2..4 { + assert!(tx.room(&*page).expect("the consumer keeps the protocol") > 0); + put(&mut tx, &bytes[..], at); + } + let (producer_page, producer_bytes) = (Rc::clone(&page), Rc::clone(&bytes)); + let producer = loom::thread::spawn(move || { + // The consumer may already be reading: the switch is the old + // ring past whatever head this look finds. + tx.room(&*producer_page).expect("the consumer keeps the protocol"); + tx.grow(&*producer_page); + for at in 4..7 { + while tx.room(&*producer_page).expect("the consumer keeps the protocol") == 0 { + loom::thread::yield_now(); + } + for span in tx.span(0, 1).expect("a byte of room") { + (span.offset..span.offset + span.len).for_each(|i| producer_bytes[i].set(at)); + } + assert!(tx.advance(1)); + let _ = tx.publish(&*producer_page); + } + tx.capacity() + }); + let mut read = Vec::new(); + while read.len() < 5 { + match rx.ready(&*page).expect("the producer keeps the protocol") { + 0 => loom::thread::yield_now(), + _ => { + for span in rx.span(0, 1).expect("a byte ready") { + read.extend((span.offset..span.offset + span.len).map(|i| bytes[i].get())); + } + assert!(rx.advance(1)); + let _ = rx.release(&*page); + } + } + } + assert_eq!(producer.join().expect("the producer thread"), MOST); + assert_eq!(read, [2, 3, 4, 5, 6], "a byte was read other than as written"); + }); +} + // ---- a session: the client's sequences and the server's rings ---- // // The models below drive `session`'s own code: a holder's `read`, `write`, @@ -328,10 +402,10 @@ fn bounded_by(preemptions: usize, model: impl Fn() + Sync + Send + 'static) { } const BYTES: u32 = 2; -const SESSION_WORDS: usize = 14; +const SESSION_WORDS: usize = 16; const SESSION: Layout = Layout::new::<8, 9, 10, 11, 12, 13>( - stream::Place::new::<0, 1, 2, 3, 0>(), - stream::Place::new::<4, 5, 6, 7, { BYTES as usize }>(), + stream::Place::new::<0, 1, 2, 3, 14, 0>(), + stream::Place::new::<4, 5, 6, 7, 15, { BYTES as usize }>(), ); /// A failure the server stores in the receive ring's `end`. const RESET: u32 = 2; @@ -563,9 +637,9 @@ impl Session { /// A session whose end has `holders` holders, and its server end. fn new(holders: u16) -> (Rc, End) { let page: [Shared; SESSION_WORDS] = core::array::from_fn(|_| Shared(AtomicU32::new(0))); - let mut all = vec![Holding::new(session::client(&page, SESSION))]; + let mut all = vec![Holding::new(session::client(&page, SESSION, BYTES))]; all.extend((2..=holders).map(|n| Holding::new(session::holder(SESSION, Holder::new(n).unwrap())))); - let server = Server::new(&page, SESSION); + let server = Server::new(&page, SESSION, BYTES); let s = Session { page, bytes: core::array::from_fn(|_| Cell::new(0)), @@ -602,7 +676,7 @@ impl Session { /// A holder reads at most `max` bytes over `page`: the bytes it read. fn read_by(&self, page: &[W; SESSION_WORDS], by: &By, mode: Mode<()>, max: u32) -> Result, Refused> { let mut read = Vec::new(); - session::read(page, &self.holders[by.at].rx, by, mode, max, |span, _| { + session::read(page, &self.holders[by.at].rx, by, mode, max, |_| true, |span, _| { read.extend((span.offset..span.offset + span.len).map(|at| self.bytes[at].get())); })?; Ok(read) @@ -610,7 +684,7 @@ impl Session { /// A holder writes `byte` over `page`, once at most. fn write_by(&self, page: &[W; SESSION_WORDS], by: &By, mode: Mode<()>, byte: u32) -> Result { - session::write(page, &self.holders[by.at].tx, by, mode, 1, |span, _| { + session::write(page, &self.holders[by.at].tx, by, mode, 1, |_| true, |span, _| { for at in span.offset..span.offset + span.len { if lives(by.fate) { self.bytes[at].set(byte); diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 930cf3f1e56..4c349a3c0b6 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -1132,3 +1132,33 @@ fn the_last_holder_leaving_is_the_streams_close() { assert!(net.far.fin, "the last holder left"); assert_eq!((first.dropped(), net.node.streams()), (true, 0)); } + +// ---- the rings grow under load ---- + +// A client that writes more than its send ring holds doubles the ring at each write the room +// cannot take whole, to its place, and netstack reads the grown ring by the size the client +// stored: the peer's stream is the client's, byte for byte. +#[test] +fn an_upload_larger_than_its_send_ring_grows_it_to_its_place() { + use toyos::net::stream::{SEND_BYTES, SEND_RING}; + const PERIOD: usize = 251; + let byte = |i: usize| (i % PERIOD) as u8; + const WRITE: usize = 2 * SEND_BYTES as usize; + let pattern: Vec = (0..WRITE + PERIOD).map(byte).collect(); + let (mut net, _, client) = established(); + let mut offset = 0; + for _ in 0..1_000 { + if client.region.committed(SEND_RING) == SEND_BYTES as usize { + break; + } + match client.write(&pattern[offset % PERIOD..][..WRITE]) { + Ok(n) => offset += n, + Err(Refused::WouldBlock) => {} + Err(other) => panic!("{other:?}"), + } + net.bridge(); + } + assert_eq!(client.region.committed(SEND_RING), SEND_BYTES as usize, "the send ring after {offset} bytes"); + let heard = &net.far.received; + assert!(!heard.is_empty() && heard.iter().enumerate().all(|(i, &b)| b == byte(i)), "the peer's stream is the client's"); +} diff --git a/tests/guest/rust-tests/src/bin/shm_commit.rs b/tests/guest/rust-tests/src/bin/shm_commit.rs new file mode 100644 index 00000000000..68b7c0b50be --- /dev/null +++ b/tests/guest/rust-tests/src/bin/shm_commit.rs @@ -0,0 +1,83 @@ +//! A region made with only its first page committed is memory there and +//! nowhere else until `shm_commit` makes more of it: a touch of a page past +//! it ends the process, a commit makes the page zeroed memory every mapping +//! sees, a second commit of it changes nothing, and `fstat` says how much is +//! committed. +//! +//! It needs the guest: a fault that never allocates, and a commit that a +//! mapping made before it sees, are the kernel's page tables and frames. + +use std::process::{Command, Stdio}; + +use toyos_abi::syscall::{self, SyscallError, PAGE_SIZE}; + +const ME: &str = "/system/bin/test_rs_shm_commit"; +const PAGES: usize = 4; + +fn main() { + if std::env::args().nth(1).as_deref() == Some("touch") { + return touch(); + } + let shm = syscall::shm_reserve(PAGES * PAGE_SIZE, PAGE_SIZE).expect("a region of four pages, one committed"); + let committed = |what: &str| { + let stat = syscall::fstat(shm).expect("fstat of the region"); + assert_eq!(stat.size, (PAGES * PAGE_SIZE) as u64, "{what}: the size is the whole region"); + stat.committed + }; + assert_eq!(committed("made"), PAGE_SIZE as u64); + // SAFETY: a fresh region, mapped whole, which nothing else names. + let at = unsafe { syscall::shm_map(shm) }.expect("map the region"); + // SAFETY: the first page is committed. + unsafe { + assert_eq!(at.read_volatile(), 0, "a committed page is zeroed"); + at.write_volatile(0x5A); + } + println!(" PASS: a region made with one page committed is memory there"); + + dies(); + + syscall::shm_commit(shm, PAGE_SIZE + 100, 2 * PAGE_SIZE).expect("commit what pages 1 to 3 are touched by"); + assert_eq!(committed("committed"), 4 * PAGE_SIZE as u64, "every page the range touched"); + // SAFETY: pages 1 to 3 are committed now, and the mapping was made before. + unsafe { + for page in 1..PAGES { + let byte = at.add(page * PAGE_SIZE + 7); + assert_eq!(byte.read_volatile(), 0, "page {page} is zeroed memory"); + byte.write_volatile(page as u8); + } + } + syscall::shm_commit(shm, 0, PAGES * PAGE_SIZE).expect("a commit of what is committed"); + // SAFETY: as above. + unsafe { + assert_eq!(at.read_volatile(), 0x5A, "a second commit kept page 0"); + assert_eq!(at.add(3 * PAGE_SIZE + 7).read_volatile(), 3, "a second commit kept page 3"); + } + println!(" PASS: a commit makes zeroed memory a mapping made before sees, and a second keeps it"); + + assert_eq!(syscall::shm_commit(shm, 0, 0), Err(SyscallError::InvalidArgument), "no bytes"); + assert_eq!(syscall::shm_commit(shm, PAGES * PAGE_SIZE, 1), Err(SyscallError::InvalidArgument), "past the region"); + assert_eq!(syscall::shm_reserve(PAGE_SIZE, 2 * PAGE_SIZE), Err(SyscallError::InvalidArgument), "more committed than made"); + println!(" PASS: a commit of no bytes or past the region, and a reservation committing more than it makes, are refused"); + println!("shm_commit: committed pages are memory, and only they"); +} + +/// A child touches page 1 of a region committed only on page 0, and dies of it. +fn dies() { + let child = Command::new(ME).arg("touch").stdout(Stdio::piped()).spawn().expect("spawn the toucher"); + let out = child.wait_with_output().expect("wait for the toucher"); + let said = String::from_utf8_lossy(&out.stdout); + assert!(said.contains("armed"), "the child never reached the touch, so it proved nothing:\n{said}"); + assert!(!said.contains("SURVIVED"), "a touch past what was committed was served:\n{said}"); + assert!(!out.status.success(), "a touch past what was committed did not end the process ({:?})", out.status.code()); + println!(" PASS: a touch of a page not committed ends the process"); +} + +fn touch() { + let shm = syscall::shm_reserve(2 * PAGE_SIZE, PAGE_SIZE).expect("a region of two pages, one committed"); + // SAFETY: a fresh region, mapped whole; page 1 is past what is committed, which is the test. + let at = unsafe { syscall::shm_map(shm) }.expect("map the region"); + println!("armed"); + // SAFETY: as above. + let byte = unsafe { at.add(PAGE_SIZE).read_volatile() }; + println!("SURVIVED with {byte}"); +} From a140dcdbdb0e55f3d42d78a0d0f82d34c8e6c739 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 04:44:48 +0100 Subject: [PATCH 03/16] stream region: the issue record keeps what is left, and the node's pass reads cleanly Co-Authored-By: Claude Opus 5.5 --- issues/a-process-holds-at-most-128-streams.md | 23 ++++++++++++++++ ...eight-mebibytes-and-a-process-holds-128.md | 27 ------------------- system/netstack/node/src/streams.rs | 13 ++++----- 3 files changed, 30 insertions(+), 33 deletions(-) create mode 100644 issues/a-process-holds-at-most-128-streams.md delete mode 100644 issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md diff --git a/issues/a-process-holds-at-most-128-streams.md b/issues/a-process-holds-at-most-128-streams.md new file mode 100644 index 00000000000..5de2986611a --- /dev/null +++ b/issues/a-process-holds-at-most-128-streams.md @@ -0,0 +1,23 @@ +--- +status: open +kind: defect +opened: 2026-10-14 +--- + +# A process holds at most 128 streams + +A stream's region now commits its header and 64 KiB of each ring at the +connect, and each ring's producer doubles it as it needs more +(`toyos::transport::stream`), so a stream commits what it holds. + +The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 +handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` +allocates nothing: past it a connect or an accept is refused +`ResourceExhausted` before anything is made. 128 is libc's `MAX_SOCKETS`, an +estimate no measurement set, and a new limit for a std program. + +**Exit condition**: the table grown on demand, or its bound read off a +measured program's need. + +**Owner**: the orchestrator, who stages +`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`. diff --git a/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md b/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md deleted file mode 100644 index 5840bb152e0..00000000000 --- a/issues/a-stream-commits-eight-mebibytes-and-a-process-holds-128.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-14 ---- - -# A stream commits 8 MiB at its connect, and a process holds at most 128 - -Every connect and accept makes its region whole before a byte moves: -`toyos::net::stream::REGION_BYTES`, four 2 MiB pages from `SYS_SHM_CREATE`, -physically contiguous, where the pipes it replaced took a page on first write -and [tcp]'s rings grew as text was held. A program that opens many short -connections pays 8 MiB for each, and a fragmented machine can refuse a -connect a pipe would have taken. #863 accepted the cost with the design. - -The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 -handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` -allocates nothing: past it a connect or an accept is refused -`ResourceExhausted` before anything is made. 128 is libc's `MAX_SOCKETS`, an -estimate no measurement set, and a new limit for a std program. - -**Exit condition**: regions whose pages are allocated as the rings fill, so a -stream commits what it holds; and the table grown on demand, or its bound -read off a measured program's need. - -**Owner**: the orchestrator, who stages -`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`. diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index b8576485449..b7679ef0940 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -80,12 +80,15 @@ struct Shape { wanted: usize, } +/// A ring's shape, shared by the pass and [tcp]'s storage. +type Shared = Rc>; + /// One of the region's rings, as [tcp]'s storage. #[derive(Debug)] struct LentRing { memory: Rc, place: Place, - shape: Rc>, + shape: Shared, } impl LentRing { @@ -196,7 +199,7 @@ struct Stream { conn: ConnId, session: Box, /// The receive ring's and the send ring's shapes, as [tcp]'s storage reads them. - shapes: Option<(Rc>, Rc>)>, + shapes: Option<(Shared, Shared)>, /// What the next pass ends the stream with. broken: Option, /// The session's server end, once the connection is established and its buffers are the @@ -321,10 +324,8 @@ impl Stream { let sent = server.sent(words)?; let tx = tx_shape.get(); let cap = server.send_size_cap(); - if cap > tx.holds { - if !self.session.memory().commit(SEND_RING, usize::try_from(cap).unwrap_or(usize::MAX)) { - return Err(Violation::Size); - } + if cap > tx.holds && !self.session.memory().commit(SEND_RING, usize::try_from(cap).unwrap_or(usize::MAX)) { + return Err(Violation::Size); } tx_shape.set(Shape { size: server.send_size(), holds: cap.max(tx.holds), ..tx }); let fresh = sent.checked_sub(queued).ok_or(Violation::TailPastDepth)?; From 9f250dc79c0539725159893d844537fe5d146e15 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 05:06:21 +0100 Subject: [PATCH 04/16] stream region: the guest's own netstack servers make their receive ring's first size Co-Authored-By: Claude Opus 5.5 --- tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs | 4 ++-- tests/guest/rust-tests/src/bin/stream_handles.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs index 075d974a3a0..dfc6a14e667 100644 --- a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs +++ b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs @@ -31,7 +31,7 @@ use std::io::{BufRead, BufReader, Write}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicI32, Ordering}; -use toyos::net::stream::{Server, Words, LAYOUT, RECEIVE_RING, REGION_BYTES}; +use toyos::net::stream::{Server, Words, LAYOUT, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES}; use toyos::wake::{self, Hungup}; use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; use toyos::shm::SharedMemory; @@ -135,7 +135,7 @@ fn test(role: &str) { let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); // SAFETY: the mapping is `REGION_BYTES`, 2 MiB aligned, and its header is reached only as atomics. let words = unsafe { &*(region.as_ptr() as *const Words) }; - let mut server = Server::new(words, LAYOUT); + let mut server = Server::new(words, LAYOUT, RECEIVE_INITIAL); let (receive, receive_bell) = wake::pair().expect("the receive bell"); let (_send, send_bell) = wake::pair().expect("the send bell"); let (rung_waker, rung) = wake::pair().expect("the rung bell"); diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs index a1b2e46754b..fcadfec7e45 100644 --- a/tests/guest/rust-tests/src/bin/stream_handles.rs +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -52,7 +52,7 @@ use std::process::{Child, ChildStdout, Command, Stdio}; use std::sync::atomic::Ordering; use toyos::ipc::{FrameRx, IpcError, RxStep}; -use toyos::net::stream::{Adopt, Server, Words, ADOPT, EXPORT, EXPORTED, LAYOUT, RECEIVE_RING, REGION_BYTES, SEND_RING}; +use toyos::net::stream::{Adopt, Server, Words, ADOPT, EXPORT, EXPORTED, LAYOUT, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_RING}; use toyos::net::{MsgType, RespType, TcpConnectRequest, TcpConnectResponse}; use toyos::poller::{Poller, READABLE}; use toyos::shm::SharedMemory; @@ -533,7 +533,7 @@ fn run_answering(role: &str, peer: &[u8], local: [u8; 4]) -> (Child, Netstack) { let _: TcpConnectRequest = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the connect's request"); let [region] = rx.take_handles::<1>().expect("the stream's region"); let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); - let mut server = Server::new(words_of(®ion), LAYOUT); + let mut server = Server::new(words_of(®ion), LAYOUT, RECEIVE_INITIAL); region.bytes().write(RECEIVE_RING, peer); let _ = server.publish(words_of(®ion), peer.len() as u32).expect("the client keeps the protocol"); let (handed, (receive, send, rung)) = bells(); From 3d84dad81cf5799146328fc91a08d53eadd64859 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 05:11:37 +0100 Subject: [PATCH 05/16] stream region: the growth model borrows its words as the others do Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/tests/transport_loom.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/toyos/tests/transport_loom.rs b/sdk/toyos/tests/transport_loom.rs index 0f1fd42f797..e9c6c0555c7 100644 --- a/sdk/toyos/tests/transport_loom.rs +++ b/sdk/toyos/tests/transport_loom.rs @@ -301,55 +301,55 @@ fn a_ring_grown_while_it_holds_bytes_reads_each_as_written() { bounded(|| { let page: Rc<[Shared; 5]> = Rc::new(core::array::from_fn(|_| Shared(AtomicU32::new(0)))); let bytes: Rc<[Cell; MOST as usize]> = Rc::new(core::array::from_fn(|_| Cell::new(0))); - let (mut tx, mut rx) = (stream::Producer::new(&*page, GROWN, 2), stream::Consumer::new(&*page, GROWN)); + let (mut tx, mut rx) = (stream::Producer::new(&page, GROWN, 2), stream::Consumer::new(&page, GROWN)); let put = |tx: &mut stream::Producer, bytes: &[Cell], at: u32| { for span in tx.span(0, 1).expect("a byte of room") { (span.offset..span.offset + span.len).for_each(|i| bytes[i].set(at)); } assert!(tx.advance(1)); - let _ = tx.publish(&*page); + let _ = tx.publish(&page); }; // Positions 0 and 1 go round once and are read, then 2 and 3 are // written and left unread. for at in 0..2 { - assert_eq!(tx.room(&*page), Ok(2 - at)); + assert_eq!(tx.room(&page), Ok(2 - at)); put(&mut tx, &bytes[..], at); } - assert_eq!(rx.ready(&*page), Ok(2)); + assert_eq!(rx.ready(&page), Ok(2)); assert!(rx.advance(2)); - let _ = rx.release(&*page); + let _ = rx.release(&page); for at in 2..4 { - assert!(tx.room(&*page).expect("the consumer keeps the protocol") > 0); + assert!(tx.room(&page).expect("the consumer keeps the protocol") > 0); put(&mut tx, &bytes[..], at); } let (producer_page, producer_bytes) = (Rc::clone(&page), Rc::clone(&bytes)); let producer = loom::thread::spawn(move || { // The consumer may already be reading: the switch is the old // ring past whatever head this look finds. - tx.room(&*producer_page).expect("the consumer keeps the protocol"); - tx.grow(&*producer_page); + tx.room(&producer_page).expect("the consumer keeps the protocol"); + tx.grow(&producer_page); for at in 4..7 { - while tx.room(&*producer_page).expect("the consumer keeps the protocol") == 0 { + while tx.room(&producer_page).expect("the consumer keeps the protocol") == 0 { loom::thread::yield_now(); } for span in tx.span(0, 1).expect("a byte of room") { (span.offset..span.offset + span.len).for_each(|i| producer_bytes[i].set(at)); } assert!(tx.advance(1)); - let _ = tx.publish(&*producer_page); + let _ = tx.publish(&producer_page); } tx.capacity() }); let mut read = Vec::new(); while read.len() < 5 { - match rx.ready(&*page).expect("the producer keeps the protocol") { + match rx.ready(&page).expect("the producer keeps the protocol") { 0 => loom::thread::yield_now(), _ => { for span in rx.span(0, 1).expect("a byte ready") { read.extend((span.offset..span.offset + span.len).map(|i| bytes[i].get())); } assert!(rx.advance(1)); - let _ = rx.release(&*page); + let _ = rx.release(&page); } } } From 928fd0c58920a208ab9d2babceaeba385f7a7b41 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 05:50:00 +0100 Subject: [PATCH 06/16] review r1: no uncommitted page reaches the kernel's reads, a syscall's window or netstack B1: SharedMemObject::read answers zeros for a page not committed, as the module promised; a program image spawned or opened from a region with pages not committed is refused by its loader rather than stopping the machine. A kernel write still panics there: every writer commits its object whole. B3: a syscall's window over an object's page not committed is BadAddress (Space::allows asks the object), where the copy used to drop the bytes and answer success; the fault path already refused the same access. B2, S2: netstack commits the header and both rings' first sizes where it judges a region, charged to the client, and refuses it there if it cannot; the node's deferred `broken` state is gone. B4: a span the pass's own size cannot lay out is unreachable, said with its arguments, not a silent no-op. S1: the ring's oracle runs across the u32 wrap with growth. S3: fstat's `committed` is gone: readers commit rather than ask. S4: a write that fills the send ring grows it too. S5, B6: shm_commit reads the job's account around each commit and the close, covers B1 and B3, and runs on QEMU as shm_commit_alone. A client's region with nothing committed is driven at real netstack in netcase's boot (stream_region_refused). B5: a ring that never shrinks is recorded with its owner and exit. Co-Authored-By: Claude Opus 5.5 --- issues/a-process-holds-at-most-128-streams.md | 4 - issues/a-stream-ring-never-shrinks.md | 23 +++++ kernel/src/mm/space.rs | 18 +++- kernel/src/object/ops.rs | 5 +- kernel/src/object/shm.rs | 37 +++----- kernel/src/vfs.rs | 2 +- sdk/abi/src/syscall.rs | 9 +- sdk/toyos/src/transport/session.rs | 8 +- sdk/toyos/src/transport/stream.rs | 53 ++++++++++++ system/netstack/node/src/streams.rs | 32 +++---- system/netstack/src/lent.rs | 12 ++- tests/guest/rust-tests/src/bin/shm_commit.rs | 85 ++++++++++++++++--- .../src/bin/stream_region_refused.rs | 34 ++++++++ tests/toyos.rs | 41 ++++++++- 14 files changed, 282 insertions(+), 81 deletions(-) create mode 100644 issues/a-stream-ring-never-shrinks.md create mode 100644 tests/guest/rust-tests/src/bin/stream_region_refused.rs diff --git a/issues/a-process-holds-at-most-128-streams.md b/issues/a-process-holds-at-most-128-streams.md index 5de2986611a..d1cd512164c 100644 --- a/issues/a-process-holds-at-most-128-streams.md +++ b/issues/a-process-holds-at-most-128-streams.md @@ -6,10 +6,6 @@ opened: 2026-10-14 # A process holds at most 128 streams -A stream's region now commits its header and 64 KiB of each ring at the -connect, and each ring's producer doubles it as it needs more -(`toyos::transport::stream`), so a stream commits what it holds. - The process's ends are a fixed table, `MAX_STREAMS` = 128 streams and 256 handles of their connections (`sdk/toyos/src/net/stream.rs`), since `toyos` allocates nothing: past it a connect or an accept is refused diff --git a/issues/a-stream-ring-never-shrinks.md b/issues/a-stream-ring-never-shrinks.md new file mode 100644 index 00000000000..7acef1095db --- /dev/null +++ b/issues/a-stream-ring-never-shrinks.md @@ -0,0 +1,23 @@ +--- +status: open +kind: defect +opened: 2026-10-28 +--- + +# A stream's ring never shrinks + +A stream region commits its header and 64 KiB of each ring at the connect, +and each ring's producer doubles it, up to 4 MiB, when it needs more +(`toyos::transport::stream`). Nothing gives a page back: a long-lived stream +that once carried a burst keeps the most it ever held, up to 8 MiB + 4 KiB, +for as long as it lives, counted to its client's job. netstack prices each +place at that most (`PLACE_BYTES`, `system/netstack/src/main.rs`). + +Owner: the orchestrator, who assigns it after #1077 lands. + +Exit: a ring whose stream has gone idle gives back what it no longer holds, +at a quiescent point both ends agree on (both cursors equal, under the +session's gates), through a kernel call that decommits pages of a memory +object; a host test of the ring and a loom model of the shrink beside a +growth and a write, and a guest test that reads the client's account fall +after an idle stream's shrink. diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index b229a5286fe..ffad7153008 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -852,8 +852,22 @@ impl AddressSpace { /// what a copy the kernel makes for a thread asks first. A device's /// registers or firmware's framebuffer, an aperture, are no buffer. pub fn allows(&self, start: u64, end: u64, access: Access) -> bool { - self.inventory - .covered_where(start & !(PAGE_SIZE - 1), end, |entry| entry.prot.allows(access) && entry.backed() && copyable(entry)) + let start = start & !(PAGE_SIZE - 1); + self.inventory.covered_where(start, end, |entry| entry.prot.allows(access) && entry.backed() && copyable(entry)) + && self.committed(start, end) + } + + /// Whether every page of `[start, end)` an object maps is one its + /// object committed: a copy into one that is not would be lost, and a + /// fault on it is refused. + fn committed(&self, start: u64, end: u64) -> bool { + self.inventory.meeting(start, end).all(|(at, entry)| match &entry.kind { + Kind::Object { object, offset, .. } => { + let (lo, hi) = (start.max(at), end.min(entry.end)); + (lo..hi).step_by(PAGE_SIZE as usize).all(|page| object.page(offset + (page - at)).is_some()) + } + Kind::Private { .. } | Kind::Copy { .. } => true, + }) } /// The frame and protection a present leaf maps `va` with, populating diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 1e74c802f1c..286b2cae8e6 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -645,7 +645,6 @@ toyos_abi::user_safe! { pub mtime: u64, pub volume: u64, pub node: u64, - pub committed: u64, } } @@ -654,7 +653,7 @@ const _: () = assert!(core::mem::size_of::() == core::mem::size_of:: Stat { - let plain = |t: FileType| Stat { file_type: t as u64, size: 0, mtime: 0, volume: 0, node: 0, committed: 0 }; + let plain = |t: FileType| Stat { file_type: t as u64, size: 0, mtime: 0, volume: 0, node: 0 }; match object { KObjectRef::File(f) => f.with(|state| Stat { file_type: FileType::File as u64, @@ -662,7 +661,6 @@ pub fn fstat(object: &KObjectRef) -> Stat { mtime: state.mtime, volume: state.volume, node: state.file_id, - committed: 0, }), KObjectRef::PipeRead(r) => { plain(if r.is_tty() { FileType::Tty } else { FileType::Pipe }) @@ -687,7 +685,6 @@ pub fn fstat(object: &KObjectRef) -> Stat { mtime: 0, volume: 0, node: 0, - committed: m.committed(), }, KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index dc33eee4802..d0de724df06 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -15,8 +15,9 @@ //! commits them: `SYS_SHM_CREATE`, all or a first part of them, then //! `SYS_SHM_COMMIT`, which any holder that may map it calls. A committed //! page is the object's to its end. A fault on a page not committed is -//! refused as an access outside every range is, and a copy reads it as -//! zeros. +//! refused as an access outside every range is, a syscall's window over one +//! is `BadAddress`, and the kernel's own read of one ([`SharedMemObject::read`], +//! a program image's loader among its callers) answers zeros. use alloc::boxed::Box; use alloc::sync::{Arc, Weak}; @@ -151,20 +152,6 @@ impl SharedMemObject { Ok(()) } - /// The bytes of it that are committed. - pub fn committed(&self) -> u64 { - match &self.backing { - Backing::Frames(pieces) => { - let pages = pieces.iter().flat_map(|piece| piece.iter()).filter(|word| word.load(Ordering::Relaxed) != 0).count(); - pages as u64 * PAGE_SIZE - } - Backing::Sub { whole, offset, len } => { - (0..len.div_ceil(PAGE_SIZE)).filter(|page| whole.page(offset + page * PAGE_SIZE).is_some()).count() as u64 * PAGE_SIZE - } - Backing::Run(_) | Backing::Boot { .. } | Backing::Aperture { .. } => self.size, - } - } - /// `size` bytes of zeroed memory in whole pages, over one run of whole /// 2 MiB pages: what a device can be lent, since a domain maps 2 MiB at a /// time and a device with no domain reaches memory by its physical address. @@ -287,6 +274,7 @@ impl SharedMemObject { pub fn write(&self, offset: u64, bytes: &[u8]) { assert!(self.is_ram(), "shm koid {}: the kernel writes no aperture", self.core.koid().raw()); self.each_piece(offset, bytes.len(), |phys, at, n| { + let phys = phys.unwrap_or_else(|| panic!("shm koid {}: the kernel wrote {:#x}, not committed", self.core.koid().raw(), offset + at as u64)); // SAFETY: `each_piece` bounds `[at, at + n)` inside `bytes` and // `phys..+n` inside one of this object's own pages, which the // direct map covers; no reference forms over a page a process may @@ -295,19 +283,23 @@ impl SharedMemObject { }); } - /// Copy out `out.len()` bytes at `offset`, each read once. + /// Copy out `out.len()` bytes at `offset`, each read once: a page not + /// committed reads as zeros. pub fn read(&self, offset: u64, out: &mut [u8]) { assert!(self.is_ordinary(), "shm koid {}: the kernel reads no aperture", self.core.koid().raw()); let into = out.as_mut_ptr(); - self.each_piece(offset, out.len(), |phys, at, n| { + self.each_piece(offset, out.len(), |phys, at, n| match phys { // SAFETY: as `write`, mirrored; `out` is the caller's own. - unsafe { core::ptr::copy_nonoverlapping(DirectMap::from_phys(phys).as_ptr(), into.add(at), n) } + Some(phys) => unsafe { core::ptr::copy_nonoverlapping(DirectMap::from_phys(phys).as_ptr(), into.add(at), n) }, + // SAFETY: `each_piece` bounds `[at, at + n)` inside `out`. + None => unsafe { core::ptr::write_bytes(into.add(at), 0, n) }, }); } /// `f(phys, at, n)` for each piece of `[offset, offset + len)` inside one - /// page, `at` its offset from `offset`. - fn each_piece(&self, offset: u64, len: usize, mut f: impl FnMut(u64, usize, usize)) { + /// page, `at` its offset from `offset`; `phys` is `None` for a page not + /// committed. + fn each_piece(&self, offset: u64, len: usize, mut f: impl FnMut(Option, usize, usize)) { assert!( offset.checked_add(len as u64).is_some_and(|end| end <= self.size), "shm koid {}: {len} bytes at {offset:#x} of {:#x}", @@ -318,8 +310,7 @@ impl SharedMemObject { while done < len { let at = offset + done as u64; let n = ((PAGE_SIZE - at % PAGE_SIZE) as usize).min(len - done); - let page = self.page(at).unwrap_or_else(|| panic!("shm koid {}: the kernel reached {at:#x}, not committed", self.core.koid().raw())); - f(page + at % PAGE_SIZE, done, n); + f(self.page(at).map(|page| page + at % PAGE_SIZE), done, n); done += n; } } diff --git a/kernel/src/vfs.rs b/kernel/src/vfs.rs index 6b0bbac59c3..8f98cf5c2db 100644 --- a/kernel/src/vfs.rs +++ b/kernel/src/vfs.rs @@ -488,7 +488,7 @@ impl Vfs { pub fn stat(&mut self, path: &str, follow: bool) -> Result { let (mount, file) = if follow { self.followed(path)? } else { self.resolve_path("/", path) }; let named = |file_type: FileType, size, mtime, volume, node| { - Stat { file_type: file_type as u64, size, mtime, volume, node, committed: 0 } + Stat { file_type: file_type as u64, size, mtime, volume, node } }; if mount.is_empty() { return Ok(named(FileType::Directory, 0, 0, NAMESPACE_VOLUME, ROOT_NODE)); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index c2fd05c2299..f22278ba1fd 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -834,9 +834,6 @@ pub struct Stat { /// reaches it, for as long as it exists and its volume stays mounted, and /// no other file's while both exist. 0 exactly when `volume` is. pub node: u64, - /// A memory object's bytes that are committed ([`shm_commit`]): its - /// `size` but for a region made with fewer; 0 for anything else. - pub committed: u64, } #[cfg(target_arch = "x86_64")] @@ -2327,8 +2324,10 @@ pub fn shm_reserve(size: usize, committed: usize) -> Result Result<(), SyscallError> { diff --git a/sdk/toyos/src/transport/session.rs b/sdk/toyos/src/transport/session.rs index c268aaa01ce..8a7c39608be 100644 --- a/sdk/toyos/src/transport/session.rs +++ b/sdk/toyos/src/transport/session.rs @@ -878,9 +878,9 @@ where /// into the caller's buffer: the count, at least one. A shutdown, the server's /// hang-up or a failure it stored is answered, and nothing copied, whatever /// room there is. A write of no bytes answers 0 at once. A write the room -/// cannot take whole doubles the ring, up to its place, where `reach` makes -/// the ring's first twice its capacity memory; where it cannot, the ring -/// stays as it is. +/// cannot take whole, or that fills the ring, doubles the ring, up to its +/// place, where `reach` makes the ring's first twice its capacity memory; +/// where it cannot, the ring stays as it is. pub fn write( page: &[W; N], lock: &L, @@ -915,7 +915,7 @@ where if !half.ring.advance(n) { return (Err(Refused::Violation(Violation::HeadPastTail)), false); } - if n < len && half.ring.can_grow() { + if (n < len || n == room) && half.ring.can_grow() { let cap = half.ring.capacity().saturating_mul(2); if half.reached >= cap || reach(cap) { half.reached = half.reached.max(cap); diff --git a/sdk/toyos/src/transport/stream.rs b/sdk/toyos/src/transport/stream.rs index 1b976f450db..49eac5f838e 100644 --- a/sdk/toyos/src/transport/stream.rs +++ b/sdk/toyos/src/transport/stream.rs @@ -665,6 +665,59 @@ mod tests { } } + /// Both ends resumed from a position just short of 2^32, so every + /// growth's switch and its 24-bit reconstruction cross the wrap. + fn agrees_across_the_wrap(start: u32, seed: u64) { + let page = page(); + let from = u32::MAX - 3 * C; + let (mut tx, mut rx) = (Producer::new(&page, place::(), start), Consumer::new(&page, place::())); + page[0].store(from, Ordering::Release); + page[1].store(from, Ordering::Release); + assert_eq!((tx.resume(&page), rx.resume(&page)), (Ok(()), Ok(()))); + let mut region = region::(); + let (mut written, mut published, mut read, mut released) = (0u64, 0u64, 0u64, 0u64); + let mut model = Model { cap: start, switch: None }; + let mut rng = Rng(seed); + for _ in 0..200 * C { + match rng.below(4) { + 0 => { + let room = tx.room(&page).unwrap(); + assert_eq!(u64::from(room), model.room(written, released), "the room"); + if tx.can_grow() && rng.below(4) == 0 { + tx.grow(&page); + model = Model { cap: model.cap * 2, switch: Some(released + u64::from(model.cap)) }; + } + let n = rng.below(room + 1); + put(&mut region, tx.span(0, n).unwrap(), written); + assert!(tx.advance(n)); + written += u64::from(n); + let _ = tx.publish(&page); + published = written; + } + 1 | 2 => { + let ready = rx.ready(&page).unwrap(); + assert_eq!(u64::from(ready), published - read, "the bytes ready"); + let n = rng.below(ready + 1); + assert_eq!(get(®ion, rx.span(0, n).unwrap()), expected(read, n), "bytes {read}.. past the wrap"); + assert!(rx.advance(n)); + read += u64::from(n); + } + _ => { + let _ = rx.release(&page); + released = read; + } + } + } + assert!(u64::from(from) + read > 1 << 32, "the stream did not cross 2^32"); + } + + #[test] + fn a_ring_that_grows_across_the_wrap_agrees_with_the_stream() { + for seed in 1..=64 { + agrees_across_the_wrap::<64>(1, seed); + } + } + #[test] fn a_ring_that_grows_agrees_with_the_stream_too() { for seed in 1..=64 { diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index b7679ef0940..e4572e02a04 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -54,7 +54,8 @@ use crate::Node; pub trait Session { /// The region's header. fn words(&self) -> &Words; - /// The region's bytes, which the node lays its two rings out in. + /// The region's bytes, which the node lays its two rings out in: its header and each + /// ring's first size are memory already (`RECEIVE_INITIAL`, `SEND_INITIAL`). fn memory(&self) -> Rc; /// Rings the doorbells `rings` names, of every holder, never waiting. fn ring(&self, rings: Rings); @@ -96,8 +97,12 @@ impl LentRing { fn spans(&self, at: usize, len: usize) -> [(Span, usize); 4] { let none = (Span { offset: 0, len: 0 }, 0); let mut out = [none; 4]; - let (Ok(at), Ok(len)) = (u32::try_from(at & 0xFFFF_FFFF), u32::try_from(len)) else { return out }; - let Some(spans) = self.place.spans(self.shape.get().size, at, len) else { return out }; + let (Ok(at), Ok(len)) = (u32::try_from(at & 0xFFFF_FFFF), u32::try_from(len)) else { + unreachable!("[tcp] hands a ring at most its capacity, {len} bytes at {at}") + }; + let Some(spans) = self.place.spans(self.shape.get().size, at, len) else { + unreachable!("{len} bytes at {at} lie in at most four spans of a size the pass judged") + }; let mut from = 0usize; for (slot, span) in out.iter_mut().zip(spans) { *slot = (span, from); @@ -200,8 +205,6 @@ struct Stream { session: Box, /// The receive ring's and the send ring's shapes, as [tcp]'s storage reads them. shapes: Option<(Shared, Shared)>, - /// What the next pass ends the stream with. - broken: Option, /// The session's server end, once the connection is established and its buffers are the /// region's; `None` while a connect is not answered. server: Option, @@ -222,19 +225,13 @@ struct Stream { } impl Stream { - /// A connection whose buffers are the region's from here on, and its server end. A region - /// whose first rings cannot be made memory holds nothing, and the next pass ends the stream - /// with [`Violation::Size`]. + /// A connection whose buffers are the region's from here on, and its server end. fn lent(&mut self, stack: &mut Stack) { let session = &self.session; let memory = session.memory(); - let first = |ring, cap: u32| memory.commit(ring, usize::try_from(cap).unwrap_or(usize::MAX)); - let made = first(RECEIVE_RING, RECEIVE_INITIAL) && first(SEND_RING, SEND_INITIAL); - self.broken = (!made).then_some(Violation::Size); let server = Server::new(session.words(), LAYOUT, RECEIVE_INITIAL); - let (receive, send) = if made { (RECEIVE_INITIAL, SEND_INITIAL) } else { (0, 0) }; - let rx = Rc::new(Cell::new(Shape { size: server.receive_size(), holds: receive, wanted: 0 })); - let tx = Rc::new(Cell::new(Shape { size: server.send_size(), holds: send, wanted: 0 })); + let rx = Rc::new(Cell::new(Shape { size: server.receive_size(), holds: RECEIVE_INITIAL, wanted: 0 })); + let tx = Rc::new(Cell::new(Shape { size: server.send_size(), holds: SEND_INITIAL, wanted: 0 })); let receive = LentRing { memory: memory.clone(), place: LAYOUT.receive_place(), shape: rx.clone() }; let send = LentRing { memory, place: LAYOUT.send_place(), shape: tx.clone() }; if stack.tcp_lend(self.conn, Box::new(receive), Box::new(send)).is_err() { @@ -284,9 +281,6 @@ impl Stream { /// more, and rings the client once: `false` once the connection ended. fn moved(&mut self, now: Instant, stack: &mut Stack) -> Result { let conn = self.conn; - if let Some(violation) = self.broken.take() { - return Err(violation); - } let Some(server) = self.server.as_mut() else { unreachable!("a stream moves only once established") }; let Some((rx_shape, tx_shape)) = self.shapes.as_ref() else { unreachable!("a lent stream has its shapes") }; let words = self.session.words(); @@ -397,7 +391,7 @@ impl Streams { /// has, `nodelay` among them written into the region's header. pub(crate) fn accepted(&mut self, stack: &mut Stack, conn: ConnId, options: Options, session: Box) -> StreamId { session.words()[NODELAY].store(u32::from(options.nodelay), Ordering::Relaxed); - let mut stream = Stream { conn, session, shapes: None, broken: None, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; + let mut stream = Stream { conn, session, shapes: None, server: None, deadline: None, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 }; stream.lent(stack); self.hold(stream) } @@ -424,7 +418,7 @@ impl Node { let options = Options { nodelay: session.words()[NODELAY].load(Ordering::Relaxed) != 0, ..Options::default() }; self.stack.tcp_set_options(now, conn, options); let deadline = timeout.map(|within| now.after(within)); - Ok(self.streams.hold(Stream { conn, session, shapes: None, broken: None, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) + Ok(self.streams.hold(Stream { conn, session, shapes: None, server: None, deadline, held: 0, queued: 0, fin_queued: false, options, rung: false, more_holders: 0 })) } /// One pass over every stream: netstack calls it once the kernel answered the connections it diff --git a/system/netstack/src/lent.rs b/system/netstack/src/lent.rs index ed153882d54..002a2914f22 100644 --- a/system/netstack/src/lent.rs +++ b/system/netstack/src/lent.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::rc::Rc; use toyos::ipc::FrameRx; -use toyos::net::stream::{Words, REGION_BYTES}; +use toyos::net::stream::{Words, RECEIVE_INITIAL, RECEIVE_RING, REGION_BYTES, SEND_INITIAL, SEND_RING}; use toyos::shm::{self, SharedMemory}; use toyos::transport::session::{Holder, Rings}; use toyos::wake::{Bell, Waker}; @@ -98,10 +98,14 @@ pub struct Region { } impl Region { - /// The region `handle` names, judged and mapped. Refused, nothing is mapped and the handle - /// is closed. + /// The region `handle` names, judged and mapped, and its header and both rings' first + /// sizes made memory, charged to the client as each commit is. Refused, nothing is mapped + /// and the handle is closed. pub fn judged(handle: toyos::OwnedHandle) -> Result { - SharedMemory::lent(handle, REGION_BYTES) + let mapping = SharedMemory::lent(handle, REGION_BYTES)?; + mapping.commit(0, RECEIVE_RING + RECEIVE_INITIAL as usize).map_err(shm::Lent::Kernel)?; + mapping.commit(SEND_RING, SEND_INITIAL as usize).map_err(shm::Lent::Kernel)?; + Ok(mapping) } /// The session over `mapping`, `first` its first holder, and its holders, which `serve` diff --git a/tests/guest/rust-tests/src/bin/shm_commit.rs b/tests/guest/rust-tests/src/bin/shm_commit.rs index 68b7c0b50be..87b2854b8c7 100644 --- a/tests/guest/rust-tests/src/bin/shm_commit.rs +++ b/tests/guest/rust-tests/src/bin/shm_commit.rs @@ -1,15 +1,17 @@ //! A region made with only its first page committed is memory there and //! nowhere else until `shm_commit` makes more of it: a touch of a page past -//! it ends the process, a commit makes the page zeroed memory every mapping -//! sees, a second commit of it changes nothing, and `fstat` says how much is -//! committed. +//! it ends the process, a syscall's output into one is `BadAddress`, a spawn +//! from a region nothing committed is refused and the machine goes on, a +//! commit makes the page zeroed memory every mapping sees, a second commit of +//! it changes nothing, and the job's account rises by each page a commit +//! makes and falls back when the region goes. //! //! It needs the guest: a fault that never allocates, and a commit that a //! mapping made before it sees, are the kernel's page tables and frames. use std::process::{Command, Stdio}; -use toyos_abi::syscall::{self, SyscallError, PAGE_SIZE}; +use toyos_abi::syscall::{self, SpawnArgs, SyscallError, PAGE_SIZE}; const ME: &str = "/system/bin/test_rs_shm_commit"; const PAGES: usize = 4; @@ -18,13 +20,11 @@ fn main() { if std::env::args().nth(1).as_deref() == Some("touch") { return touch(); } + let own = toyos::job::own().expect("a handle to this job"); + let account = || own.account().expect("this job's account").committed; + let before = account(); let shm = syscall::shm_reserve(PAGES * PAGE_SIZE, PAGE_SIZE).expect("a region of four pages, one committed"); - let committed = |what: &str| { - let stat = syscall::fstat(shm).expect("fstat of the region"); - assert_eq!(stat.size, (PAGES * PAGE_SIZE) as u64, "{what}: the size is the whole region"); - stat.committed - }; - assert_eq!(committed("made"), PAGE_SIZE as u64); + assert_eq!(account() - before, PAGE_SIZE as u64, "a region made with one page committed counts one"); // SAFETY: a fresh region, mapped whole, which nothing else names. let at = unsafe { syscall::shm_map(shm) }.expect("map the region"); // SAFETY: the first page is committed. @@ -32,12 +32,15 @@ fn main() { assert_eq!(at.read_volatile(), 0, "a committed page is zeroed"); at.write_volatile(0x5A); } - println!(" PASS: a region made with one page committed is memory there"); + println!(" PASS: a region made with one page committed is memory there, and counted"); dies(); + output_is_refused(at); + a_spawn_from_nothing_committed_is_refused(); + let held = account(); syscall::shm_commit(shm, PAGE_SIZE + 100, 2 * PAGE_SIZE).expect("commit what pages 1 to 3 are touched by"); - assert_eq!(committed("committed"), 4 * PAGE_SIZE as u64, "every page the range touched"); + assert_eq!(account() - held, (3 * PAGE_SIZE) as u64, "every page the range touched is counted"); // SAFETY: pages 1 to 3 are committed now, and the mapping was made before. unsafe { for page in 1..PAGES { @@ -46,21 +49,77 @@ fn main() { byte.write_volatile(page as u8); } } + let held = account(); syscall::shm_commit(shm, 0, PAGES * PAGE_SIZE).expect("a commit of what is committed"); + assert_eq!(account(), held, "a second commit counts nothing more"); // SAFETY: as above. unsafe { assert_eq!(at.read_volatile(), 0x5A, "a second commit kept page 0"); assert_eq!(at.add(3 * PAGE_SIZE + 7).read_volatile(), 3, "a second commit kept page 3"); } - println!(" PASS: a commit makes zeroed memory a mapping made before sees, and a second keeps it"); + println!(" PASS: a commit makes zeroed memory a mapping made before sees, counted once, and a second keeps it"); assert_eq!(syscall::shm_commit(shm, 0, 0), Err(SyscallError::InvalidArgument), "no bytes"); assert_eq!(syscall::shm_commit(shm, PAGES * PAGE_SIZE, 1), Err(SyscallError::InvalidArgument), "past the region"); assert_eq!(syscall::shm_reserve(PAGE_SIZE, 2 * PAGE_SIZE), Err(SyscallError::InvalidArgument), "more committed than made"); println!(" PASS: a commit of no bytes or past the region, and a reservation committing more than it makes, are refused"); + + let held = account(); + // SAFETY: the mapping is this test's, and nothing reads it after. + unsafe { syscall::vm_release(at, PAGES * PAGE_SIZE) }.expect("unmap the region"); + syscall::close(shm); + assert_eq!(held - account(), (PAGES * PAGE_SIZE) as u64, "the region's pages went back with it"); + println!(" PASS: the region's pages leave the account with it"); println!("shm_commit: committed pages are memory, and only they"); } +/// A syscall's output into a page not committed is refused, not dropped: +/// a pipe's bytes read into page 1, and nothing taken off the pipe. +fn output_is_refused(at: *mut u8) { + let ends = syscall::pipe().expect("a pipe"); + assert_eq!(syscall::write(ends.write, b"held"), Ok(4)); + // SAFETY: page 1 of the mapping, never touched: the kernel judges it. + let page = unsafe { core::slice::from_raw_parts_mut(at.add(PAGE_SIZE), 4) }; + assert_eq!(syscall::read(ends.read, page), Err(SyscallError::BadAddress), "a read into a page not committed"); + let mut out = [0u8; 4]; + assert_eq!(syscall::read(ends.read, &mut out), Ok(4), "the refused read took nothing"); + assert_eq!(&out, b"held"); + syscall::close(ends.read); + syscall::close(ends.write); + println!(" PASS: a syscall's output into a page not committed is BadAddress, and nothing is lost"); +} + +/// A spawn whose image is a region nothing committed: the loader reads its +/// pages as zeros, refuses the image, and the machine goes on. +fn a_spawn_from_nothing_committed_is_refused() { + let image = syscall::shm_reserve(2 * PAGE_SIZE, 0).expect("a region of two pages, none committed"); + let argv = format!("{ME}\0"); + let cwd = "/"; + // SAFETY: every pointer names a live local for the whole call. + let spawned = unsafe { + syscall::spawn(&SpawnArgs { + path_ptr: ME.as_ptr() as u64, + path_len: ME.len() as u64, + argv_ptr: argv.as_ptr() as u64, + argv_len: argv.len() as u64, + env_ptr: 0, + env_len: 0, + endow_ptr: 0, + endow_count: 0, + labels_ptr: 0, + labels_len: 0, + cwd_ptr: cwd.as_ptr() as u64, + cwd_len: cwd.len() as u64, + image: image.0 as u64, + image_len: (2 * PAGE_SIZE) as u64, + place: u64::from(toyos_abi::HANDLE_INVALID.0), + }) + }; + assert!(spawned.is_err(), "an image of nothing committed was spawned"); + syscall::close(image); + println!(" PASS: a spawn from a region nothing committed is refused: {spawned:?}"); +} + /// A child touches page 1 of a region committed only on page 0, and dies of it. fn dies() { let child = Command::new(ME).arg("touch").stdout(Stdio::piped()).spawn().expect("spawn the toucher"); diff --git a/tests/guest/rust-tests/src/bin/stream_region_refused.rs b/tests/guest/rust-tests/src/bin/stream_region_refused.rs new file mode 100644 index 00000000000..16a05132eab --- /dev/null +++ b/tests/guest/rust-tests/src/bin/stream_region_refused.rs @@ -0,0 +1,34 @@ +//! A connect whose region has none of its pages committed does not fault +//! netstack, which goes on serving: the header and both rings' first sizes +//! are made memory where netstack judges the region, charged to the client, +//! before anything reads them, or the region is refused there. +//! +//! It needs the guest's real netstack: the judge is netstack's own, and the +//! fault it prevents would end netstack, which only the kernel delivers. +//! `netstack_holders_exported_to_children`'s boot runs it, on +//! tests/images/netcase.toml, whose netstack has its lease. + +use std::net::{Ipv4Addr, SocketAddrV4, TcpListener}; + +use toyos::net::stream::REGION_BYTES; +use toyos::net::{MsgType, NetstackConn, TcpConnectRequest, TcpConnectResponse}; +use toyos::OwnedHandle; +use toyos_abi::syscall; + +fn main() { + let region = syscall::shm_reserve(REGION_BYTES, 0).expect("a region with nothing committed"); + // SAFETY: a handle the kernel just made, which nothing else answers for. + let region = unsafe { OwnedHandle::from_raw(region) }; + let request = TcpConnectRequest { addr: [192, 0, 2, 9], port: 9, _pad: 0, timeout_ms: 1_000 }; + let answer = NetstackConn::connect() + .and_then(|conn| conn.request_with_handles([region], MsgType::TcpConnect, &request)) + .and_then(|pending| pending.answer_with_handles::()); + match answer { + Err(refused) => println!(" PASS: a connect with no page of its region committed was refused: {refused:?}"), + Ok(_) => println!(" PASS: a connect with no page of its region committed had its first pages made memory, and was answered"), + } + let listener = TcpListener::bind(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)).expect("netstack serves on after the connect"); + drop(listener); + println!(" PASS: netstack serves on"); + println!("stream_region_refused: a client's uncommitted region never faults netstack"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 4f33f1c52b2..65bc646695d 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -254,6 +254,12 @@ const RUST_SKIP: &[&str] = &[ // `netstack_holders_exported_to_children`'s boot sets up. "stream_export", "socket_export", + // A memory object's commit and its faults: `shm_commit_alone` runs it on + // QEMU, which establishes them; nothing in it is hardware. + "shm_commit", + // A connect whose region nothing committed, to real netstack, which goes + // on serving: `netstack_holders_exported_to_children`'s boot runs it. + "stream_region_refused", // A stream's holders across processes, as their own netstack: it spawns a // child that faults, which a shared boot's other members must not watch, // and `stream_holders` runs it. @@ -757,6 +763,13 @@ const MACHINE_TESTS: &[&str] = &[ // refused meanwhile. QEMU establishes the refusals and that the machine // goes on; nothing in it is timing. "kernel_objects_refused_alone", + // A region committed in parts: a touch past the commit ends the process, a + // syscall's output into it is `BadAddress`, a spawn from nothing committed + // is refused, a commit is zeroed memory an earlier mapping sees, counted + // to the job once and given back with the region. These are the kernel's + // page tables, frames and accounts, which no host model has; order and + // contents only, on the shared boot's q35 guest in a boot of its own. + "shm_commit_alone", // With one ordinary job holding every frame it may, the supervisor answers // a stop's request, a key reaches the owner of input through ps2server, a // system holder kills the greedy job, and a service the exhaustion ended is @@ -4853,6 +4866,7 @@ fn run_machine_test(name: &str, test_config: &Path) -> Result<(), String> { "std_threads_count_the_cpus_and_give_back_ended_threads" => std_threads_count_the_cpus_and_give_back_ended_threads(test_config), "affinity_pins_hold_on_every_cpu" => affinity_pins_hold_on_every_cpu(test_config), "kernel_objects_refused_alone" => kernel_objects_refused_alone(test_config), + "shm_commit_alone" => shm_commit_alone(test_config), "memory_floor_holds_the_controls" => memory_floor_holds_the_controls(), "the_opening_exchange_with_qemus_agent_completes" => the_opening_exchange_with_qemus_agent_completes(), "crypto_instrument_prints_its_table" => crypto_instrument_prints_its_table(test_config), @@ -5685,6 +5699,24 @@ fn kernel_objects_refused_alone(test_config: &Path) -> Result<(), String> { Ok(()) } +fn shm_commit_alone(test_config: &Path) -> Result<(), String> { + const JOB: &str = "shm_commit"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], BootOptions::default()); + serial::Serial::boot(&qemu).must_be_clean()?; + let result = qemu.run_test("test_rs_shm_commit", Duration::from_secs(120)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + for line in result.stdout.lines().filter(|l| l.contains("PASS: ")) { + eprintln!(" [commit] {}", line.trim()); + } + Ok(()) +} + /// `memory_floor --input` alone in `tests/images/floorcase.toml`'s boot, a key /// typed once every frame is held: the job holds what the controls did, and the /// supervisor said it was refused the starved service's restart, then started @@ -5797,7 +5829,7 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str const LEASED: &str = "netstack: ready, at most"; let tests = compile::repo_root().join("tests/guest/rust-tests"); let bins: Vec<(String, Vec)> = - ["stream_export", "socket_export"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); + ["stream_export", "socket_export", "stream_region_refused"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); // The host's end of the forward: a port found free and given up, which // QEMU binds as it starts and refuses to start without. Another process // can take it between the two, which fails the boot by name. @@ -5821,7 +5853,12 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str return Err(format!("{driver}: netstack leased without saying {opened:?}\n{said}")); } stream_exported_to_a_child(&mut qemu)?; - sockets_exported_to_a_child(&mut qemu, forwarded) + sockets_exported_to_a_child(&mut qemu, forwarded)?; + let result = qemu.run_test("test_rs_stream_region_refused", Duration::from_secs(60)); + match (&result.error, result.exit_code) { + (None, Some(0)) => Ok(()), + _ => Err(format!("{:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)), + } } /// `test_rs_stream_export`: the child's line, then the parent's, then the FIN From e2510f31b87adacda7f803a40badf3d84bc8d1a6 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:17:44 +0100 Subject: [PATCH 07/16] stream region: onto main, the paged objects' frees and the clock page The rebase onto main brings #1123's paged objects, which free a frame with `pmm::free_drawn` as the shared regions now do, and #1104's clock page, which reads its object's one page: both take the commit's types. SYS_SHM_COMMIT is 160, past #1121's SYS_MESSAGE_RECV. The guest tests map with a protection, as main's `shm_map` asks, and a connect's answer carries main's three bells. The kernel's `Stat` size assertion, left from the removed `committed` field, goes. Co-Authored-By: Claude Opus 5.5 --- kernel/src/clock.rs | 2 +- kernel/src/object/ops.rs | 2 -- kernel/src/object/paged.rs | 4 ++-- sdk/toyos/src/net/stream.rs | 2 +- sdk/toyos/src/shm.rs | 2 +- tests/guest/rust-tests/src/bin/shm_commit.rs | 6 +++--- tests/guest/rust-tests/src/bin/stream_region_refused.rs | 4 ++-- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 68829bad25a..2c7ba330bc8 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -64,7 +64,7 @@ pub fn set_counter(boot: u64, mult: u64) { let stated = STATED_MULT.load(Relaxed); let at_boot = Line { counter: 0, stamp: 0, mult: if stated != 0 { stated } else { mult } }.nanos(boot); let page = SharedMemObject::create(crate::mm::charge::boot(), crate::mm::PAGE_SIZE).expect("clock: no frame for the clock page"); - let lines = crate::DirectMap::from_phys(page.page(0)).as_mut_ptr::(); + let lines = crate::DirectMap::from_phys(page.page(0).expect("clock: its page is committed whole")).as_mut_ptr::(); // SAFETY: the page is this object's own, which no address space maps yet // and none unmaps after: held below for the machine's life. Every field of // `ClockPage` is an atomic, so a shared reference reads and writes it as diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 286b2cae8e6..5198dbffdb9 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -648,8 +648,6 @@ toyos_abi::user_safe! { } } -const _: () = assert!(core::mem::size_of::() == core::mem::size_of::()); - /// What kind of thing this is, how big, and which file: a file's node is its /// `FileId`, one per file for as long as it exists (`vfs::FileSystem::open_file`). pub fn fstat(object: &KObjectRef) -> Stat { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index a7b74d6b77d..54889072508 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -132,7 +132,7 @@ impl Held { } // SAFETY: `phys` is a frame this pager drew, out of its index or never // in it, which nothing maps; a reader's pin defers its reuse. - unsafe { pmm::free_drawn(job, &[phys]) }; + unsafe { pmm::free_drawn(job, [phys]) }; } /// Move each held-aside frame no copy pins any more to the spare ones. @@ -948,7 +948,7 @@ impl ZeroHandles for PagerObject { while let Some(phys) = state.held.spare.pop().or_else(|| state.held.draining.pop()) { // SAFETY: a frame this pager drew and held aside, which nothing // maps; a pin defers its free to its last unpin. - unsafe { pmm::free_drawn(job, &[phys]) }; + unsafe { pmm::free_drawn(job, [phys]) }; } let pool = state.held.core.end(); pmm::unpromise(job, pool); diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index f3bb517ed40..7fe46a836eb 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -358,7 +358,7 @@ struct Call { impl Call { /// Make a ring's first `cap` bytes, at `ring`, memory. fn commit(&self, ring: usize, cap: u32) -> bool { - self.end.region.is_some_and(|(region, _)| syscall::shm_commit(region, ring, cap as usize).is_ok()) + self.end.region.is_some_and(|mapped| syscall::shm_commit(mapped.handle, ring, cap as usize).is_ok()) } fn words(&self) -> &Words { diff --git a/sdk/toyos/src/shm.rs b/sdk/toyos/src/shm.rs index 1b5fac358f3..de2acdc5091 100644 --- a/sdk/toyos/src/shm.rs +++ b/sdk/toyos/src/shm.rs @@ -65,7 +65,7 @@ impl SharedMemory { /// [`create`](Self::create), only the pages of its first `committed` /// bytes memory until [`Self::commit`] makes more of it. pub fn reserve(size: usize, committed: usize) -> Result { - Self::adopt(syscall::shm_reserve(size, committed)?, size) + Self::adopt(OwnedHandle(syscall::shm_reserve(size, committed)?), size) } /// Make every page `[offset, offset + len)` of the region touches memory diff --git a/tests/guest/rust-tests/src/bin/shm_commit.rs b/tests/guest/rust-tests/src/bin/shm_commit.rs index 87b2854b8c7..43f8efc1192 100644 --- a/tests/guest/rust-tests/src/bin/shm_commit.rs +++ b/tests/guest/rust-tests/src/bin/shm_commit.rs @@ -11,7 +11,7 @@ use std::process::{Command, Stdio}; -use toyos_abi::syscall::{self, SpawnArgs, SyscallError, PAGE_SIZE}; +use toyos_abi::syscall::{self, SpawnArgs, SyscallError, VmProt, PAGE_SIZE}; const ME: &str = "/system/bin/test_rs_shm_commit"; const PAGES: usize = 4; @@ -26,7 +26,7 @@ fn main() { let shm = syscall::shm_reserve(PAGES * PAGE_SIZE, PAGE_SIZE).expect("a region of four pages, one committed"); assert_eq!(account() - before, PAGE_SIZE as u64, "a region made with one page committed counts one"); // SAFETY: a fresh region, mapped whole, which nothing else names. - let at = unsafe { syscall::shm_map(shm) }.expect("map the region"); + let at = unsafe { syscall::shm_map(shm, VmProt::ReadWrite) }.expect("map the region"); // SAFETY: the first page is committed. unsafe { assert_eq!(at.read_volatile(), 0, "a committed page is zeroed"); @@ -134,7 +134,7 @@ fn dies() { fn touch() { let shm = syscall::shm_reserve(2 * PAGE_SIZE, PAGE_SIZE).expect("a region of two pages, one committed"); // SAFETY: a fresh region, mapped whole; page 1 is past what is committed, which is the test. - let at = unsafe { syscall::shm_map(shm) }.expect("map the region"); + let at = unsafe { syscall::shm_map(shm, VmProt::ReadWrite) }.expect("map the region"); println!("armed"); // SAFETY: as above. let byte = unsafe { at.add(PAGE_SIZE).read_volatile() }; diff --git a/tests/guest/rust-tests/src/bin/stream_region_refused.rs b/tests/guest/rust-tests/src/bin/stream_region_refused.rs index 16a05132eab..d02bb57b26b 100644 --- a/tests/guest/rust-tests/src/bin/stream_region_refused.rs +++ b/tests/guest/rust-tests/src/bin/stream_region_refused.rs @@ -10,7 +10,7 @@ use std::net::{Ipv4Addr, SocketAddrV4, TcpListener}; -use toyos::net::stream::REGION_BYTES; +use toyos::net::stream::{BELLS, REGION_BYTES}; use toyos::net::{MsgType, NetstackConn, TcpConnectRequest, TcpConnectResponse}; use toyos::OwnedHandle; use toyos_abi::syscall; @@ -22,7 +22,7 @@ fn main() { let request = TcpConnectRequest { addr: [192, 0, 2, 9], port: 9, _pad: 0, timeout_ms: 1_000 }; let answer = NetstackConn::connect() .and_then(|conn| conn.request_with_handles([region], MsgType::TcpConnect, &request)) - .and_then(|pending| pending.answer_with_handles::()); + .and_then(|pending| pending.answer_with_handles::()); match answer { Err(refused) => println!(" PASS: a connect with no page of its region committed was refused: {refused:?}"), Ok(_) => println!(" PASS: a connect with no page of its region committed had its first pages made memory, and was answered"), From ababbf44653bdde622d765d941f78b45d1ec2dd0 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:18:28 +0100 Subject: [PATCH 08/16] issues: a connect's region commits 132 KiB, and its T14 reading is owed Co-Authored-By: Claude Opus 5.5 --- issues/a-stream-ring-never-shrinks.md | 4 ++-- issues/a-tcp-connect-zeroes-twelve-mebibytes.md | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/issues/a-stream-ring-never-shrinks.md b/issues/a-stream-ring-never-shrinks.md index 7acef1095db..40192113f9c 100644 --- a/issues/a-stream-ring-never-shrinks.md +++ b/issues/a-stream-ring-never-shrinks.md @@ -13,9 +13,9 @@ that once carried a burst keeps the most it ever held, up to 8 MiB + 4 KiB, for as long as it lives, counted to its client's job. netstack prices each place at that most (`PLACE_BYTES`, `system/netstack/src/main.rs`). -Owner: the orchestrator, who assigns it after #1077 lands. +**Owner**: the orchestrator, who assigns it after #1077 lands. -Exit: a ring whose stream has gone idle gives back what it no longer holds, +**Exit**: a ring whose stream has gone idle gives back what it no longer holds, at a quiescent point both ends agree on (both cursors equal, under the session's gates), through a kernel call that decommits pages of a memory object; a host test of the ring and a loom model of the shrink beside a diff --git a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md index e279f776b90..1fc8b64393c 100644 --- a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md +++ b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md @@ -9,10 +9,11 @@ opened: 2026-10-27 Before a ToyOS TCP connect returns, it makes and zeroes memory the connection may never use: -- **The stream region**, 8 MiB (`toyos::net::stream::REGION_BYTES`). - `SYS_SHM_CREATE` takes and zeroes each of its 2048 frames before it answers - (`kernel/src/object/shm.rs`), and netstack maps all of them before the SYN - leaves. +- **The stream region**, 8 MiB (`toyos::net::stream::REGION_BYTES`), whose + 2048 frames `SYS_SHM_CREATE` took and zeroed before it answered. Since + #1077 a connect commits its header and 64 KiB of each ring, 132 KiB, and + each ring's producer commits more as it doubles the ring + (`toyos::transport::stream`); the T14's reading of that is owed. - **The connection's two directions**, 2 MiB each: the request's page on the client's write and the answer's on netstack's, each zeroed whole on that first write (`issues/a-pipe-is-a-two-mebibyte-page-for-a-mapping-nothing-makes.md`). From 13a9ad6aa0738dd93ff09e42d651bdc206ee6a93 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:35:02 +0100 Subject: [PATCH 09/16] stream region: a loopback stream's bytes through its rings' growth, on the guest The ring's growth was tested on the host, through models of the region's memory, and on no guest: nothing ran the client's commit before it reads or writes past a ring's first size, or netstack's before it grows the receive ring, against the kernel that refuses a touch past a commit. `stream_grows` sends 4 MiB each way over loopback, in writes of a byte to more than a ring, and checks every byte's place. It runs in the netcase boot that already runs `stream_region_refused`. Co-Authored-By: Claude Opus 5.5 --- .../guest/rust-tests/src/bin/stream_grows.rs | 72 +++++++++++++++++++ tests/toyos.rs | 23 +++--- 2 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 tests/guest/rust-tests/src/bin/stream_grows.rs diff --git a/tests/guest/rust-tests/src/bin/stream_grows.rs b/tests/guest/rust-tests/src/bin/stream_grows.rs new file mode 100644 index 00000000000..34e9a204249 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/stream_grows.rs @@ -0,0 +1,72 @@ +//! A loopback stream carries 4 MiB each way, every byte where it was +//! written, while its rings grow: each send ring at the first write its room +//! cannot take, here the third, and each receive ring as [tcp] asks for more +//! buffer, every page committed by the end that touches it before it does. A +//! missing commit ends the process or netstack; a byte laid out by the wrong +//! size arrives out of place. +//! +//! It needs the guest: the commits are the kernel's, and the receive ring's +//! growth is the real netstack's. `netstack_holders_exported_to_children`'s +//! boot runs it, on tests/images/netcase.toml. + +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, Shutdown, TcpListener, TcpStream}; +use std::thread; + +const BYTES: usize = 4 << 20; +/// Write sizes in turn: a byte, a part of a ring, one ring and more than one, +/// so a write meets a full ring, a partial room and a growth pending. +const WRITES: [usize; 4] = [1, 1000, 64 << 10, 300 << 10]; + +/// The byte at stream position `at`, which repeats at no power of two a ring +/// could be. +fn byte(at: usize) -> u8 { + ((at as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +fn send(mut stream: &TcpStream) { + let all: Vec = (0..BYTES).map(byte).collect(); + let mut at = 0; + for size in WRITES.iter().cycle() { + if at == BYTES { + break; + } + let end = (at + size).min(BYTES); + stream.write_all(&all[at..end]).expect("a write the stream takes"); + at = end; + } + stream.shutdown(Shutdown::Write).expect("the sender's FIN"); +} + +fn receive(mut stream: &TcpStream, way: &str) { + let mut buf = vec![0u8; 200 << 10]; + let mut at = 0; + loop { + // Reads of a varying size: some take less than is ready. + let want = [7usize, 4096, 200 << 10][at % 3]; + let n = stream.read(&mut buf[..want]).expect("a read the stream answers"); + if n == 0 { + break; + } + if let Some(wrong) = (0..n).find(|&i| buf[i] != byte(at + i)) { + panic!("{way}: the byte at {} is {:#04x}, written {:#04x}", at + wrong, buf[wrong], byte(at + wrong)); + } + at += n; + } + assert_eq!(at, BYTES, "{way}: the stream ended after {at} bytes"); +} + +fn main() { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("a loopback listener"); + let addr = listener.local_addr().expect("the listener's address"); + let connected = TcpStream::connect(addr).expect("a loopback connect"); + let (accepted, _) = listener.accept().expect("the connect's accept"); + thread::scope(|scope| { + for (way, from, to) in [("connected to accepted", &connected, &accepted), ("accepted to connected", &accepted, &connected)] { + scope.spawn(move || send(from)); + scope.spawn(move || receive(to, way)); + } + }); + println!(" PASS: 4 MiB each way arrived in order through its rings' growth"); + println!("stream_grows: a stream's rings grow under load and keep every byte in place"); +} diff --git a/tests/toyos.rs b/tests/toyos.rs index 65bc646695d..17ff5cc78bf 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -260,6 +260,9 @@ const RUST_SKIP: &[&str] = &[ // A connect whose region nothing committed, to real netstack, which goes // on serving: `netstack_holders_exported_to_children`'s boot runs it. "stream_region_refused", + // A loopback stream's bytes through its rings' growth, which needs the + // real netstack: `netstack_holders_exported_to_children`'s boot runs it. + "stream_grows", // A stream's holders across processes, as their own netstack: it spawns a // child that faults, which a shared boot's other members must not watch, // and `stream_holders` runs it. @@ -5813,9 +5816,11 @@ const SOCKET_EXPORT_PORT: u16 = 7070; /// `tests/images/netcase.toml`, the harness every socket's peer: a stream's /// child's line, then its parent's, then the FIN once both holders are gone; /// then a listener's and a datagram socket's child's datagram, its accept and -/// its line, and the parent's after it. Once over virtio, and once over QEMU's -/// `e1000e`, which the image's rows name no machine for: the I219 driver's -/// row matches it, and its link is the one netstack opens. +/// its line, and the parent's after it; then a connect whose region nothing +/// committed, and a loopback stream through its rings' growth. Once over +/// virtio, and once over QEMU's `e1000e`, which the image's rows name no +/// machine for: the I219 driver's row matches it, and its link is the one +/// netstack opens. fn netstack_holders_exported_to_children() -> Result<(), String> { for (profile, driver) in [(qemu::Profile::Headless, "virtio-net"), (qemu::Profile::HeadlessE1000e, "82574")] { holders_exported_over(profile, driver)?; @@ -5829,7 +5834,7 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str const LEASED: &str = "netstack: ready, at most"; let tests = compile::repo_root().join("tests/guest/rust-tests"); let bins: Vec<(String, Vec)> = - ["stream_export", "socket_export", "stream_region_refused"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); + ["stream_export", "socket_export", "stream_region_refused", "stream_grows"].iter().map(|job| (job.to_string(), qemu::build_toyos_bin(qemu::SUITE_ARCH, &tests, job))).collect(); // The host's end of the forward: a port found free and given up, which // QEMU binds as it starts and refuses to start without. Another process // can take it between the two, which fails the boot by name. @@ -5854,11 +5859,13 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str } stream_exported_to_a_child(&mut qemu)?; sockets_exported_to_a_child(&mut qemu, forwarded)?; - let result = qemu.run_test("test_rs_stream_region_refused", Duration::from_secs(60)); - match (&result.error, result.exit_code) { - (None, Some(0)) => Ok(()), - _ => Err(format!("{:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)), + for job in ["test_rs_stream_region_refused", "test_rs_stream_grows"] { + let result = qemu.run_test(job, Duration::from_secs(120)); + if result.error.is_some() || result.exit_code != Some(0) { + return Err(format!("{driver}: {job}: {:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)); + } } + Ok(()) } /// `test_rs_stream_export`: the child's line, then the parent's, then the FIN From f0e75f23813c6f4942e094d346a9e58fea74897e Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 19:19:23 +0100 Subject: [PATCH 10/16] stream region: the growth test's peer is the harness, as ToyOS has no loopback netstack refuses 127/8 as a destination, so `stream_grows` listens on the port its boot forwards and the harness connects through it, as for `socket_export`: both ends send 4 MiB and check every byte's place. `shm_commit` names main's `SpawnArgs.pager` and closes in `unsafe`. Co-Authored-By: Claude Opus 5.5 --- tests/guest/rust-tests/src/bin/shm_commit.rs | 14 ++-- .../guest/rust-tests/src/bin/stream_grows.rs | 57 +++++++++------- tests/toyos.rs | 66 ++++++++++++++++--- 3 files changed, 100 insertions(+), 37 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/shm_commit.rs b/tests/guest/rust-tests/src/bin/shm_commit.rs index 43f8efc1192..f671c4e2e84 100644 --- a/tests/guest/rust-tests/src/bin/shm_commit.rs +++ b/tests/guest/rust-tests/src/bin/shm_commit.rs @@ -67,7 +67,8 @@ fn main() { let held = account(); // SAFETY: the mapping is this test's, and nothing reads it after. unsafe { syscall::vm_release(at, PAGES * PAGE_SIZE) }.expect("unmap the region"); - syscall::close(shm); + // SAFETY: this test's own handle, used no more. + unsafe { syscall::close(shm) }; assert_eq!(held - account(), (PAGES * PAGE_SIZE) as u64, "the region's pages went back with it"); println!(" PASS: the region's pages leave the account with it"); println!("shm_commit: committed pages are memory, and only they"); @@ -84,8 +85,11 @@ fn output_is_refused(at: *mut u8) { let mut out = [0u8; 4]; assert_eq!(syscall::read(ends.read, &mut out), Ok(4), "the refused read took nothing"); assert_eq!(&out, b"held"); - syscall::close(ends.read); - syscall::close(ends.write); + // SAFETY: this test's own pipe, used no more. + unsafe { + syscall::close(ends.read); + syscall::close(ends.write); + } println!(" PASS: a syscall's output into a page not committed is BadAddress, and nothing is lost"); } @@ -113,10 +117,12 @@ fn a_spawn_from_nothing_committed_is_refused() { image: image.0 as u64, image_len: (2 * PAGE_SIZE) as u64, place: u64::from(toyos_abi::HANDLE_INVALID.0), + pager: u64::from(toyos_abi::HANDLE_INVALID.0), }) }; assert!(spawned.is_err(), "an image of nothing committed was spawned"); - syscall::close(image); + // SAFETY: this test's own handle, used no more. + unsafe { syscall::close(image) }; println!(" PASS: a spawn from a region nothing committed is refused: {spawned:?}"); } diff --git a/tests/guest/rust-tests/src/bin/stream_grows.rs b/tests/guest/rust-tests/src/bin/stream_grows.rs index 34e9a204249..92282a941f7 100644 --- a/tests/guest/rust-tests/src/bin/stream_grows.rs +++ b/tests/guest/rust-tests/src/bin/stream_grows.rs @@ -1,25 +1,35 @@ -//! A loopback stream carries 4 MiB each way, every byte where it was -//! written, while its rings grow: each send ring at the first write its room -//! cannot take, here the third, and each receive ring as [tcp] asks for more -//! buffer, every page committed by the end that touches it before it does. A -//! missing commit ends the process or netstack; a byte laid out by the wrong -//! size arrives out of place. +//! A stream carries 4 MiB each way, every byte where it was written, while +//! its rings grow: the send ring at the first write its room cannot take, +//! here the third, and the receive ring as [tcp] asks for more buffer, every +//! page committed by the end that touches it before it does. A missing +//! commit ends this process or netstack; a byte laid out by the wrong size +//! arrives out of place. //! //! It needs the guest: the commits are the kernel's, and the receive ring's -//! growth is the real netstack's. `netstack_holders_exported_to_children`'s -//! boot runs it, on tests/images/netcase.toml. +//! growth is the real netstack's. ToyOS has no loopback, so the harness is +//! the peer, as for `socket_export`: this job's argument is the port of its +//! datagram socket, which QEMU's user network reaches as `10.0.2.2`. The job +//! listens on the port the boot forwards, says it is ready in a datagram, +//! accepts the harness's connection, and both ends send [`BYTES`] and check +//! what they receive. `netstack_holders_exported_to_children`'s boot runs it, +//! on tests/images/netcase.toml. use std::io::{Read, Write}; -use std::net::{Ipv4Addr, Shutdown, TcpListener, TcpStream}; +use std::net::{Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; use std::thread; +/// QEMU's user network's address of its host. +const HOST: Ipv4Addr = Ipv4Addr::new(10, 0, 2, 2); +/// The port the boot forwards to (`tests/toyos.rs`'s `SOCKET_EXPORT_PORT`). +const LISTENING: u16 = 7070; +/// What each end sends, as `tests/toyos.rs`'s `STREAM_GROWS_BYTES`. const BYTES: usize = 4 << 20; /// Write sizes in turn: a byte, a part of a ring, one ring and more than one, -/// so a write meets a full ring, a partial room and a growth pending. +/// so a write meets a partial room, a full ring and a growth pending. const WRITES: [usize; 4] = [1, 1000, 64 << 10, 300 << 10]; /// The byte at stream position `at`, which repeats at no power of two a ring -/// could be. +/// could be: `tests/toyos.rs`'s `stream_grows_byte`. fn byte(at: usize) -> u8 { ((at as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 } @@ -35,10 +45,10 @@ fn send(mut stream: &TcpStream) { stream.write_all(&all[at..end]).expect("a write the stream takes"); at = end; } - stream.shutdown(Shutdown::Write).expect("the sender's FIN"); + stream.shutdown(Shutdown::Write).expect("this end's FIN"); } -fn receive(mut stream: &TcpStream, way: &str) { +fn receive(mut stream: &TcpStream) { let mut buf = vec![0u8; 200 << 10]; let mut at = 0; loop { @@ -49,24 +59,23 @@ fn receive(mut stream: &TcpStream, way: &str) { break; } if let Some(wrong) = (0..n).find(|&i| buf[i] != byte(at + i)) { - panic!("{way}: the byte at {} is {:#04x}, written {:#04x}", at + wrong, buf[wrong], byte(at + wrong)); + panic!("the byte at {} is {:#04x}, sent {:#04x}", at + wrong, buf[wrong], byte(at + wrong)); } at += n; } - assert_eq!(at, BYTES, "{way}: the stream ended after {at} bytes"); + assert_eq!(at, BYTES, "the stream ended after {at} bytes"); } fn main() { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("a loopback listener"); - let addr = listener.local_addr().expect("the listener's address"); - let connected = TcpStream::connect(addr).expect("a loopback connect"); - let (accepted, _) = listener.accept().expect("the connect's accept"); + let port: u16 = std::env::args().nth(1).expect("the harness's datagram port").parse().expect("a port"); + let listener = TcpListener::bind((Ipv4Addr::UNSPECIFIED, LISTENING)).expect("a listener through netstack"); + let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).expect("a datagram socket through netstack"); + socket.send_to(b"grows ready", SocketAddr::from((HOST, port))).expect("the ready datagram"); + let (stream, _) = listener.accept().expect("the harness's connection"); thread::scope(|scope| { - for (way, from, to) in [("connected to accepted", &connected, &accepted), ("accepted to connected", &accepted, &connected)] { - scope.spawn(move || send(from)); - scope.spawn(move || receive(to, way)); - } + scope.spawn(|| send(&stream)); + receive(&stream); }); - println!(" PASS: 4 MiB each way arrived in order through its rings' growth"); + println!(" PASS: 4 MiB each way arrived in order through the rings' growth"); println!("stream_grows: a stream's rings grow under load and keep every byte in place"); } diff --git a/tests/toyos.rs b/tests/toyos.rs index 17ff5cc78bf..e14a30b55a7 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -260,8 +260,8 @@ const RUST_SKIP: &[&str] = &[ // A connect whose region nothing committed, to real netstack, which goes // on serving: `netstack_holders_exported_to_children`'s boot runs it. "stream_region_refused", - // A loopback stream's bytes through its rings' growth, which needs the - // real netstack: `netstack_holders_exported_to_children`'s boot runs it. + // A stream's bytes through its rings' growth, the harness its peer: + // `netstack_holders_exported_to_children`'s boot runs it. "stream_grows", // A stream's holders across processes, as their own netstack: it spawns a // child that faults, which a shared boot's other members must not watch, @@ -5817,7 +5817,7 @@ const SOCKET_EXPORT_PORT: u16 = 7070; /// child's line, then its parent's, then the FIN once both holders are gone; /// then a listener's and a datagram socket's child's datagram, its accept and /// its line, and the parent's after it; then a connect whose region nothing -/// committed, and a loopback stream through its rings' growth. Once over +/// committed, and a stream to the harness through its rings' growth. Once over /// virtio, and once over QEMU's `e1000e`, which the image's rows name no /// machine for: the I219 driver's row matches it, and its link is the one /// netstack opens. @@ -5859,13 +5859,11 @@ fn holders_exported_over(profile: qemu::Profile, driver: &str) -> Result<(), Str } stream_exported_to_a_child(&mut qemu)?; sockets_exported_to_a_child(&mut qemu, forwarded)?; - for job in ["test_rs_stream_region_refused", "test_rs_stream_grows"] { - let result = qemu.run_test(job, Duration::from_secs(120)); - if result.error.is_some() || result.exit_code != Some(0) { - return Err(format!("{driver}: {job}: {:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)); - } + let result = qemu.run_test("test_rs_stream_region_refused", Duration::from_secs(60)); + if result.error.is_some() || result.exit_code != Some(0) { + return Err(format!("{driver}: {:?}, ended {:?}; the job said:\n{}", result.error, result.exit_code, result.stdout)); } - Ok(()) + stream_grows_with_the_harness(&mut qemu, forwarded) } /// `test_rs_stream_export`: the child's line, then the parent's, then the FIN @@ -5976,6 +5974,56 @@ fn sockets_exported_to_a_child(qemu: &mut QemuInstance, forwarded: u16) -> Resul heard.join().map_err(|_| "the peer's thread panicked".to_string())? } +/// What `test_rs_stream_grows` and the harness each send. +const STREAM_GROWS_BYTES: usize = 4 << 20; + +/// The byte at stream position `at` of either way: `stream_grows.rs`'s `byte`. +fn stream_grows_byte(at: usize) -> u8 { + ((at as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 56) as u8 +} + +/// `test_rs_stream_grows`, the harness its peer through the forwarded port: +/// once the job's datagram says it listens, the harness connects, sends +/// [`STREAM_GROWS_BYTES`] and its FIN, and reads as many back, each in place. +fn stream_grows_with_the_harness(qemu: &mut QemuInstance, forwarded: u16) -> Result<(), String> { + use std::io::{Read, Write}; + /// A liveness ceiling on each of the peer's waits, never a timing. + const CEILING: Duration = Duration::from_secs(120); + let datagrams = std::net::UdpSocket::bind("127.0.0.1:0").map_err(|e| format!("no loopback datagram socket: {e}"))?; + datagrams.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let port = datagrams.local_addr().map_err(|e| format!("the datagram socket's port: {e}"))?.port(); + let peer = std::thread::spawn(move || -> Result<(), String> { + let mut ready = [0u8; 64]; + let (n, _) = datagrams.recv_from(&mut ready).map_err(|e| format!("the job's ready datagram: {e}"))?; + if &ready[..n] != b"grows ready" { + return Err(format!("the peer heard {:?}", String::from_utf8_lossy(&ready[..n]))); + } + let conn = std::net::TcpStream::connect(("127.0.0.1", forwarded)).map_err(|e| format!("the connect: {e}"))?; + conn.set_read_timeout(Some(CEILING)).map_err(|e| e.to_string())?; + let mut sending = conn.try_clone().map_err(|e| format!("a second handle to the connection: {e}"))?; + let sent = std::thread::spawn(move || -> Result<(), String> { + let all: Vec = (0..STREAM_GROWS_BYTES).map(stream_grows_byte).collect(); + sending.write_all(&all).map_err(|e| format!("the peer's send: {e}"))?; + sending.shutdown(std::net::Shutdown::Write).map_err(|e| format!("the peer's FIN: {e}")) + }); + let mut got = Vec::with_capacity(STREAM_GROWS_BYTES); + (&conn).read_to_end(&mut got).map_err(|e| format!("the job's bytes, until its FIN: {e} after {}", got.len()))?; + sent.join().map_err(|_| "the peer's sender panicked".to_string())??; + if got.len() != STREAM_GROWS_BYTES { + return Err(format!("the job sent {} bytes, not {STREAM_GROWS_BYTES}", got.len())); + } + match got.iter().enumerate().find(|&(at, &b)| b != stream_grows_byte(at)) { + Some((at, &b)) => Err(format!("the job's byte at {at} is {b:#04x}, not {:#04x}", stream_grows_byte(at))), + None => Ok(()), + } + }); + let result = qemu.run_test(&format!("test_rs_stream_grows {port}"), Duration::from_secs(300)); + if result.error.is_some() || result.exit_code != Some(0) { + return Err(format!("{:?}, ended {:?}; {}\nthe job said:\n{}", result.error, result.exit_code, peer_verdict(peer), result.stdout)); + } + peer.join().map_err(|_| "the peer's thread panicked".to_string())? +} + /// What the peer's thread had come to when its job failed: a guest that waits /// for a step the peer could not take is no stall of the guest's. fn peer_verdict(heard: std::thread::JoinHandle>) -> String { From 54c00092de5e2f8a2109c22347d3df28c94dca40 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 19:31:41 +0100 Subject: [PATCH 11/16] stream region: netstack's receive ring growth, through [tcp], on the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No test drove the receive ring's growth end to end: the node harness configured [tcp] with 64 KiB buffers, not netstack's 4 MiB, and its peer offered no window scale, so [tcp] never wanted more than the first ring. The harness now configures netstack's buffers, and its peer can scale windows (RFC 7323 §2.2) and read the node's scaled windows. The new test downloads with a reader that keeps up and a millisecond round trip: [tcp] wants more buffer, and netstack commits and doubles the receive ring twice, every byte read in place. Its client commits nothing, so every page either end touches is one netstack committed: with netstack's commit before the growth removed, the node's put past what was committed reds it. `stream_grows` on the guest claims the send ring's growth alone: QEMU's user network's round trips seldom make [tcp] want more. Co-Authored-By: Claude Opus 5.5 --- system/netstack/node/tests/common/client.rs | 12 +++- system/netstack/node/tests/common/mod.rs | 3 +- system/netstack/node/tests/streams.rs | 71 +++++++++++++++++-- .../guest/rust-tests/src/bin/stream_grows.rs | 33 +++++---- tests/toyos.rs | 4 +- 5 files changed, 98 insertions(+), 25 deletions(-) diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs index 541ffb32045..b58e250d2c3 100644 --- a/system/netstack/node/tests/common/client.rs +++ b/system/netstack/node/tests/common/client.rs @@ -77,6 +77,9 @@ pub struct Client { ringing: Ringing, rx: Held>, tx: Held>, + /// Its reads commit what a grown receive ring reaches, as the SDK's do; a client breaking + /// the protocol commits nothing, and reads only what netstack committed. + commits: Cell, } impl std::fmt::Debug for Region { @@ -179,7 +182,7 @@ impl Client { let bells = Rc::new(Doorbells::default()); region.holders.borrow_mut().push(bells.clone()); let ringing = Ringing { region: region.clone(), bells }; - Rc::new(Client { region: region.clone(), ringing, rx: Held(RefCell::new(rx)), tx: Held(RefCell::new(tx)) }) + Rc::new(Client { region: region.clone(), ringing, rx: Held(RefCell::new(rx)), tx: Held(RefCell::new(tx)), commits: Cell::new(true) }) } /// A client that set Nagle's algorithm off before it connected. @@ -189,11 +192,16 @@ impl Client { (client, session) } + /// From here on its reads commit nothing. + pub fn commit_nothing(&self) { + self.commits.set(false); + } + /// A read of at most `max` bytes that does not wait. pub fn read(&self, max: usize) -> Result, Refused> { let mut out = vec![0; max]; let region = &self.region; - let reach = |cap: u32| region.commit(RECEIVE_RING, cap as usize); + let reach = |cap: u32| !self.commits.get() || region.commit(RECEIVE_RING, cap as usize); let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), reach, |span, at| { assert!(region.reaches(span.offset, span.len), "a read of {span:?} past what was committed"); out[at..at + span.len].copy_from_slice(®ion.bytes.borrow()[span.offset..][..span.len]); diff --git a/system/netstack/node/tests/common/mod.rs b/system/netstack/node/tests/common/mod.rs index 36f54fad9dd..7d0e7b5f3cd 100644 --- a/system/netstack/node/tests/common/mod.rs +++ b/system/netstack/node/tests/common/mod.rs @@ -360,7 +360,8 @@ impl Wire { tcp: toyos_net_tcp::Secrets { isn: [3; 16], timestamp: [4; 16], port_offset: [5; 16], port_index: [6; 16], port_table: [0; 16] }, }; let now = Instant::from_millis(3_600_000); - let config = Config { receive_buffer: 65_535, send_buffer: 65_535, secrets }; + // netstack's own buffers: each a stream's ring at its most. + let config = Config { receive_buffer: toyos::net::stream::RECEIVE_BYTES, send_buffer: toyos::net::stream::SEND_BYTES, secrets }; let mut draws = 0x5a00_0000; let mut node = Node::new(now, config, HostName::new("toyos")).unwrap(); let link = node.add_link(now, IndividualMac::new(MacAddr(mac)).unwrap(), Rank(0), draw(&mut draws)); diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 4c349a3c0b6..160729ce5ee 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -90,9 +90,13 @@ struct Far { snd_nxt: u32, /// The node's next sequence number the peer has everything before. rcv_nxt: u32, - /// The node's last acknowledgment, and the window it came with. + /// The node's last acknowledgment, and the window it came with, scaled. acked: u32, - window: u16, + window: u32, + /// The peer offers window scaling in its SYN-ACK (RFC 7323 §2.2). + scales: bool, + /// The shift the node's SYN asked for, where both scale. + shift: u8, received: Vec, /// The node's FIN arrived, after everything before it. fin: bool, @@ -101,6 +105,18 @@ struct Far { segments: Vec, } +/// The window scale a SYN the node emitted asks for, or 0 where it asks none. +fn window_shift(frame: &[u8]) -> u8 { + let packet = SlicedPacket::from_ethernet(frame).expect("Ethernet II"); + let Some(TransportSlice::Tcp(tcp)) = &packet.transport else { panic!("a TCP segment") }; + tcp.options_iterator() + .find_map(|option| match option { + Ok(TcpOptionElement::WindowScale(shift)) => Some(shift), + _ => None, + }) + .unwrap_or(0) +} + fn v4(bytes: &[u8]) -> Ipv4Addr { Ipv4Addr::from(<[u8; 4]>::try_from(bytes).expect("four bytes")) } @@ -123,6 +139,8 @@ impl Far { rcv_nxt: 0, acked: 0, window: 0, + scales: false, + shift: 0, received: Vec::new(), fin: false, resets: Vec::new(), @@ -133,7 +151,9 @@ impl Far { /// A segment from the peer in its frame, built by `etherparse`: PSH with text. fn frame(&self, seq: u32, ack: Option, syn: bool, fin: bool, rst: bool, text: &[u8]) -> Vec { let mut step = PacketBuilder::ethernet2(self.mac, MAC).ipv4(self.addr.octets(), A.octets(), 64).tcp(PORT, self.port, seq, 65_535); - if syn { + if syn && self.scales { + step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460), TcpOptionElement::WindowScale(7)]).unwrap(); + } else if syn { step = step.syn().options(&[TcpOptionElement::MaximumSegmentSize(1460)]).unwrap(); } if let Some(ack) = ack { @@ -174,7 +194,7 @@ impl Far { /// How many bytes the node's window still lets the peer send. fn open(&self) -> usize { - self.acked.wrapping_add(u32::from(self.window)).wrapping_sub(self.snd_nxt) as usize + self.acked.wrapping_add(self.window).wrapping_sub(self.snd_nxt) as usize } /// The outside reading of a frame the node emitted, and the frames it is answered with. @@ -193,6 +213,7 @@ impl Far { self.port = segment.from_port; self.received.clear(); self.fin = false; + self.shift = if self.scales { window_shift(frame) } else { 0 }; } else if segment.from_port != self.port { return Vec::new(); } @@ -221,7 +242,9 @@ impl Far { return Vec::new(); } if let Some(ack) = segment.ack { - (self.acked, self.window) = (ack, segment.window); + // A SYN's window is never scaled (RFC 7323 §2.2). + let shift = if segment.syn { 0 } else { self.shift }; + (self.acked, self.window) = (ack, u32::from(segment.window) << shift); } match self.manner { Manner::Deaf => return Vec::new(), @@ -1162,3 +1185,41 @@ fn an_upload_larger_than_its_send_ring_grows_it_to_its_place() { let heard = &net.far.received; assert!(!heard.is_empty() && heard.iter().enumerate().all(|(i, &b)| b == byte(i)), "the peer's stream is the client's"); } + +// RFC 7323 §2: a peer that scales windows lets [tcp] want a receive buffer past the receive +// ring's first size as the client keeps reading a round trip's worth; the node commits the +// region below twice the ring and doubles it, and the client reads every byte in place across +// each switch. Each round trip the peer fills the window it was offered over a millisecond, the +// round trip [tcp] measures, and the client reads all of it. The client commits nothing itself, +// so every page either end touches is one the node committed. +#[test] +fn a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte() { + use toyos::net::stream::{RECEIVE_INITIAL, RECEIVE_RING}; + const GROWN: usize = 4 * RECEIVE_INITIAL as usize; + let mut net = Net::new(); + net.far.scales = true; + let (id, client) = net.connect(None); + client.commit_nothing(); + let local = Port::new(net.far.port).expect("the SYN's source port"); + assert_eq!(net.events(), [StreamEvent::Connected { id, local }]); + assert!(net.far.shift > 0, "the node's SYN asked to scale its window"); + let sent = text(16 << 20); + let (mut offset, mut got) = (0, Vec::new()); + for _ in 0..1_000 { + if client.region.committed(RECEIVE_RING) >= GROWN { + break; + } + while let len @ 1.. = net.far.open().min(1_460).min(sent.len() - offset) { + net.now = net.now.after(Duration::from_micros(20)); + let frame = net.far.text(&sent[offset..offset + len]); + offset += len; + net.deliver(&frame); + } + net.now = net.now.after(Duration::from_millis(1)); + got.extend(inbox(&client)); + net.bridge(); + } + assert!(client.region.committed(RECEIVE_RING) >= GROWN, "the receive ring after {offset} bytes: {}", client.region.committed(RECEIVE_RING)); + got.extend(inbox(&client)); + assert!(got.len() > GROWN && got[..] == sent[..got.len()], "the client's stream is the peer's: {} bytes", got.len()); +} diff --git a/tests/guest/rust-tests/src/bin/stream_grows.rs b/tests/guest/rust-tests/src/bin/stream_grows.rs index 92282a941f7..ed8532880d5 100644 --- a/tests/guest/rust-tests/src/bin/stream_grows.rs +++ b/tests/guest/rust-tests/src/bin/stream_grows.rs @@ -1,18 +1,21 @@ //! A stream carries 4 MiB each way, every byte where it was written, while -//! its rings grow: the send ring at the first write its room cannot take, -//! here the third, and the receive ring as [tcp] asks for more buffer, every -//! page committed by the end that touches it before it does. A missing -//! commit ends this process or netstack; a byte laid out by the wrong size -//! arrives out of place. +//! this end's send ring grows, at the first write its room cannot take, here +//! the third: each page committed by this process before it writes it, and +//! by netstack before it reads it. A missing commit ends this process or +//! netstack; a byte laid out by the wrong size arrives out of place. The +//! receive ring grows only as [tcp] wants more buffer, which QEMU's user +//! network's round trips seldom ask for: the node's host test +//! `a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte` +//! holds that growth. //! -//! It needs the guest: the commits are the kernel's, and the receive ring's -//! growth is the real netstack's. ToyOS has no loopback, so the harness is -//! the peer, as for `socket_export`: this job's argument is the port of its -//! datagram socket, which QEMU's user network reaches as `10.0.2.2`. The job -//! listens on the port the boot forwards, says it is ready in a datagram, -//! accepts the harness's connection, and both ends send [`BYTES`] and check -//! what they receive. `netstack_holders_exported_to_children`'s boot runs it, -//! on tests/images/netcase.toml. +//! It needs the guest: the commits and the refused touch are the kernel's. +//! ToyOS has no loopback, so the harness is the peer, as for `socket_export`: +//! this job's argument is the port of its datagram socket, which QEMU's user +//! network reaches as `10.0.2.2`. The job listens on the port the boot +//! forwards, says it is ready in a datagram, accepts the harness's +//! connection, and both ends send [`BYTES`] and check what they receive. +//! `netstack_holders_exported_to_children`'s boot runs it, on +//! tests/images/netcase.toml. use std::io::{Read, Write}; use std::net::{Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream, UdpSocket}; @@ -76,6 +79,6 @@ fn main() { scope.spawn(|| send(&stream)); receive(&stream); }); - println!(" PASS: 4 MiB each way arrived in order through the rings' growth"); - println!("stream_grows: a stream's rings grow under load and keep every byte in place"); + println!(" PASS: 4 MiB each way arrived in order through the send ring's growth"); + println!("stream_grows: a stream's send ring grows under load and keeps every byte in place"); } diff --git a/tests/toyos.rs b/tests/toyos.rs index e14a30b55a7..208f0580cea 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -260,7 +260,7 @@ const RUST_SKIP: &[&str] = &[ // A connect whose region nothing committed, to real netstack, which goes // on serving: `netstack_holders_exported_to_children`'s boot runs it. "stream_region_refused", - // A stream's bytes through its rings' growth, the harness its peer: + // A stream's bytes through its send ring's growth, the harness its peer: // `netstack_holders_exported_to_children`'s boot runs it. "stream_grows", // A stream's holders across processes, as their own netstack: it spawns a @@ -5817,7 +5817,7 @@ const SOCKET_EXPORT_PORT: u16 = 7070; /// child's line, then its parent's, then the FIN once both holders are gone; /// then a listener's and a datagram socket's child's datagram, its accept and /// its line, and the parent's after it; then a connect whose region nothing -/// committed, and a stream to the harness through its rings' growth. Once over +/// committed, and a stream to the harness through its send ring's growth. Once over /// virtio, and once over QEMU's `e1000e`, which the image's rows name no /// machine for: the I219 driver's row matches it, and its link is the one /// netstack opens. From 99f5c3b145546979f87c0f9da6d0cf47f8882c73 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 19:36:46 +0100 Subject: [PATCH 12/16] stream region: each producer's commit alone holds its growth, on the host Both ends commit what a grown ring reaches, so in an honest exchange either end's commit hides the other's absence: the guest showed it, with the client's commits removed and `stream_grows` still green. The node harness's region now says whose commits make memory. The upload test counts the client's alone and the download test netstack's alone: with the client's commit before its growth removed, the node's read past what was committed reds the first, and with netstack's, its put reds the second. `stream_grows` claims what only the guest has: netstack's own mapping read past the ring's first size, over pages the kernel committed after both ends mapped it. Co-Authored-By: Claude Opus 5.5 --- system/netstack/node/tests/common/client.rs | 40 +++++++++++++------ system/netstack/node/tests/streams.rs | 6 ++- .../guest/rust-tests/src/bin/stream_grows.rs | 16 ++++---- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs index b58e250d2c3..56c0f24f555 100644 --- a/system/netstack/node/tests/common/client.rs +++ b/system/netstack/node/tests/common/client.rs @@ -18,6 +18,9 @@ pub struct Region { bytes: RefCell>, /// Each commit's range, the client's and the node's. committed: RefCell>, + /// Whether the client's and the node's commits make memory: one that makes none leaves + /// every page the other's to commit, so a test shows the other's commits alone suffice. + commits: Cell<(bool, bool)>, holders: RefCell>>, /// A holder rang netstack since the test last asked. rang: Cell, @@ -77,9 +80,6 @@ pub struct Client { ringing: Ringing, rx: Held>, tx: Held>, - /// Its reads commit what a grown receive ring reaches, as the SDK's do; a client breaking - /// the protocol commits nothing, and reads only what netstack committed. - commits: Cell, } impl std::fmt::Debug for Region { @@ -100,12 +100,32 @@ impl Memory for Region { } fn commit(&self, at: usize, len: usize) -> bool { - self.committed.borrow_mut().push((at, at + len)); + if self.commits.get().1 { + self.committed.borrow_mut().push((at, at + len)); + } true } } impl Region { + /// The client's commit of `[at, at + len)`. + fn client_commit(&self, at: usize, len: usize) -> bool { + if self.commits.get().0 { + self.committed.borrow_mut().push((at, at + len)); + } + true + } + + /// From here on the client's commits make no memory. + pub fn client_commits_nothing(&self) { + self.commits.set((false, self.commits.get().1)); + } + + /// From here on the node's commits make no memory. + pub fn netstack_commits_nothing(&self) { + self.commits.set((self.commits.get().0, false)); + } + /// Whether `[at, at + len)` lies in what a commit made memory, as a fault would ask. fn reaches(&self, at: usize, len: usize) -> bool { self.committed.borrow().iter().any(|&(start, end)| start <= at && at + len <= end) @@ -161,6 +181,7 @@ impl Client { words: Box::new(std::array::from_fn(|_| AtomicU32::new(0))), bytes: RefCell::new(vec![0; REGION_BYTES]), committed: RefCell::new(vec![(0, RECEIVE_RING + RECEIVE_INITIAL as usize), (SEND_RING, SEND_RING + SEND_INITIAL as usize)]), + commits: Cell::new((true, true)), holders: RefCell::new(Vec::new()), rang: Cell::new(false), dropped: Cell::new(false), @@ -182,7 +203,7 @@ impl Client { let bells = Rc::new(Doorbells::default()); region.holders.borrow_mut().push(bells.clone()); let ringing = Ringing { region: region.clone(), bells }; - Rc::new(Client { region: region.clone(), ringing, rx: Held(RefCell::new(rx)), tx: Held(RefCell::new(tx)), commits: Cell::new(true) }) + Rc::new(Client { region: region.clone(), ringing, rx: Held(RefCell::new(rx)), tx: Held(RefCell::new(tx)) }) } /// A client that set Nagle's algorithm off before it connected. @@ -192,16 +213,11 @@ impl Client { (client, session) } - /// From here on its reads commit nothing. - pub fn commit_nothing(&self) { - self.commits.set(false); - } - /// A read of at most `max` bytes that does not wait. pub fn read(&self, max: usize) -> Result, Refused> { let mut out = vec![0; max]; let region = &self.region; - let reach = |cap: u32| !self.commits.get() || region.commit(RECEIVE_RING, cap as usize); + let reach = |cap: u32| region.client_commit(RECEIVE_RING, cap as usize); let n = session::read(®ion.words, &self.rx, &self.ringing, Mode::Nonblocking, u32::try_from(max).unwrap(), reach, |span, at| { assert!(region.reaches(span.offset, span.len), "a read of {span:?} past what was committed"); out[at..at + span.len].copy_from_slice(®ion.bytes.borrow()[span.offset..][..span.len]); @@ -225,7 +241,7 @@ impl Client { /// A write that does not wait: how much of `data` the send ring took. pub fn write(&self, data: &[u8]) -> Result { let region = &self.region; - let reach = |cap: u32| region.commit(SEND_RING, cap as usize); + let reach = |cap: u32| region.client_commit(SEND_RING, cap as usize); let n = session::write(®ion.words, &self.tx, &self.ringing, Mode::Nonblocking, u32::try_from(data.len()).unwrap(), reach, |span, at| { assert!(region.reaches(span.offset, span.len), "a write of {span:?} past what was committed"); region.bytes.borrow_mut()[span.offset..][..span.len].copy_from_slice(&data[at..at + span.len]); diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 160729ce5ee..82c5ac66d1b 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -1160,7 +1160,8 @@ fn the_last_holder_leaving_is_the_streams_close() { // A client that writes more than its send ring holds doubles the ring at each write the room // cannot take whole, to its place, and netstack reads the grown ring by the size the client -// stored: the peer's stream is the client's, byte for byte. +// stored: the peer's stream is the client's, byte for byte. Netstack's commits make no memory +// here, so every page either end touches is one the client committed. #[test] fn an_upload_larger_than_its_send_ring_grows_it_to_its_place() { use toyos::net::stream::{SEND_BYTES, SEND_RING}; @@ -1169,6 +1170,7 @@ fn an_upload_larger_than_its_send_ring_grows_it_to_its_place() { const WRITE: usize = 2 * SEND_BYTES as usize; let pattern: Vec = (0..WRITE + PERIOD).map(byte).collect(); let (mut net, _, client) = established(); + client.region.netstack_commits_nothing(); let mut offset = 0; for _ in 0..1_000 { if client.region.committed(SEND_RING) == SEND_BYTES as usize { @@ -1199,7 +1201,7 @@ fn a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte let mut net = Net::new(); net.far.scales = true; let (id, client) = net.connect(None); - client.commit_nothing(); + client.region.client_commits_nothing(); let local = Port::new(net.far.port).expect("the SYN's source port"); assert_eq!(net.events(), [StreamEvent::Connected { id, local }]); assert!(net.far.shift > 0, "the node's SYN asked to scale its window"); diff --git a/tests/guest/rust-tests/src/bin/stream_grows.rs b/tests/guest/rust-tests/src/bin/stream_grows.rs index ed8532880d5..3269730cf0e 100644 --- a/tests/guest/rust-tests/src/bin/stream_grows.rs +++ b/tests/guest/rust-tests/src/bin/stream_grows.rs @@ -1,14 +1,14 @@ //! A stream carries 4 MiB each way, every byte where it was written, while //! this end's send ring grows, at the first write its room cannot take, here -//! the third: each page committed by this process before it writes it, and -//! by netstack before it reads it. A missing commit ends this process or -//! netstack; a byte laid out by the wrong size arrives out of place. The -//! receive ring grows only as [tcp] wants more buffer, which QEMU's user -//! network's round trips seldom ask for: the node's host test -//! `a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte` -//! holds that growth. +//! the third: netstack's mapping of the region read past the ring's first +//! size, over pages the kernel committed after both ends mapped it. A page +//! neither end committed ends this process or netstack; a byte laid out by +//! the wrong size arrives out of place. Which end's commit suffices, and the +//! receive ring's growth, which QEMU's user network's round trips seldom +//! make [tcp] want, are the node's host tests'. //! -//! It needs the guest: the commits and the refused touch are the kernel's. +//! It needs the guest: the commits and the mappings are the kernel's, and the +//! region's adapter is netstack's own. //! ToyOS has no loopback, so the harness is the peer, as for `socket_export`: //! this job's argument is the port of its datagram socket, which QEMU's user //! network reaches as `10.0.2.2`. The job listens on the port the boot From f6363b9d03edf2d7942ebd34bdb0120e4cd694f8 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 20:31:42 +0100 Subject: [PATCH 13/16] review r2: netstack's and the client's consumer commits are pinned, and a joined writer commits what it resumed B1: no test failed with netstack's commit of a client-grown send ring removed, which is all that keeps a client from faulting netstack. A client that claims its send ring grown to 4 MiB without committing a page, and publishes 100,000 bytes, is now read only through netstack's own commit, and the peer hears a fresh commit's zeros. With that commit removed, the harness's take past what was committed reds it. S1: a holder that joined copied by the size the last holder stored without committing it, so a co-holder's bogus size ended the process. It now commits a resumed capacity it has not reached before it copies, and one it cannot commit is Violation::Size, as a read's already was. S2: the client's own commit before it reads by a size netstack stored is pinned the same way, netstack's commits switched off. S3: a size netstack's commit is refused for resets that stream alone, before [tcp] reads by it. S4: a shape's `holds` says what each ring uses it as. Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/src/transport/session.rs | 36 ++++++++++-- system/netstack/node/src/streams.rs | 3 +- system/netstack/node/tests/common/client.rs | 11 ++++ system/netstack/node/tests/streams.rs | 62 +++++++++++++++++++++ 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/sdk/toyos/src/transport/session.rs b/sdk/toyos/src/transport/session.rs index 8a7c39608be..ad54bd766d2 100644 --- a/sdk/toyos/src/transport/session.rs +++ b/sdk/toyos/src/transport/session.rs @@ -877,10 +877,13 @@ where /// Writes at most `len` bytes, each span copied in by `copy` from its offset /// into the caller's buffer: the count, at least one. A shutdown, the server's /// hang-up or a failure it stored is answered, and nothing copied, whatever -/// room there is. A write of no bytes answers 0 at once. A write the room -/// cannot take whole, or that fills the ring, doubles the ring, up to its -/// place, where `reach` makes the ring's first twice its capacity memory; -/// where it cannot, the ring stays as it is. +/// room there is. A write of no bytes answers 0 at once. Before it copies +/// past a capacity this process has not reached, as another holder may have +/// grown the ring to, `reach` makes the ring's first that many bytes memory; +/// one it cannot is [`Violation::Size`]. A write the room cannot take whole, +/// or that fills the ring, doubles the ring, up to its place, where `reach` +/// makes the ring's first twice its capacity memory; where it cannot, the +/// ring stays as it is. pub fn write( page: &[W; N], lock: &L, @@ -904,6 +907,14 @@ where Writable::Failed(end) => (Err(Refused::Failed(end)), false), Writable::Room(room) => { let n = room.min(len); + // A size another holder stored is believed only once it is memory. + let resumed = half.ring.capacity(); + if resumed > half.reached { + if !reach(resumed) { + return (Err(Refused::Violation(Violation::Size)), false); + } + half.reached = resumed; + } let Some(spans) = half.ring.span(0, n) else { return (Err(Refused::Violation(Violation::HeadPastTail)), false); }; @@ -1586,6 +1597,23 @@ mod tests { /// Once one holder saw the server hang up, a write with room in another /// answers it and copies nothing, and a read answers it after the bytes. + /// A holder that joined believes the size the last holder stored only + /// once it made that much of the ring memory: a size it cannot commit, + /// which another holder may have stored without committing, is + /// `Violation::Size`, and nothing is copied. + #[test] + fn a_joined_writer_commits_the_size_it_resumed_before_it_copies() { + let page = page(); + let ((_, _), (_, other_tx)) = two(&page); + let bells = quiet(); + let mut copied = 0; + let refused = write(&page, &other_tx, &bells, Mode::Nonblocking, 1, |_| false, |span, _| copied += span.len); + assert_eq!((refused, copied), (Err(Refused::Violation(Violation::Size)), 0)); + let mut reached = Vec::new(); + let wrote = write(&page, &other_tx, &bells, Mode::Nonblocking, 1, |cap| { reached.push(cap); true }, |_, _| {}); + assert_eq!((wrote, reached), (Ok(1), vec![C]), "the resumed capacity, committed once"); + } + #[test] fn a_hang_up_one_holder_saw_is_every_holders() { let page = page(); diff --git a/system/netstack/node/src/streams.rs b/system/netstack/node/src/streams.rs index e4572e02a04..7d48234b777 100644 --- a/system/netstack/node/src/streams.rs +++ b/system/netstack/node/src/streams.rs @@ -75,7 +75,8 @@ pub trait Memory: core::fmt::Debug { #[derive(Clone, Copy, Debug)] struct Shape { size: Size, - /// What it holds from the oldest byte unread. + /// [tcp]'s storage length: the receive ring's capacity from its oldest byte unread, and the + /// most of the send ring netstack committed. holds: u32, /// What [tcp] last asked it to hold. wanted: usize, diff --git a/system/netstack/node/tests/common/client.rs b/system/netstack/node/tests/common/client.rs index 56c0f24f555..559b6544518 100644 --- a/system/netstack/node/tests/common/client.rs +++ b/system/netstack/node/tests/common/client.rs @@ -21,6 +21,8 @@ pub struct Region { /// Whether the client's and the node's commits make memory: one that makes none leaves /// every page the other's to commit, so a test shows the other's commits alone suffice. commits: Cell<(bool, bool)>, + /// The node's commits are refused, as the kernel refuses one past the job's memory. + refused: Cell, holders: RefCell>>, /// A holder rang netstack since the test last asked. rang: Cell, @@ -100,6 +102,9 @@ impl Memory for Region { } fn commit(&self, at: usize, len: usize) -> bool { + if self.refused.get() { + return false; + } if self.commits.get().1 { self.committed.borrow_mut().push((at, at + len)); } @@ -126,6 +131,11 @@ impl Region { self.commits.set((self.commits.get().0, false)); } + /// From here on the node's commits are refused. + pub fn refuse_netstacks_commits(&self) { + self.refused.set(true); + } + /// Whether `[at, at + len)` lies in what a commit made memory, as a fault would ask. fn reaches(&self, at: usize, len: usize) -> bool { self.committed.borrow().iter().any(|&(start, end)| start <= at && at + len <= end) @@ -182,6 +192,7 @@ impl Client { bytes: RefCell::new(vec![0; REGION_BYTES]), committed: RefCell::new(vec![(0, RECEIVE_RING + RECEIVE_INITIAL as usize), (SEND_RING, SEND_RING + SEND_INITIAL as usize)]), commits: Cell::new((true, true)), + refused: Cell::new(false), holders: RefCell::new(Vec::new()), rang: Cell::new(false), dropped: Cell::new(false), diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 82c5ac66d1b..36765f494c9 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -1002,6 +1002,68 @@ fn a_tail_past_the_send_rings_room_resets_that_stream() { assert_eq!((client.dropped(), net.node.streams(), net.far.resets.len()), (true, 0, 1)); } +/// The send ring's size word, which only the client writes. +const TX_SIZE: usize = 16 * 16; + +/// A client that claims its send ring grown to its place without committing a page of it, and +/// publishes `len` bytes, past the ring's first size. +fn grown_without_a_commit(client: &Client, len: usize) { + client.region.client_commits_nothing(); + client.word(TX_SIZE).store(toyos::net::stream::SEND_BYTES.trailing_zeros(), Ordering::Release); + client.word(TX_TAIL).store(u32::try_from(len).unwrap(), Ordering::Release); + client.region_ring(); +} + +// netstack commits what a client's size claims before [tcp] reads by it: every read stays inside +// netstack's own commits, and the peer hears the bytes, a fresh commit's zeros, past the first +// size. +#[test] +fn a_send_ring_a_client_grew_without_committing_is_read_through_netstacks_own_commit() { + let (mut net, id, client) = established(); + const LEN: usize = 100_000; + grown_without_a_commit(&client, LEN); + for _ in 0..1_000 { + if net.far.received.len() == LEN { + break; + } + let before = net.far.received.len(); + net.bridge(); + if net.far.received.len() == before { + let at = net.node.next_deadline().expect("a timer, if nothing else"); + net.fire(at); + } + } + assert!(net.far.received == vec![0; LEN], "the peer heard {} bytes", net.far.received.len()); + assert!(net.events().iter().all(|event| !matches!(event, StreamEvent::Violated { id: v, .. } if *v == id))); +} + +// A size netstack cannot make memory, the kernel refusing the commit, is a violation of that +// stream alone, before [tcp] reads a byte by it. +#[test] +fn a_send_ring_size_netstack_cannot_commit_resets_that_stream() { + let (mut net, id, client) = established(); + client.region.refuse_netstacks_commits(); + grown_without_a_commit(&client, 100_000); + net.bridge(); + assert_eq!(net.events(), [StreamEvent::Violated { id, violation: Violation::Size }]); + assert_eq!((client.dropped(), net.node.streams(), net.far.received.len()), (true, 0, 0)); +} + +// The client, as the receive ring's consumer, commits what a size claims before it reads by it: +// a size netstack stored without committing, here written in its place, is read only through +// the client's own commit. +#[test] +fn a_receive_ring_grown_without_netstacks_commit_is_read_through_the_clients_own() { + const RX_TAIL: usize = 16; + const RX_SIZE: usize = 15 * 16; + const LEN: usize = 100_000; + let (_net, _, client) = established(); + client.region.netstack_commits_nothing(); + client.word(RX_SIZE).store(toyos::net::stream::RECEIVE_BYTES.trailing_zeros(), Ordering::Release); + client.word(RX_TAIL).store(u32::try_from(LEN).unwrap(), Ordering::Release); + assert!(inbox(&client) == vec![0; LEN]); +} + // Text published after the client's own shutdown is past the FIN [tcp] queued: a violation. #[test] fn text_published_after_the_shutdown_resets_that_stream() { From a97ce1204d17e2d9639389b56d6754bb69d5c058 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 23:49:29 +0100 Subject: [PATCH 14/16] space: a syscall's window is judged committed in the one descent that judges it covered The A/B at 342ecc142 read every bench's syscall-heavy rows slower at the head, resolved on both: fs.mem.fstat +7% on the T14 and +15% on the AMD bench, pipe.throughput.64 -7%, fs.disk.pread_4k +4 to +6%. `allows`, which every user window asks first, had gained a second walk of the inventory, `meeting`'s two descents, to judge an object's pages committed, even for a window in private memory. `covered_where` now hands its predicate each entry's start, and `allows` asks an object entry for its pages in the descent it already made; a private entry costs nothing more. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/vm.rs | 11 ++++++----- kernel/src/mm/space.rs | 26 +++++++++++--------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index 79fa1b181f7..64979150e89 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -214,18 +214,19 @@ impl Inventory { /// Whether entries cover every page of `[start, end)`. pub fn covered(&self, start: u64, end: u64) -> bool { - self.covered_where(start, end, |_| true) + self.covered_where(start, end, |_, _| true) } /// Whether entries cover every page of `[start, end)` and each that meets - /// it answers `ok`: one descent of the tree, walking down from `end`. - pub fn covered_where(&self, start: u64, end: u64, ok: impl Fn(&Entry) -> bool) -> bool { + /// it answers `ok`, given where it starts: one descent of the tree, + /// walking down from `end`. + pub fn covered_where(&self, start: u64, end: u64, ok: impl Fn(u64, &Entry) -> bool) -> bool { // `[low, end)` is covered by entries that answered `ok`. let mut low = end; let mut below = self.entries.range(..end).rev(); while low > start { let Some((&s, entry)) = below.next() else { return false }; - if entry.end < low || !ok(entry) { + if entry.end < low || !ok(s, entry) { return false; } low = s; @@ -574,7 +575,7 @@ mod tests { let pages = &map[page(start)..page(end)]; assert_eq!(v.covered(start, end), pages.iter().all(Option::is_some), "round {round}: covered"); assert_eq!( - v.covered_where(start, end, |e| e.prot == Prot::ReadWrite), + v.covered_where(start, end, |_, e| e.prot == Prot::ReadWrite), pages.iter().all(|p| p.is_some_and(|(prot, _)| prot == Prot::ReadWrite)), "round {round}: covered by writable entries" ); diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index ffad7153008..72b8b5b8253 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -850,23 +850,19 @@ impl AddressSpace { /// Whether every page of `[start, end)` is in a range that allows /// `access` and is memory the kernel can copy through the direct map: /// what a copy the kernel makes for a thread asks first. A device's - /// registers or firmware's framebuffer, an aperture, are no buffer. + /// registers or firmware's framebuffer, an aperture, are no buffer. A + /// page an object maps is one only once its object committed it: a copy + /// into one that is not would be lost, and a fault on it is refused. pub fn allows(&self, start: u64, end: u64, access: Access) -> bool { let start = start & !(PAGE_SIZE - 1); - self.inventory.covered_where(start, end, |entry| entry.prot.allows(access) && entry.backed() && copyable(entry)) - && self.committed(start, end) - } - - /// Whether every page of `[start, end)` an object maps is one its - /// object committed: a copy into one that is not would be lost, and a - /// fault on it is refused. - fn committed(&self, start: u64, end: u64) -> bool { - self.inventory.meeting(start, end).all(|(at, entry)| match &entry.kind { - Kind::Object { object, offset, .. } => { - let (lo, hi) = (start.max(at), end.min(entry.end)); - (lo..hi).step_by(PAGE_SIZE as usize).all(|page| object.page(offset + (page - at)).is_some()) - } - Kind::Private { .. } | Kind::Copy { .. } => true, + self.inventory.covered_where(start, end, |at, entry| { + let committed = || match &entry.kind { + Kind::Object { object, offset, .. } => (start.max(at)..end.min(entry.end)) + .step_by(PAGE_SIZE as usize) + .all(|page| object.page(offset + (page - at)).is_some()), + Kind::Private { .. } | Kind::Copy { .. } => true, + }; + entry.prot.allows(access) && entry.backed() && copyable(entry) && committed() }) } From 1ff5a6ff5eed8d078fff3b9198990a1932304f3c Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 3 Nov 2026 20:25:54 +0100 Subject: [PATCH 15/16] stream region: the download test names its local endpoint, as #1147 made it Co-Authored-By: Claude Opus 5.5 --- system/netstack/node/tests/streams.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/netstack/node/tests/streams.rs b/system/netstack/node/tests/streams.rs index 36765f494c9..781d805f9f3 100644 --- a/system/netstack/node/tests/streams.rs +++ b/system/netstack/node/tests/streams.rs @@ -1264,7 +1264,7 @@ fn a_download_a_reader_keeps_up_with_grows_the_receive_ring_and_keeps_every_byte net.far.scales = true; let (id, client) = net.connect(None); client.region.client_commits_nothing(); - let local = Port::new(net.far.port).expect("the SYN's source port"); + let local = Endpoint { addr: A, port: Port::new(net.far.port).expect("the SYN's source port") }; assert_eq!(net.events(), [StreamEvent::Connected { id, local }]); assert!(net.far.shift > 0, "the node's SYN asked to scale its window"); let sent = text(16 << 20); From 2798d7fa229d4fee4c2d13b0691b60c0e3bf2f93 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 3 Nov 2026 21:10:11 +0100 Subject: [PATCH 16/16] shm: a syscall's window over an object committed whole asks one word, not each page `Space::allows` asked an object for each page of a window, so a read of a program image into its region, or a server's I/O through its shared buffers, paid a look at every page it spanned for a property that holds of every object but a stream region. An object now counts its pages not yet committed, and one with none missing answers at once; a partly committed one still answers page by page. Co-Authored-By: Claude Opus 5.5 --- kernel/src/mm/space.rs | 7 ++++--- kernel/src/object/shm.rs | 27 ++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 72b8b5b8253..7256e79374c 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -857,9 +857,10 @@ impl AddressSpace { let start = start & !(PAGE_SIZE - 1); self.inventory.covered_where(start, end, |at, entry| { let committed = || match &entry.kind { - Kind::Object { object, offset, .. } => (start.max(at)..end.min(entry.end)) - .step_by(PAGE_SIZE as usize) - .all(|page| object.page(offset + (page - at)).is_some()), + Kind::Object { object, offset, .. } => { + let (lo, hi) = (start.max(at), end.min(entry.end)); + object.committed(offset + (lo - at), hi - lo) + } Kind::Private { .. } | Kind::Copy { .. } => true, }; entry.prot.allows(access) && entry.backed() && copyable(entry) && committed() diff --git a/kernel/src/object/shm.rs b/kernel/src/object/shm.rs index d0de724df06..888f43f56db 100644 --- a/kernel/src/object/shm.rs +++ b/kernel/src/object/shm.rs @@ -73,12 +73,20 @@ pub struct SharedMemObject { backing: Backing, size: u64, cache: CachePolicy, + /// [`Backing::Frames`]' pages not yet committed; 0 for every other + /// backing. A syscall's window over an object asks it, so one committed + /// whole answers without a look at each page. + missing: AtomicU64, } impl SharedMemObject { /// Over `core`, charged with what `backing` keeps on the heap. fn new(core: ObjectCore, backing: Backing, size: u64, cache: CachePolicy) -> Arc { - Arc::new(Self { core, backing, size, cache }) + let missing = match &backing { + Backing::Frames(_) => AtomicU64::new(size / PAGE_SIZE), + _ => AtomicU64::new(0), + }; + Arc::new(Self { core, backing, size, cache, missing }) } /// `size` bytes of zeroed memory in whole pages, frame by frame, counted @@ -143,9 +151,13 @@ impl SharedMemObject { let frame = pmm::alloc_promised().into_raw(); drawn += 1; // Release: a fault that reads the frame finds it zeroed. - if word.compare_exchange(0, frame, Ordering::Release, Ordering::Acquire).is_err() { + match word.compare_exchange(0, frame, Ordering::Release, Ordering::Acquire) { + // Release: whoever reads none missing finds every frame stored. + Ok(_) => { + self.missing.fetch_sub(1, Ordering::Release); + } // SAFETY: drawn just now and named by nothing. - unsafe { pmm::free_drawn(self.core.job(), [frame]) }; + Err(_) => unsafe { pmm::free_drawn(self.core.job(), [frame]) }, } } pmm::unpromise(self.core.job(), wanted - drawn); @@ -251,6 +263,15 @@ impl SharedMemObject { } } + /// Whether every page `[offset, offset + len)` touches is committed. + pub fn committed(&self, offset: u64, len: u64) -> bool { + match &self.backing { + Backing::Sub { whole, offset: base, .. } => whole.committed(base + offset, len), + _ if self.missing.load(Ordering::Acquire) == 0 => true, + _ => (offset & !(PAGE_SIZE - 1)..offset + len).step_by(PAGE_SIZE as usize).all(|at| self.page(at).is_some()), + } + } + /// The physical address of the page `offset` bytes in, which lies inside /// it; `None` while that page is not committed. pub fn page(&self, offset: u64) -> Option {