From 91b68e796911465538945406f123ba2259dc5089 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 31 Oct 2026 18:01:58 +0100 Subject: [PATCH 1/5] a connection carries whole messages Each direction of a connection is a kernel message queue: the bytes of its messages in the frames #1074 takes as a pipe fills, and beside them the messages' lengths and their handles, in the order they were sent. A send admits the whole message under the queue's lock or refuses it with nothing of it queued and every handle back at its number; a receive takes the head message whole into the caller's buffers, its handles installed under the same lock, or leaves it queued. No stream position exists any more, so nothing pins a batch of handles at one. The bounds are the approved ones, per direction: 2 MiB of bytes, taken as used, 256 messages and 64 handles. Without room the send waits, or answers WouldBlock; a receiver's buffer too small for the head message is InvalidArgument and a table without room for its handles is ResourceExhausted, both leaving it queued; a peer gone is Gone. A connection polls writable once the largest message would fit, so a refused non-blocking send retried on readiness lands. ABI: SYS_MESSAGE_RECV takes one message. SYS_HANDLE_RECV, HandlesPending and MAX_QUEUED_BATCHES are gone, and plain read and write on a connection are refused PermissionDenied: a connection is never a byte stream. SDK: a frame is a 4-byte message type and its payload, the kernel reporting length and handle count, so IpcHeader goes. FrameRx is a buffer and one receive, non-blocking (pump) or blocking (recv), and replaces recv_header/recv_payload/recv_bytes/recv_handles and the partial-frame state machine. A send frames into an uninitialised buffer instead of a zeroed 8 KiB one. Every consumer moves to it in this commit. netstack's Client::gone asks with a non-blocking receive instead of a one-byte read. The link (#1106) rang its doorbell as a byte on its connection, the last such doorbell; it rings an event each way now, as the block service's do. The hello carries the bell the driver rings, the answer the bell the stack rings, and a bell whose waker is gone is the link's end, so the connection goes once the link is open. Deleted: kernel/pure/message.rs's batches and windows, HandleQueue, AtBatch, try_read_within, the pipe's written/read counters and Landing (a pipe's write landing is its own rule now), the connection arms of the pipe read and write paths, the acceptor's close_now of each queued connection (a pending connection's receiver goes with it). Tests: the pure queue (order, wholeness, each bound, a refused receive taking nothing, a 50,000-step random run against the order sent); the SDK's framing and refusals; messages_are_whole, replacing message_carries_its_handles, for the syscalls' refusals and the bounds as a sender meets them. Co-Authored-By: Claude Opus 5.5 --- ...on-the-compositors-answer-with-no-bound.md | 2 +- ...ns-a-user-copy-and-its-first-allocation.md | 24 +- .../a-received-handle-has-no-knowable-type.md | 18 +- .../a-tcp-connect-zeroes-twelve-mebibytes.md | 4 +- ...ves-every-byte-through-two-kernel-pipes.md | 2 +- ...-capability-end-state-is-twelve-answers.md | 2 +- kernel/pure/message.rs | 360 +++++----- kernel/pure/pipe.rs | 94 +-- kernel/src/frame_bytes.rs | 176 +++++ kernel/src/main.rs | 2 + kernel/src/message.rs | 251 +++++++ kernel/src/object/ops.rs | 51 +- kernel/src/object/port.rs | 20 +- kernel/src/object/service.rs | 186 +---- kernel/src/pipe.rs | 221 +----- kernel/src/syscall/dispatch.rs | 76 ++- kernel/src/syscall/io.rs | 18 +- kernel/src/syscall/ipc.rs | 165 +++-- lib/link/src/driver.rs | 86 ++- lib/link/src/lib.rs | 9 +- lib/link/src/wire.rs | 16 +- lib/logkeeper-api/src/lib.rs | 13 +- lib/session/src/lib.rs | 12 +- sdk/abi/src/syscall.rs | 152 +++-- sdk/filepicker/src/lib.rs | 13 +- sdk/libc/src/errcode.rs | 3 - sdk/std/sys/pal/mod.rs | 2 - sdk/toyos/src/audio.rs | 20 +- sdk/toyos/src/device.rs | 2 +- sdk/toyos/src/fs.rs | 17 +- sdk/toyos/src/ipc.rs | 646 +++++++----------- sdk/toyos/src/launch.rs | 7 +- sdk/toyos/src/net.rs | 96 ++- sdk/toyos/src/net/socket.rs | 30 +- sdk/toyos/src/net/stream.rs | 28 +- sdk/toyos/src/power.rs | 8 +- sdk/toyos/src/surface.rs | 11 +- sdk/window/src/display.rs | 12 +- sdk/window/src/lib.rs | 58 +- system/CLAUDE.md | 2 +- system/compositor/src/client.rs | 28 +- system/diskserver/src/session.rs | 38 +- system/diskserver/src/tables.rs | 8 +- system/grants/src/main.rs | 9 +- system/hda/src/main.rs | 15 +- system/logout/src/main.rs | 9 +- system/netstack/src/client.rs | 21 +- system/netstack/src/link.rs | 73 +- system/netstack/src/main.rs | 29 +- system/soundserver/src/control.rs | 8 +- system/supervisor/src/main.rs | 10 +- system/swap/src/main.rs | 9 +- tests/guest/osbench/src/sys/toyos.rs | 38 +- .../rust-tests/src/bin/abuse_connect_flood.rs | 16 +- .../rust-tests/src/bin/abuse_shared_grant.rs | 5 +- .../src/bin/connect_before_serve.rs | 57 +- .../src/bin/cpal_drop_unreleased.rs | 14 +- .../src/bin/libc_recv_beside_close.rs | 9 +- .../guest/rust-tests/src/bin/memory_floor.rs | 22 +- .../src/bin/message_carries_its_handles.rs | 91 --- .../rust-tests/src/bin/messages_are_whole.rs | 176 +++++ tests/guest/rust-tests/src/bin/mutual_kill.rs | 7 +- .../src/bin/netstack_connect_killed.rs | 9 +- .../src/bin/netstack_gone_mid_bind.rs | 15 +- .../rust-tests/src/bin/pipe_other_end_gone.rs | 5 +- .../guest/rust-tests/src/bin/process_tree.rs | 12 +- .../src/bin/shm_release_reclaims.rs | 5 +- .../rust-tests/src/bin/spawn_lands_claimed.rs | 7 +- .../rust-tests/src/bin/stream_handles.rs | 13 +- .../rust-tests/src/bin/window_refusal.rs | 7 +- tests/guest/rust-tests/src/master.rs | 9 +- tests/guest/rust-tests/src/usbd_spare.rs | 9 +- 72 files changed, 1929 insertions(+), 1769 deletions(-) create mode 100644 kernel/src/frame_bytes.rs create mode 100644 kernel/src/message.rs delete mode 100644 tests/guest/rust-tests/src/bin/message_carries_its_handles.rs create mode 100644 tests/guest/rust-tests/src/bin/messages_are_whole.rs diff --git a/issues/a-client-waits-on-the-compositors-answer-with-no-bound.md b/issues/a-client-waits-on-the-compositors-answer-with-no-bound.md index e5d6e1d8264..7306807708d 100644 --- a/issues/a-client-waits-on-the-compositors-answer-with-no-bound.md +++ b/issues/a-client-waits-on-the-compositors-answer-with-no-bound.md @@ -7,7 +7,7 @@ opened: 2026-09-28 # A client waits on the compositor's answer with no bound `window::clipboard_set` (`sdk/window/src/lib.rs`) blocks in -`recv_header` for `MSG_COPY_REGION` on a copy past `MAX_INLINE_PAYLOAD`, and +`FrameRx::recv` for `MSG_COPY_REGION` on a copy past `MAX_INLINE_PAYLOAD`, and `Window::create` blocks the same way for `MSG_WINDOW_CREATED`. Neither wait has a deadline, so a compositor that is alive and not answering wedges every terminal or editor that copies, and every program that asks for a window. diff --git a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md index cf022d69616..4ce80934d39 100644 --- a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md +++ b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md @@ -8,17 +8,25 @@ opened: 2026-08-28 Each pipe owns its state behind its own `sync::Lock` (`kernel/src/pipe.rs`), which disables preemption before it spins (`kernel/src/sync.rs`). The bulk copy -runs inside that lock: +runs inside that lock, through the frame store pipes and connections share +(`kernel/src/frame_bytes.rs`): -* `PipeRef::try_read_within` makes one walk of the caller's window, +* `FrameBytes::read` makes one walk of the caller's window, `buf.write_pieces`, and copies each pinned piece out of the frames it covers with `bytes.peek`, under it; -* `PipeRef::write` makes one walk, `buf.read_pieces`, and copies each piece +* `FrameBytes::write` makes one walk, `buf.read_pieces`, and copies each piece into its frames with `bytes.place`, in the same acquisition. -The copy is bounded only by the pipe: `PIPE_CAPACITY` is 2 MiB, the largest -single copy under the lock, and nothing above caps it (`SYS_READ`/`SYS_WRITE` -pass the caller's length through). Each 16 pages of it take the caller's space +A pipe's copy is bounded only by the pipe: `PIPE_CAPACITY` is 2 MiB, the +largest single copy under the lock, and nothing above caps it +(`SYS_READ`/`SYS_WRITE` pass the caller's length through). + +A connection direction (`kernel/src/message.rs`) copies one whole message under +its queue's lock, so its copy is bounded at `MAX_MESSAGE_LEN`, 16 KiB, and its +frames are taken the same way. Both its send and its receive run that copy +inside `process::with_process_data`, nesting the queue's lock in the process +lock: the receive installs the message's handles in the caller's table and +copies its bytes in one hold, so a refusal takes nothing. Each 16 pages of it take the caller's space lock and populate any untouched page, a frame from the allocator and a 4 KiB zeroing. @@ -42,8 +50,8 @@ bounded per acquisition, has to carry the data-race argument. The frames can be taken before the lock and given back after it, as the read end's last holder already gives back what the pipe held. -**Exit**: no pipe lock is held across more than a bounded copy or any -allocation, the hold time is read before and after, and `audio_tone`'s worst +**Exit**: no pipe or queue lock is held across more than a bounded copy or any +allocation, the connection's nesting in the process lock included, the hold time is read before and after, and `audio_tone`'s worst wake does not move. This is stage S0-a of `issues/anonymous-memory-swaps-to-an-encrypted-partition.md`. diff --git a/issues/a-received-handle-has-no-knowable-type.md b/issues/a-received-handle-has-no-knowable-type.md index cd9e6df3bca..fd8c308dc7e 100644 --- a/issues/a-received-handle-has-no-knowable-type.md +++ b/issues/a-received-handle-has-no-knowable-type.md @@ -6,8 +6,8 @@ opened: 2026-08-10 # A process cannot ask what most handles it received are, and using one wrongly is fatal -`SYS_HANDLE_RECV` installs a peer's handles in the receiver's table and answers -only *how many*. `SYS_FSTAT` answers what any handle is, needing no right, and +`SYS_MESSAGE_RECV` installs a message's handles in the receiver's table and +answers only *how many*. `SYS_FSTAT` answers what any handle is, needing no right, and settles two kinds: only a region of memory the kernel allocated answers `SharedMemory`, which is how `SharedMemory::lent` refuses every other handle before it maps one (`sdk/toyos/src/shm.rs`), and only a `Connection` answers @@ -47,11 +47,10 @@ in its place sees it refused while the receiver lives. ## Three ways out, and none is free -1. **Report the kind.** `SYS_HANDLE_RECV` writes `(RawHandle, kind)` pairs - instead of bare handles. One ABI shape change, in-tree only — no fork names - `handle_recv` (the whole fork estate was swept for this branch and - `handle_recv` is not in it). The receiver then refuses by - name and the fail-fast policy is untouched. +1. **Report the kind.** `SYS_MESSAGE_RECV` writes `(RawHandle, kind)` pairs + instead of bare handles. One ABI shape change, in-tree only: no fork calls + `message_recv` or the SDK's receive. The receiver then refuses by name and + the fail-fast policy is untouched. 2. **A syscall** answering the kind of a handle. Cheaper to write and worse: it is a second place to ask, and it makes "what is this" a round trip rather than part of the answer that produced it. @@ -67,8 +66,9 @@ Judged while clearing PR #22's blockers, with the reasoning written down because the next reader will ask why a class this wide was left open. - **The fix is an ABI shape change and the ABI was the owner's to approve.** It - widens `SYS_HANDLE_RECV`'s answer from `n` to `n` pairs, which is a syscall - the owner approved changing shape after the fact. + widens the receive's answer from `n` to `n` pairs (`SYS_HANDLE_RECV` then, + `SYS_MESSAGE_RECV` since), which is a syscall the owner approved changing + shape after the fact. So it stays open, and what it is waiting on is a decision rather than an instrument. diff --git a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md index afb610ca294..e279f776b90 100644 --- a/issues/a-tcp-connect-zeroes-twelve-mebibytes.md +++ b/issues/a-tcp-connect-zeroes-twelve-mebibytes.md @@ -13,9 +13,11 @@ may never use: `SYS_SHM_CREATE` takes and zeroes each of its 2048 frames before it answers (`kernel/src/object/shm.rs`), and netstack maps all of them before the SYN leaves. -- **The connection's two pipes**, 2 MiB each: the request's page on the +- **The connection's two directions**, 2 MiB each: the request's page on the client's write and the answer's on netstack's, each zeroed whole on that first write (`issues/a-pipe-is-a-two-mebibyte-page-for-a-mapping-nothing-makes.md`). + Since #1074 a direction takes 4 KiB frames as it fills and zeroes none, so + this part is measured no longer. **Measured** by the paired `osbench lan` sessions from the T14's I219 to the AMD bench's Linux peer over its Wi-Fi: diff --git a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md index 7812d58ff67..be8c190c230 100644 --- a/issues/netstack-moves-every-byte-through-two-kernel-pipes.md +++ b/issues/netstack-moves-every-byte-through-two-kernel-pipes.md @@ -74,7 +74,7 @@ The design and its roast are PR #863's body. datagram socket are already their holders' connections and bells (`toyos::net::socket`), each datagram one frame on its holder's connection, and the pipe ABI's messages and `toyos::net`'s pipe handles are - gone; a datagram is still copied through the connection's pipes. netstack copies each datagram record's + gone; a datagram is still copied through the connection. netstack copies each datagram record's `len`, `port` and `addr` out of the region once and judges the copy: a `len` past the bytes the record holds ends that socket alone. `toyos::net`'s `send_to` refuses what a sender can judge and writes no record: a payload past diff --git a/issues/the-capability-end-state-is-twelve-answers.md b/issues/the-capability-end-state-is-twelve-answers.md index 8e78cf9645b..5ea93c2aa0f 100644 --- a/issues/the-capability-end-state-is-twelve-answers.md +++ b/issues/the-capability-end-state-is-twelve-answers.md @@ -229,7 +229,7 @@ process has. is created with `Rights::NONE` there because "every bit on a `SysCap` is an authority init decides per program" — the machine's only one is minted with `TRANSFER` at `kernel/src/loader/mod.rs:946`. The mechanisms are -`SYS_MESSAGE_SEND`/`SYS_HANDLE_RECV` over a `Connection`, where the connection +`SYS_MESSAGE_SEND`/`SYS_MESSAGE_RECV` over a `Connection`, where the connection and every handle must carry `TRANSFER`, no handle may be named twice, and the connection may not be sent over itself (`4a98107f^:kernel/src/arch/syscall.rs:2086`, `:2090`); and `SpawnArgs::endow`. Nothing else crosses a process boundary — the diff --git a/kernel/pure/message.rs b/kernel/pure/message.rs index e02d3f7b32d..66db4be8686 100644 --- a/kernel/pure/message.rs +++ b/kernel/pure/message.rs @@ -1,114 +1,117 @@ -//! Where each batch of handles a connection carries sits in its byte stream. +//! One direction of a connection: whole messages in the order they were sent, +//! each its length and the handles it carries. //! -//! A message is bytes and the handles they announce, sent together: the bytes -//! go into the stream whole, and the batch is pinned at the byte just past -//! them. The reader's reads never cross a pinned batch it has not taken, and -//! it takes a batch only where it is pinned. So a batch is never received -//! without its bytes, nor its bytes read past without the batch noticed. -//! -//! Positions count every byte the stream ever carried, never wrapping: a -//! `u64` of bytes outlives any connection. +//! A message is queued whole or refused, and taken whole or left queued, so a +//! reader never sees part of one and its handles never come apart from its +//! bytes. The bytes live beside this queue in the same order; it counts them +//! only to say where a message ends. #![forbid(unsafe_code)] use alloc::collections::VecDeque; -/// What a read at the reader's position may take now. +use toyos_abi::syscall::{MAX_MESSAGE_LEN, MAX_QUEUED_HANDLES, MAX_QUEUED_MESSAGES, MAX_TRANSFER_HANDLES}; + +/// The message at the front of a queue. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum Window { - /// Up to this many bytes; zero is nothing to read yet. - Bytes(usize), - /// A batch is pinned exactly here, and the read must take it first. - AtBatch, +pub struct Head { + pub len: usize, + pub handles: usize, } -/// Why a message is refused before its bytes are written. Nothing of it is -/// written or moved; room in the stream and the reader are the stream's to answer. +/// Why the front message stays queued. #[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum Refused { - /// Handles with no bytes announcing them. - Empty, - /// The reader holds as many batches as it may. - QueueFull, +pub enum Untaken { + /// The reader's byte or handle buffer cannot hold it whole. + TooSmall, + /// The reader has no room in its table for the handles it carries. + NoRoom, } -/// The batches a reader has not taken, each at the stream position it -/// follows, in the order they were sent. -pub struct Batches { - pinned: VecDeque<(u64, B)>, +/// The messages a reader has not taken, and their handles, at most +/// [`MAX_QUEUED_MESSAGES`] and [`MAX_QUEUED_HANDLES`]. +pub struct Queue { + messages: VecDeque, + handles: VecDeque, } -impl Default for Batches { +impl Default for Queue { fn default() -> Self { Self::new() } } -impl Batches { +impl Queue { pub const fn new() -> Self { - Self { pinned: VecDeque::new() } + Self { messages: VecDeque::new(), handles: VecDeque::new() } } pub fn len(&self) -> usize { - self.pinned.len() + self.messages.len() } pub fn is_empty(&self) -> bool { - self.pinned.is_empty() + self.messages.is_empty() } - /// Whether a message of `len` bytes, carrying a batch or not, may be - /// pinned here, `cap` batches at most. - pub fn admit(&self, len: usize, carries: bool, cap: usize) -> Result<(), Refused> { - if carries && len == 0 { - return Err(Refused::Empty); - } - if carries && self.pinned.len() >= cap { - return Err(Refused::QueueFull); - } - Ok(()) + pub fn head(&self) -> Option { + self.messages.front().copied() } - /// Pin `batch` at `at`, the position just past the bytes that announce it. - /// - /// # Panics - /// When `at` is before the last batch's position: positions only grow, so - /// that is a caller that wrote out of order. - pub fn pin(&mut self, at: u64, batch: B) { - if let Some((last, _)) = self.pinned.back() { - assert!(*last <= at, "message: a batch pinned at {at} after one at {last}"); - } - self.pinned.push_back((at, batch)); + /// Whether a message of `len` bytes carrying `handles` fits now, beside + /// `space` bytes of room in the bytes it is written to. + pub fn fits(&self, space: usize, len: usize, handles: usize) -> bool { + len <= space + && self.messages.len() < MAX_QUEUED_MESSAGES + && self.handles.len() + handles <= MAX_QUEUED_HANDLES } - /// What a read at `read` may take of the `available` bytes the stream holds. - pub fn window(&self, read: u64, available: usize) -> Window { - match self.pinned.front() { - Some((at, _)) if *at == read => Window::AtBatch, - Some((at, _)) => { - let before = at.checked_sub(read).expect("message: the reader read past a pinned batch"); - Window::Bytes(available.min(usize::try_from(before).unwrap_or(usize::MAX))) - } - None => Window::Bytes(available), - } + /// Whether the largest message one send may make fits now: what a + /// writable connection promises. + pub fn fits_any(&self, space: usize) -> bool { + self.fits(space, MAX_MESSAGE_LEN, MAX_TRANSFER_HANDLES) } - /// The batch pinned at `read`, if any, without taking it. - pub fn at(&self, read: u64) -> Option<&B> { - self.pinned.front().filter(|(at, _)| *at == read).map(|(_, b)| b) + /// Queue a message whose bytes are written. + /// + /// # Panics + /// When it does not fit by count: [`Self::fits`] is asked under the same + /// hold, before its bytes are written. + pub fn push(&mut self, len: usize, handles: impl ExactSizeIterator) { + let count = handles.len(); + assert!( + self.messages.len() < MAX_QUEUED_MESSAGES && self.handles.len() + count <= MAX_QUEUED_HANDLES, + "message: a message queued past the queue's bounds" + ); + self.handles.extend(handles); + self.messages.push_back(Head { len, handles: count }); } - /// Take the batch pinned at `read`; none elsewhere. - pub fn take(&mut self, read: u64) -> Option { - match self.pinned.front() { - Some((at, _)) if *at == read => self.pinned.pop_front().map(|(_, b)| b), - _ => None, + /// Take the front message into a reader holding `buf` bytes and `cap` + /// handles, whose table `room` says can take its handles, which come + /// back in order. Its bytes are the caller's to copy out before this + /// hold ends. + pub fn pop( + &mut self, + buf: usize, + cap: usize, + room: impl FnOnce(usize) -> bool, + ) -> Option + '_), Untaken>> { + let head = self.head()?; + if head.len > buf || head.handles > cap { + return Some(Err(Untaken::TooSmall)); + } + if head.handles > 0 && !room(head.handles) { + return Some(Err(Untaken::NoRoom)); } + self.messages.pop_front(); + Some(Ok((head, self.handles.drain(..head.handles)))) } - /// Every batch still pinned, for its end's release. - pub fn drain(&mut self) -> impl Iterator + '_ { - self.pinned.drain(..).map(|(_, b)| b) + /// Every handle still queued, for the reader's end, with the messages. + pub fn clear(&mut self) -> VecDeque { + self.messages.clear(); + core::mem::take(&mut self.handles) } } @@ -117,76 +120,115 @@ mod tests { use super::*; use alloc::vec::Vec; - /// A stream and its batches as the kernel holds them: the bytes are - /// numbered by their position, so a read says exactly which it took. - struct Stream { - batches: Batches, + /// A message as taken: its bytes, numbered by the order they were sent, + /// and its handles. + type Message = (Vec, Vec); + + /// A direction as the kernel holds it: the bytes numbered by the order + /// they were sent, in a store of `room` bytes. + struct Direction { + queue: Queue, bytes: VecDeque, - written: u64, - read: u64, - ring: usize, + next_byte: u64, + room: usize, } - impl Stream { - fn new(ring: usize) -> Self { - Self { batches: Batches::new(), bytes: VecDeque::new(), written: 0, read: 0, ring } + impl Direction { + fn new(room: usize) -> Self { + Self { queue: Queue::new(), bytes: VecDeque::new(), next_byte: 0, room } } - /// One send, whole or not at all, as the kernel makes it under the - /// queue lock and then the pipe lock; `None` is a stream without room. - fn send(&mut self, len: usize, batch: Option) -> Option> { - if let Err(refused) = self.batches.admit(len, batch.is_some(), 4) { - return Some(Err(refused)); - } - if len > self.ring - self.bytes.len() { - return None; + fn send(&mut self, len: usize, handles: &[u32]) -> bool { + if !self.queue.fits(self.room - self.bytes.len(), len, handles.len()) { + return false; } for _ in 0..len { - self.bytes.push_back(self.written); - self.written += 1; + self.bytes.push_back(self.next_byte); + self.next_byte += 1; } - if let Some(b) = batch { - self.batches.pin(self.written, b); - } - Some(Ok(())) + self.queue.push(len, handles.iter().copied()); + true } - fn read(&mut self, want: usize) -> Result, Window> { - match self.batches.window(self.read, self.bytes.len()) { - Window::AtBatch => Err(Window::AtBatch), - Window::Bytes(n) => { - let got: Vec = self.bytes.drain(..n.min(want)).collect(); - self.read += got.len() as u64; - Ok(got) - } - } + fn recv(&mut self, buf: usize, cap: usize, room: bool) -> Option> { + let (head, got) = match self.queue.pop(buf, cap, |_| room)? { + Ok((head, handles)) => (head, handles.collect()), + Err(untaken) => return Some(Err(untaken)), + }; + Some(Ok((self.bytes.drain(..head.len).collect(), got))) } } #[test] - fn a_batch_is_taken_only_where_its_bytes_end_and_no_read_crosses_it() { - let mut s = Stream::new(64); - s.send(5, Some(7)).unwrap().unwrap(); - s.send(3, None).unwrap().unwrap(); - assert_eq!(s.batches.take(s.read), None, "a batch taken before its bytes were read"); - assert_eq!(s.read(100).unwrap(), [0, 1, 2, 3, 4], "a read crossed the batch"); - assert_eq!(s.read(100), Err(Window::AtBatch), "a read went on past an untaken batch"); - assert_eq!(s.batches.take(s.read), Some(7)); - assert_eq!(s.read(100).unwrap(), [5, 6, 7]); + fn a_message_is_taken_whole_with_its_own_handles() { + let mut d = Direction::new(64); + assert!(d.send(3, &[7, 8])); + assert!(d.send(2, &[])); + assert!(d.send(1, &[9])); + assert_eq!(d.recv(64, 8, true), Some(Ok((alloc::vec![0, 1, 2], alloc::vec![7, 8])))); + assert_eq!(d.recv(64, 8, true), Some(Ok((alloc::vec![3, 4], alloc::vec![])))); + assert_eq!(d.recv(64, 8, true), Some(Ok((alloc::vec![5], alloc::vec![9])))); + assert_eq!(d.recv(64, 8, true), None); + } + + /// A reader that cannot hold the front message whole takes nothing of it, + /// and takes it whole once it can. + #[test] + fn a_message_too_large_for_the_reader_stays_queued_whole() { + let mut d = Direction::new(64); + assert!(d.send(4, &[1, 2])); + assert_eq!(d.recv(3, 8, true), Some(Err(Untaken::TooSmall)), "a buffer a byte short"); + assert_eq!(d.recv(4, 1, true), Some(Err(Untaken::TooSmall)), "a handle buffer one short"); + assert_eq!(d.recv(4, 2, false), Some(Err(Untaken::NoRoom)), "a table with no room"); + assert_eq!(d.recv(4, 2, true), Some(Ok((alloc::vec![0, 1, 2, 3], alloc::vec![1, 2])))); + } + + /// The table's room is asked only for a message that carries handles. + #[test] + fn a_message_without_handles_asks_no_room() { + let mut d = Direction::new(8); + assert!(d.send(1, &[])); + assert_eq!(d.recv(8, 0, false), Some(Ok((alloc::vec![0], alloc::vec![])))); } - /// A refusal writes no byte and pins no batch. + /// Each bound refuses the message that would cross it and nothing before. #[test] - fn a_refused_send_leaves_nothing_behind() { - let mut s = Stream::new(8); - assert_eq!(s.send(0, Some(1)), Some(Err(Refused::Empty))); - assert_eq!((s.bytes.len(), s.batches.len()), (0, 0)); - for b in 0..4 { - s.send(1, Some(b)).unwrap().unwrap(); + fn each_bound_refuses_the_message_past_it() { + let mut d = Direction::new(MAX_QUEUED_MESSAGES * 2); + for _ in 0..MAX_QUEUED_MESSAGES { + assert!(d.send(1, &[])); } - assert_eq!(s.send(1, Some(9)), Some(Err(Refused::QueueFull))); - assert_eq!((s.bytes.len(), s.batches.len()), (4, 4)); - assert_eq!(s.send(1, None), Some(Ok(())), "a message with no handles waits on no queue"); + assert!(!d.send(1, &[]), "one message past the count bound was queued"); + + let mut d = Direction::new(1 << 20); + let full = [0u32; MAX_TRANSFER_HANDLES]; + for _ in 0..MAX_QUEUED_HANDLES / MAX_TRANSFER_HANDLES { + assert!(d.send(1, &full)); + } + assert!(!d.send(1, &[1]), "one handle past the handle bound was queued"); + assert!(d.send(1, &[]), "a message with no handles waits on no handle room"); + + let mut d = Direction::new(10); + assert!(d.send(10, &[])); + assert!(!d.send(1, &[]), "a byte past the store was queued"); + } + + /// Writable promises room for the largest message one send may make. + #[test] + fn fits_any_is_room_for_the_largest_send() { + let q: Queue = Queue::new(); + assert!(q.fits_any(MAX_MESSAGE_LEN)); + assert!(!q.fits_any(MAX_MESSAGE_LEN - 1)); + let mut q: Queue = Queue::new(); + q.push(1, core::iter::repeat_n(0, MAX_QUEUED_HANDLES - MAX_TRANSFER_HANDLES + 1)); + assert!(!q.fits_any(usize::MAX), "room for fewer than a send's handles is not writable"); + } + + #[test] + #[should_panic(expected = "past the queue's bounds")] + fn a_push_past_the_bounds_is_a_caller_bug() { + let mut q = Queue::new(); + q.push(1, core::iter::repeat_n(0u32, MAX_QUEUED_HANDLES + 1)); } /// xorshift64*: one seed is one run. @@ -205,60 +247,42 @@ mod tests { } } - /// Random sends and reads of random sizes against the order they were - /// sent: every batch reaches the reader exactly after the last byte of its - /// own message and before any byte of the next, and every byte arrives - /// once, in order. + /// Random sends and receives of random sizes against the order they were + /// sent: every message arrives once, whole, in order, with exactly its own + /// handles, and a refused receive takes nothing. #[test] - fn every_batch_arrives_between_its_own_bytes_and_the_next_messages() { + fn every_message_arrives_once_whole_in_order_with_its_handles() { let mut rng = Rng(0x9E37_79B9_7F4A_7C15); - let mut s = Stream::new(256); - // What the reader must meet: per message, its bytes then its batch. - let mut sent: VecDeque<(u64, Option)> = VecDeque::new(); - let mut next_byte = 0u64; + let mut d = Direction::new(512); + let mut sent: VecDeque<(Vec, Vec)> = VecDeque::new(); + let mut next_handle = 0u32; let mut taken = 0; - for op in 0..20_000u32 { + for _ in 0..50_000u32 { if rng.below(2) == 0 { - let len = 1 + rng.below(40) as usize; - let batch = (rng.below(3) == 0).then_some(op); - if s.send(len, batch) == Some(Ok(())) { - sent.push_back((s.written, batch)); + let len = rng.below(48) as usize; + let count = if rng.below(3) == 0 { 1 + rng.below(MAX_TRANSFER_HANDLES as u64) as usize } else { 0 }; + let handles: Vec = (next_handle..next_handle + count as u32).collect(); + let first = d.next_byte; + if d.send(len, &handles) { + next_handle += count as u32; + sent.push_back(((first..first + len as u64).collect(), handles)); } } else { - match s.read(1 + rng.below(50) as usize) { - Ok(got) => { - for byte in got { - assert_eq!(byte, next_byte, "a byte out of order"); - next_byte += 1; - // A message ending before this byte with a batch must have been taken. - while sent.front().is_some_and(|(end, batch)| *end <= byte && batch.is_none()) { - sent.pop_front(); - } - assert!( - !sent.front().is_some_and(|(end, b)| *end <= byte && b.is_some()), - "byte {byte} was read past a batch pinned at {}", - sent.front().unwrap().0 - ); - } - } - Err(Window::AtBatch) => { - let (end, batch) = sent.pop_front().expect("a batch the model never sent"); - assert_eq!(end, s.read, "a batch met away from the end of its bytes"); - assert_eq!(s.batches.take(s.read), batch, "the batch taken is not its message's"); + let buf = rng.below(56) as usize; + let cap = rng.below(MAX_TRANSFER_HANDLES as u64 + 1) as usize; + let room = rng.below(4) != 0; + let before = (d.queue.len(), d.bytes.len()); + match d.recv(buf, cap, room) { + None => assert!(sent.is_empty(), "a queued message was not offered"), + Some(Ok(got)) => { + assert!(got.0.len() <= buf && got.1.len() <= cap, "a message taken into a reader too small for it"); + assert_eq!(Some(got), sent.pop_front(), "a message out of order, cut or with another's handles"); taken += 1; } - Err(Window::Bytes(_)) => unreachable!(), + Some(Err(_)) => assert_eq!((d.queue.len(), d.bytes.len()), before, "a refused receive took something"), } } } - assert!(taken > 1000, "the run took only {taken} batches"); - } - - #[test] - #[should_panic(expected = "pinned at 3 after one at 5")] - fn a_batch_pinned_before_the_last_is_a_caller_bug() { - let mut b = Batches::new(); - b.pin(5, 1); - b.pin(3, 2); + assert!(taken > 5_000, "the run took only {taken} messages"); } } diff --git a/kernel/pure/pipe.rs b/kernel/pure/pipe.rs index d171d5b2491..7b85fe6bd96 100644 --- a/kernel/pure/pipe.rs +++ b/kernel/pure/pipe.rs @@ -111,34 +111,20 @@ impl Holders { } } -/// How a write lands in a pipe. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum Landing { - /// All of it or none: a message. - Whole, - /// Whole up to [`MAX_WHOLE_PIPE_WRITE`] bytes, and as far as it fits past - /// that: a pipe end's write. - Pipe, - /// As far as it fits: a connection's plain write. - Partial, -} - -impl Landing { - /// The room a write of `len` bytes waits for: it lands as soon as there is - /// this much, and not before. - pub fn room(self, len: usize) -> usize { - match self { - Self::Whole => len, - Self::Pipe if len <= MAX_WHOLE_PIPE_WRITE => len, - Self::Pipe | Self::Partial => len.min(1), - } +/// The room a pipe write of `len` bytes waits for: it lands as soon as there +/// is this much, and not before. Whole up to [`MAX_WHOLE_PIPE_WRITE`] bytes, +/// and as far as it fits past that. +pub fn room(len: usize) -> usize { + match len <= MAX_WHOLE_PIPE_WRITE { + true => len, + false => 1, } +} - /// How many of `len` bytes land with `space` free; `None` until there is - /// [`Self::room`]. - pub fn takes(self, space: usize, len: usize) -> Option { - (space >= self.room(len)).then(|| len.min(space)) - } +/// How many of `len` bytes a pipe write lands with `space` free; `None` +/// until there is [`room`]. +pub fn takes(space: usize, len: usize) -> Option { + (space >= room(len)).then(|| len.min(space)) } @@ -356,11 +342,35 @@ mod tests { const MAX: usize = MAX_WHOLE_PIPE_WRITE; + /// How a model write lands: a pipe end's, or as far as it fits, which no + /// write makes: the tearing test's control. + #[derive(Clone, Copy, Debug)] + enum Landing { + Pipe, + Partial, + } + + impl Landing { + fn room(self, len: usize) -> usize { + match self { + Self::Pipe => room(len), + Self::Partial => len.min(1), + } + } + + fn takes(self, space: usize, len: usize) -> Option { + match self { + Self::Pipe => takes(space, len), + Self::Partial => (space >= self.room(len)).then(|| len.min(space)), + } + } + } + /// What a write answers, as `kernel::pipe::write` does. #[derive(PartialEq, Eq, Debug)] enum Wrote { Bytes(usize), - /// A blocking write waits for [`Landing::room`]; a non-blocking one + /// A blocking write waits for [`room`]; a non-blocking one /// answers `WouldBlock`. NoRoom, Broken, @@ -457,14 +467,12 @@ mod tests { /// ring, and still answers a missing reader. #[test] fn an_empty_write_waits_for_nothing() { - for landing in [Landing::Whole, Landing::Pipe, Landing::Partial] { - let mut pipe = Model::new(64); - pipe.ring.extend([0; 64]); - assert!(pipe.wakes(0, landing), "{landing:?}: an empty write waits on a full ring"); - assert_eq!(pipe.write(&[], landing), Wrote::Bytes(0)); - assert_eq!(pipe.holders.release(End::Read), Released::EndGone); - assert_eq!(pipe.write(&[], landing), Wrote::Broken); - } + let mut pipe = Model::new(64); + pipe.ring.extend([0; 64]); + assert!(pipe.wakes(0, Landing::Pipe), "an empty write waits on a full ring"); + assert_eq!(pipe.write(&[], Landing::Pipe), Wrote::Bytes(0)); + assert_eq!(pipe.holders.release(End::Read), Released::EndGone); + assert_eq!(pipe.write(&[], Landing::Pipe), Wrote::Broken); } /// A writer waiting for room is answered when the reader leaves, and its @@ -487,15 +495,13 @@ mod tests { #[test] fn a_waiting_writer_wakes_exactly_when_its_write_lands() { let cap = 2 * MAX + 3; - for landing in [Landing::Whole, Landing::Pipe, Landing::Partial] { - for len in [0, 1, 2, 40, MAX - 1, MAX, MAX + 1, 2 * MAX] { - for space in 0..=cap { - let mut pipe = Model::new(cap); - pipe.ring.extend(core::iter::repeat_n(0, cap - space)); - let wakes = pipe.wakes(len, landing); - let lands = pipe.write(&vec![7; len], landing) != Wrote::NoRoom; - assert_eq!(wakes, lands, "{landing:?}, {len} bytes, {space} free: woken {wakes}, lands {lands}"); - } + for len in [0, 1, 2, 40, MAX - 1, MAX, MAX + 1, 2 * MAX] { + for space in 0..=cap { + let mut pipe = Model::new(cap); + pipe.ring.extend(core::iter::repeat_n(0, cap - space)); + let wakes = pipe.wakes(len, Landing::Pipe); + let lands = pipe.write(&vec![7; len], Landing::Pipe) != Wrote::NoRoom; + assert_eq!(wakes, lands, "{len} bytes, {space} free: woken {wakes}, lands {lands}"); } } } diff --git a/kernel/src/frame_bytes.rs b/kernel/src/frame_bytes.rs new file mode 100644 index 00000000000..2aebed946bb --- /dev/null +++ b/kernel/src/frame_bytes.rs @@ -0,0 +1,176 @@ +//! Bytes a kernel object holds for a reader, in 4 KiB frames counted to the +//! job that made the object and taken as it fills (`kernel::pipe::Bytes`): a +//! pipe's stream and a connection direction's messages. +//! +//! The frames are kernel-only and written before they are read, so they are +//! never zeroed. Every call is made under its object's lock, which is the +//! only thing that reaches them. + +use core::ops::Range; + +use alloc::sync::Arc; + +use kernel::pipe::{Bytes, MOST_FRAMES}; +use kernel::sched::job::Job; + +use crate::mm::charge::{self, Charge, Exhausted}; +use crate::mm::{pmm, DirectMap}; +use crate::user_ptr::{UserBytes, UserBytesMut}; + +/// The frames and the heap their list takes, charged to one job; the frames +/// go back when it drops. +pub struct FrameBytes { + bytes: Bytes, + list: Charge, +} + +impl FrameBytes { + pub fn new(job: &Arc) -> Self { + Self { bytes: Bytes::new(), list: Charge::none(job) } + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + /// Room for more bytes, up to the capacity. + pub fn space(&self) -> usize { + self.bytes.space() + } + + /// Frames taken, spares included. + pub fn frames(&self) -> usize { + self.bytes.frames() + } + + pub fn job(&self) -> &Arc { + self.list.job() + } + + /// Append the first `n` bytes of `buf`, taking the frames they need, all + /// of them or none: a refusal appends nothing. `n` is at most + /// [`Self::space`]. + pub fn write(&mut self, buf: &UserBytes, n: usize) -> Result<(), Exhausted> { + self.take_frames(n)?; + let bytes = &mut self.bytes; + // One walk of the user window, each pinned piece copied into the + // frames it covers. + buf.read_pieces(0, n, |from, at, piece| { + let mut gather = Gather::NONE; + bytes.place(at, piece, |frame, within, off| { + // SAFETY: `from` is valid for `piece` bytes until this walk's + // callback returns, and `off + within.len()` is at most + // `piece`; the frame is this object's alone, reached only + // under its lock, and no process maps it. + unsafe { gather.add(from.add(off), frame_bytes(frame, within.clone()), within.len()) } + }); + // SAFETY: as above, before the callback returns. + unsafe { gather.flush() }; + }); + bytes.commit(n); + Ok(()) + } + + /// Move the first `n` bytes held into the start of `buf`, giving back the + /// spares the read leaves past those kept. `n` is at most [`Self::len`]. + pub fn read(&mut self, buf: &mut UserBytesMut, n: usize) { + let bytes = &mut self.bytes; + buf.write_pieces(0, n, |to, at, piece| { + let mut gather = Gather::NONE; + bytes.peek(at, piece, |frame, within, off| { + // SAFETY: `to` is valid for `piece` bytes until this walk's + // callback returns, and `off + within.len()` is at most + // `piece`; the frame is as in `write`. + unsafe { gather.add(frame_bytes(frame, within.clone()), to.add(off), within.len()) } + }); + // SAFETY: as above, before the callback returns. + unsafe { gather.flush() }; + }); + bytes.consume(n); + // SAFETY: a spare holds no byte, and nothing but this list names it. + unsafe { pmm::free_held(self.list.job(), self.bytes.surplus()) }; + } + + fn take_frames(&mut self, n: usize) -> Result<(), Exhausted> { + let wants = self.bytes.wants(n); + if wants == 0 { + return Ok(()); + } + let frames = self.bytes.frames() + wants; + if frames > self.bytes.listed() { + let list = frames.next_power_of_two().min(MOST_FRAMES); + self.list.resize(charge::array::(list))?; + self.bytes.list(list); + } + let bytes = &mut self.bytes; + pmm::alloc_held(self.list.job(), wants, |frame| bytes.add(frame)).then_some(()).ok_or(Exhausted) + } +} + +impl Drop for FrameBytes { + fn drop(&mut self) { + // SAFETY: nothing but this list names its frames. + unsafe { pmm::free_held(self.list.job(), self.bytes.clear()) }; + } +} + +/// One copy gathered from runs that continue each other in both source and +/// destination, so a user page's piece over two physically adjacent frames +/// is one copy, not two. +struct Gather { + from: *const u8, + to: *mut u8, + len: usize, +} + +impl Gather { + const NONE: Self = Self { from: core::ptr::null(), to: core::ptr::null_mut(), len: 0 }; + + /// Add `len` bytes from `from` to `to`, copying what was gathered first + /// unless these continue it. + /// + /// # Safety + /// `from` is readable and `to` writable for `len` bytes, and stay so + /// until the next [`Self::flush`]; the two never overlap. + #[inline(always)] + unsafe fn add(&mut self, from: *const u8, to: *mut u8, len: usize) { + if self.len > 0 && self.from.wrapping_add(self.len) == from && self.to.wrapping_add(self.len) == to { + self.len += len; + return; + } + // SAFETY: what was gathered meets this function's contract. + unsafe { self.flush() }; + *self = Self { from, to, len }; + } + + /// Copy what was gathered. + /// + /// # Safety + /// As [`Self::add`] for every run gathered. + #[inline(always)] + unsafe fn flush(&mut self) { + if self.len > 0 { + // SAFETY: the runs gathered are contiguous on both sides and + // each met `add`'s contract. + unsafe { core::ptr::copy_nonoverlapping(self.from, self.to, self.len) }; + } + *self = Self::NONE; + } +} + +/// `within` of a frame an object holds, which no process maps and only that +/// object's lock reaches. +#[inline(always)] +fn frame_bytes(frame: &u64, within: Range) -> *mut u8 { + if within.end > kernel::pipe::FRAME { + past_a_frame(within); + } + // SAFETY: inside the frame, which the direct map covers. + unsafe { DirectMap::from_phys(*frame).as_mut_ptr::().add(within.start) } +} + +#[cold] +#[inline(never)] +fn past_a_frame(within: Range) -> ! { + panic!("frame_bytes: bytes {within:?} past a frame") +} diff --git a/kernel/src/main.rs b/kernel/src/main.rs index f4e21e76433..e6b5e12fc83 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -78,6 +78,8 @@ mod watch; mod object; mod inbox; mod pipe; +mod frame_bytes; +mod message; mod device; mod pcidev; diff --git a/kernel/src/message.rs b/kernel/src/message.rs new file mode 100644 index 00000000000..beaabfc78ed --- /dev/null +++ b/kernel/src/message.rs @@ -0,0 +1,251 @@ +//! One direction of a connection: whole messages, each its bytes and the +//! handles it carries, from its senders to its receivers. +//! +//! A message is admitted whole under the queue's lock or refused with nothing +//! of it queued, and taken whole or left queued (`kernel::message`). Its +//! bytes are kept in frames counted to the job that made the connection +//! (`crate::frame_bytes`), its handles beside them in the same order. + +use alloc::sync::Arc; +use alloc::vec::Vec; + +use kernel::message::{Head, Queue, Untaken}; +use kernel::pipe::{End, Holders, Released}; +use toyos_abi::handle::RawHandle; +use toyos_abi::syscall::{MAX_QUEUED_HANDLES, MAX_QUEUED_MESSAGES, MAX_TRANSFER_HANDLES}; + +use crate::frame_bytes::FrameBytes; +use crate::mm::charge::{self, Charge, Exhausted}; +use crate::object::handle::{HandleEntry, HandleTable}; +use crate::sync::Lock; +use crate::user_ptr::{UserBytes, UserBytesMut}; +use crate::watch::Watch; + +/// A reference to a queue that holds neither end, as [`crate::pipe::PipeRef`] +/// is a pipe's. +#[derive(Clone)] +pub struct QueueRef(Arc); + +pub use handle::{Receiver, Sender}; + +/// Owned references to a queue's two ends, each counted under the queue's +/// lock as it is made and let go as it drops. +mod handle { + use super::{End, QueueRef}; + + /// One counted reference to the receiving end. + pub struct Receiver(QueueRef); + + /// One counted reference to the sending end. + pub struct Sender(QueueRef); + + impl Receiver { + pub(super) fn acquire(queue: &QueueRef) -> Self { + queue.hold(End::Read); + Self(queue.clone()) + } + + pub fn queue(&self) -> &QueueRef { + &self.0 + } + } + + impl Sender { + pub(super) fn acquire(queue: &QueueRef) -> Self { + queue.hold(End::Write); + Self(queue.clone()) + } + + pub fn queue(&self) -> &QueueRef { + &self.0 + } + } + + impl Drop for Receiver { + fn drop(&mut self) { + self.0.release(End::Read); + } + } + + impl Drop for Sender { + fn drop(&mut self) { + self.0.release(End::Write); + } + } +} + +struct MessageQueue { + state: Lock, + /// Posted by a send and by the last sender's leaving. + read_watch: Arc, + /// Posted by a receive and by the last receiver's leaving. + write_watch: Arc, + /// Itself, its watches and its queue at their bounds, charged to the job + /// that made it, which its bytes are counted to as well. + _charge: Charge, +} + +struct State { + bytes: FrameBytes, + queue: Queue, + holders: Holders, + /// A real-time sender lent its window to whoever takes the next message, + /// which reaches a receiver already runnable that the wake's boost misses. + rt_boost_pending: bool, +} + +/// Why a message was not sent; nothing of it was. +pub enum SendRefused { + /// No room for the whole message now: wait for [`QueueRef::fits`]. + NoRoom, + /// No receiver is left. + Gone, + /// Its bytes' frames could not be taken; no wait makes them appear. + NoMemory, +} + +/// What a receive answered. +pub enum Taken { + /// A whole message, its bytes copied out and its handles installed at + /// these numbers, the first `Head::handles` of them. + Message(Head, [RawHandle; MAX_TRANSFER_HANDLES]), + /// Nothing queued, and a sender is left to send. + Empty, + /// Nothing queued, and no sender is left. + Gone, + /// The front message stays queued. + Untaken(Untaken), +} + +/// Make a queue, answering its two ends. +pub fn create() -> Result<(Receiver, Sender), Exhausted> { + let heap = charge::arc::() + + 2 * charge::arc::() + // `VecDeque` doubles its storage, so each bound's next power of two. + + charge::array::(MAX_QUEUED_MESSAGES.next_power_of_two()) + + charge::array::(MAX_QUEUED_HANDLES.next_power_of_two()); + let charge = Charge::current(heap)?; + let queue = QueueRef(Arc::new(MessageQueue { + state: Lock::new(State { + bytes: FrameBytes::new(charge.job()), + queue: Queue::new(), + holders: Holders::new(), + rt_boost_pending: false, + }), + read_watch: Arc::new(Watch::new()), + write_watch: Arc::new(Watch::new()), + _charge: charge, + })); + Ok((Receiver::acquire(&queue), Sender::acquire(&queue))) +} + +impl QueueRef { + fn hold(&self, end: End) { + self.0.state.lock().holders.hold(end); + } + + /// Queue `bytes` and `batch` as one message, or nothing of it. A refusal + /// hands the batch back: dropped here, it would be capabilities destroyed + /// silently. + pub fn send(&self, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { + let mut state = self.0.state.lock(); + if !state.holders.held(End::Read) { + return Err((batch, SendRefused::Gone)); + } + if !state.queue.fits(state.bytes.space(), bytes.len(), batch.len()) { + return Err((batch, SendRefused::NoRoom)); + } + if state.bytes.write(bytes, bytes.len()).is_err() { + return Err((batch, SendRefused::NoMemory)); + } + state.queue.push(bytes.len(), batch.into_iter()); + Ok(()) + } + + /// Take the front message into `buf` and at most `cap` handles, each + /// installed in `table` under this queue's lock. + pub fn recv(&self, buf: &mut UserBytesMut, cap: usize, table: &mut HandleTable) -> Taken { + let (taken, boost) = { + let mut guard = self.0.state.lock(); + let State { bytes, queue, holders, rt_boost_pending } = &mut *guard; + // Bound, not the block's tail: a tail match's temporaries outlive `guard`. + let taken = match queue.pop(buf.len(), cap, |width| table.make_room(width)) { + Some(Ok((head, handles))) => { + let mut installed = [RawHandle(0); MAX_TRANSFER_HANDLES]; + for (slot, entry) in installed.iter_mut().zip(handles) { + *slot = table.install(entry).expect("room was made first"); + } + bytes.read(buf, head.len); + (Taken::Message(head, installed), core::mem::take(rt_boost_pending)) + } + Some(Err(untaken)) => (Taken::Untaken(untaken), false), + None if holders.held(End::Write) => (Taken::Empty, false), + None => (Taken::Gone, false), + }; + taken + }; + if boost { + // Outside the queue's lock: the scheduler takes its own CPU-queue lock. + crate::scheduler::boost_current_rt_inherited(); + } + taken + } + + /// A message to take, or no sender left: what a receive or a readable + /// poll waits for. + pub fn readable(&self) -> bool { + let state = self.0.state.lock(); + !state.queue.is_empty() || !state.holders.held(End::Write) + } + + /// Room for a message of `len` bytes and `handles`, or no receiver left: + /// what a blocking send waits for. + pub fn fits(&self, len: usize, handles: usize) -> bool { + let state = self.0.state.lock(); + !state.holders.held(End::Read) || state.queue.fits(state.bytes.space(), len, handles) + } + + /// Room for the largest message one send may make, or no receiver left: + /// a writable poll, so a non-blocking send refused and retried on + /// readiness is not refused for room again. + pub fn writable(&self) -> bool { + let state = self.0.state.lock(); + !state.holders.held(End::Read) || state.queue.fits_any(state.bytes.space()) + } + + /// Mark the queue so the next receive inherits RT priority. + pub fn set_rt_boost_pending(&self) { + self.0.state.lock().rt_boost_pending = true; + } + + pub fn read_watch(&self) -> &Arc { + &self.0.read_watch + } + + pub fn write_watch(&self) -> &Arc { + &self.0.write_watch + } + + /// One reference to `end` lets go. The receivers' last gives back every + /// message nobody can take any more and answers waiting senders; the + /// senders' last answers waiting receivers. + fn release(&self, end: End) { + let (released, untaken) = { + let mut state = self.0.state.lock(); + let released = state.holders.release(end); + let untaken = (!state.holders.held(End::Read) && (state.bytes.frames() > 0 || !state.queue.is_empty())).then(|| { + let job = Arc::clone(state.bytes.job()); + (core::mem::replace(&mut state.bytes, FrameBytes::new(&job)), state.queue.clear()) + }); + (released, untaken) + }; + // Outside the lock: frames go back to the allocator, and a handle's + // release can run another object's hook. + drop(untaken); + match (released, end) { + (Released::Held | Released::PipeGone, _) => {} + (Released::EndGone, End::Read) => self.0.write_watch.post(), + (Released::EndGone, End::Write) => self.0.read_watch.post(), + } + } +} diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 1d80f2c6a55..5198dbffdb9 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -8,12 +8,11 @@ use alloc::sync::Arc; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::{FileType, OpenFlags, SeekFrom, SyscallError, MAX_WHOLE_PIPE_WRITE}; -use kernel::sched::task::WaitClass; use crate::drivers::serial; use crate::file_cache; use crate::pipe::{self, PipeRef}; -use kernel::pipe::{End, Landing}; +use kernel::pipe::End; use crate::user_ptr::{UserBytes, UserBytesMut}; use crate::inbox::PollEntry; use crate::watch::Watch; @@ -206,21 +205,19 @@ pub fn close_all(table: &mut HandleTable) { /// The pipe a read of `object` took from, held past the handle table's lock /// to wake its writers. pub fn read_pipe(object: &KObjectRef) -> Option { - pipe_read(object).map(|(pipe, _)| pipe.clone()) + pipe_read(object).cloned() } /// The pipe a write to `object` filled, held the same way to wake its readers. pub fn written_pipe(object: &KObjectRef) -> Option { - pipe_write(object).map(|(pipe, _, _)| pipe.clone()) + pipe_write(object).cloned() } -/// The pipe a blocking read of `object` parks on, and the class its wait is -/// charged to: a connection's is its peer's answer, which is IPC. -pub fn pipe_read(object: &KObjectRef) -> Option<(&PipeRef, WaitClass)> { +/// The pipe a blocking read of `object` parks on. +pub fn pipe_read(object: &KObjectRef) -> Option<&PipeRef> { match object { - KObjectRef::PipeRead(r) => Some((r.pipe(), WaitClass::Pipe)), - KObjectRef::Connection(c) => Some((c.rx(), WaitClass::Ipc)), - KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::PipeRead(r) => Some(r.pipe()), + KObjectRef::Connection(_) | KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) @@ -229,13 +226,11 @@ pub fn pipe_read(object: &KObjectRef) -> Option<(&PipeRef, WaitClass)> { } } -/// [`pipe_read`]'s answer for a blocking write, and how a write lands in -/// that pipe. -pub fn pipe_write(object: &KObjectRef) -> Option<(&PipeRef, WaitClass, Landing)> { +/// [`pipe_read`]'s answer for a blocking write. +pub fn pipe_write(object: &KObjectRef) -> Option<&PipeRef> { match object { - KObjectRef::PipeWrite(w) => Some((w.pipe(), WaitClass::Pipe, Landing::Pipe)), - KObjectRef::Connection(c) => Some((c.tx(), WaitClass::Ipc, Landing::Partial)), - KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::PipeWrite(w) => Some(w.pipe()), + KObjectRef::Connection(_) | KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) @@ -552,12 +547,6 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { match object { KObjectRef::File(f) => read_file(f, buf), KObjectRef::PipeRead(r) => r.pipe().try_read(buf).map(|n| n as u64), - // A batch pinned where the reader is answers the read, so no byte - // after it is read until the batch is taken. - KObjectRef::Connection(c) => c.read(buf).map(|read| match read { - Ok(n) => n as u64, - Err(crate::pipe::AtBatch) => SyscallError::HandlesPending.to_u64(), - }), KObjectRef::Device(_) => unreachable!("a device claim is read by `read_device`"), KObjectRef::Console(_) => { let mut count = 0usize; @@ -586,7 +575,8 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { buf.write_at(0, &[reason.to_byte()]); 1 }), - KObjectRef::PipeWrite(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) + // A connection carries whole messages, which `SYS_MESSAGE_RECV` takes. + KObjectRef::Connection(_) | KObjectRef::PipeWrite(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) => Some(SyscallError::PermissionDenied.to_u64()), @@ -594,12 +584,12 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { } fn write_pipe(object: &KObjectRef, buf: &UserBytes) -> Option { - let (pipe, _, landing) = pipe_write(object).expect("a pipe's write end writes its pipe"); - match pipe.write(buf, landing) { + let pipe = pipe_write(object).expect("a pipe's write end writes its pipe"); + match pipe.write(buf) { // The reader is gone, not absent: `NotFound` means the name is not there and nothing else may say so. pipe::PipeWrite::BrokenPipe => Some(SyscallError::Gone.to_u64()), pipe::PipeWrite::NoMemory => Some(SyscallError::ResourceExhausted.to_u64()), - pipe::PipeWrite::Wrote { n, .. } => Some(n as u64), + pipe::PipeWrite::Wrote(n) => Some(n as u64), pipe::PipeWrite::NoRoom => None, } } @@ -611,13 +601,14 @@ pub fn try_write(object: &KObjectRef, buf: &UserBytes) -> Option { state.position += written; Some(answer) }), - KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) => write_pipe(object, buf), + KObjectRef::PipeWrite(_) => write_pipe(object, buf), // The whole lines `klogd`'s queue has room for, and a trailing partial one: // a short count is a writer ahead of the console, told so. KObjectRef::Console(c) => Some(c.write(buf) as u64), KObjectRef::Device(d) => Some(write_device(d, buf)), KObjectRef::Null(_) => Some(buf.len() as u64), - KObjectRef::PipeRead(_) | KObjectRef::Acceptor(_) + // A connection carries whole messages, which `SYS_MESSAGE_SEND` sends. + KObjectRef::Connection(_) | KObjectRef::PipeRead(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) => { @@ -735,7 +726,7 @@ pub fn ftruncate(object: &KObjectRef, size: u64) -> u64 { pub fn has_data(object: &KObjectRef) -> bool { match object { KObjectRef::PipeRead(r) => r.pipe().has_data(), - KObjectRef::Connection(c) => c.has_data(), + KObjectRef::Connection(c) => c.rx().readable(), KObjectRef::Console(_) => serial::has_data(), KObjectRef::Acceptor(a) => a.has_pending(), KObjectRef::Process(p) => p.finished(), @@ -780,7 +771,7 @@ pub fn has_space(object: &KObjectRef) -> bool { // Writable is room for any write that lands whole, so a non-blocking // one refused, retried on readiness, lands. KObjectRef::PipeWrite(w) => w.pipe().has_space(MAX_WHOLE_PIPE_WRITE), - KObjectRef::Connection(c) => c.tx().has_space(1), + KObjectRef::Connection(c) => c.tx().writable(), KObjectRef::File(_) | KObjectRef::Null(_) => true, KObjectRef::Console(_) => crate::log::console::has_room(), KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) diff --git a/kernel/src/object/port.rs b/kernel/src/object/port.rs index 11b06bad591..26901762159 100644 --- a/kernel/src/object/port.rs +++ b/kernel/src/object/port.rs @@ -13,22 +13,20 @@ use alloc::sync::Arc; use core::sync::atomic::{AtomicU64, Ordering}; use crate::mm::charge::{self, Exhausted}; -use crate::pipe::{PipeReader, PipeWriter}; +use crate::message::{Receiver, Sender}; use crate::sync::Lock; use crate::watch::Watch; -use super::service::HandleQueue; use super::{KObjectVariant, ObjectCore, ZeroHandles}; /// Unaccepted connections one port may hold; past it, connect returns `ResourceExhausted`. pub const MAX_PENDING_CONNECTIONS: usize = 32; -/// A connection nobody has accepted yet: owns the server's pipe ends and handle queues. +/// A connection nobody has accepted yet: owns the server's ends of its two +/// directions. pub struct PendingConnection { - pub rx: PipeReader, - pub tx: PipeWriter, - pub inbox: Arc, - pub outbox: Arc, + pub rx: Receiver, + pub tx: Sender, /// [`Connector::stamp`] of the connector the client connected through. pub stamp: Stamp, } @@ -187,17 +185,13 @@ impl Connector { impl ZeroHandles for Acceptor { fn on_zero_handles(&self) { // `queued` drops only after the guard releases, since dropping a - // handle can re-enter another object's hook. + // handle can re-enter another object's hook. Its receivers go with + // it, so a client's send on a connection nobody accepted is `Gone`. let queued = { let mut queue = self.shared.queue.lock(); queue.closed = true; core::mem::take(&mut queue.pending) }; - // Nobody will ever hold this inbox's read end, so a queued - // `SYS_MESSAGE_SEND` on it must say `Gone`, not queue. - for connection in &queued { - connection.inbox.close_now(); - } drop(queued); // A poll on the port is answered as gone, not as ready: nothing will ever queue again. self.shared.watch.cancel_polls(); diff --git a/kernel/src/object/service.rs b/kernel/src/object/service.rs index 2ee268e87b5..5d31dad3c4d 100644 --- a/kernel/src/object/service.rs +++ b/kernel/src/object/service.rs @@ -1,142 +1,27 @@ -//! A connection: two pipes for the bytes, two handle queues for the handles. +//! A connection: two message queues, one per direction (`crate::message`). use alloc::sync::Arc; -use alloc::vec::Vec; -use kernel::message::{Batches, Refused, Window}; -use kernel::pipe::Landing; -use toyos_abi::syscall::{SyscallError, MAX_QUEUED_BATCHES, MAX_TRANSFER_HANDLES}; +use crate::message::{QueueRef, Receiver, Sender}; +use crate::mm::charge::{self, Exhausted}; -use crate::mm::charge::{self, Charge, Exhausted}; -use crate::pipe::{AtBatch, PipeReader, PipeRef, PipeWrite, PipeWriter}; -use crate::user_ptr::{UserBytes, UserBytesMut}; -use crate::sync::Lock; - -use super::handle::HandleEntry; use super::port::{PendingConnection, Stamp}; use super::{Held, KObjectVariant, ObjectCore, ZeroHandles}; -/// Handles in flight in one direction of a connection, each batch pinned at -/// the stream position it follows (`kernel::message`); `None` once the reader -/// is gone. -/// -/// **Its lock is taken before the pipe's**, by the sender for the whole of a -/// message and by the reader for a read or a receive, so a reader that has -/// read a message's bytes finds its batch pinned. -/// -/// Charged at its bound when it is made, so that no send has a heap -/// allocation of its own to be refused. -pub struct HandleQueue { - batches: Lock>>>, - _charge: Charge, -} - -/// Why a message was not sent; nothing of it was. -pub enum SendRefused { - /// No room for the whole message now: wait on the write watch. - NoRoom, - Error(SyscallError), -} - -impl HandleQueue { - fn open() -> Result, Exhausted> { - // `VecDeque` doubles its storage, so the bound's next power of two. - let bound = charge::arc::() - + charge::array::<(u64, Vec)>(MAX_QUEUED_BATCHES.next_power_of_two()) - + MAX_QUEUED_BATCHES * charge::array::(MAX_TRANSFER_HANDLES); - let charge = Charge::current(bound)?; - Ok(Arc::new(Self { batches: Lock::new(Some(Batches::new())), _charge: charge })) - } - - /// Write `bytes` to `tx` and pin `batch` just past them, whole or not at - /// all. A refusal hands the batch back: dropped here, it would be - /// capabilities destroyed silently. - fn send(&self, tx: &PipeRef, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { - let mut guard = self.batches.lock(); - let Some(batches) = guard.as_mut() else { return Err((batch, SendRefused::Error(SyscallError::Gone))) }; - let carries = !batch.is_empty(); - match batches.admit(bytes.len(), carries, MAX_QUEUED_BATCHES) { - Ok(()) => {} - Err(Refused::Empty) => return Err((batch, SendRefused::Error(SyscallError::InvalidArgument))), - Err(Refused::QueueFull) => return Err((batch, SendRefused::Error(SyscallError::ResourceExhausted))), - } - match tx.write(bytes, Landing::Whole) { - PipeWrite::Wrote { end, .. } => { - if carries { - batches.pin(end, batch); - } - Ok(()) - } - PipeWrite::NoRoom => Err((batch, SendRefused::NoRoom)), - PipeWrite::BrokenPipe => Err((batch, SendRefused::Error(SyscallError::Gone))), - PipeWrite::NoMemory => Err((batch, SendRefused::Error(SyscallError::ResourceExhausted))), - } - } - - /// A read of `rx`, stopping at the next pinned batch. - fn read(&self, rx: &PipeRef, buf: &mut UserBytesMut) -> Option> { - let guard = self.batches.lock(); - rx.try_read_within(buf, |read, available| match guard.as_ref() { - Some(batches) => batches.window(read, available), - None => Window::Bytes(available), - }) - } - - /// Take the batch pinned at `rx`'s read position, refusing without taking - /// when it is wider than `cap`, and leaving it pinned when `room` refuses. - fn take(&self, rx: &PipeRef, cap: usize, room: impl FnOnce(usize) -> bool) -> Result>, SyscallError> { - let mut guard = self.batches.lock(); - let Some(batches) = guard.as_mut() else { return Ok(None) }; - let read = rx.read_position(); - let Some(width) = batches.at(read).map(Vec::len) else { return Ok(None) }; - if width > cap { - return Err(SyscallError::InvalidArgument); - } - if !room(width) { - return Err(SyscallError::ResourceExhausted); - } - Ok(batches.take(read)) - } - - /// Whether a batch is pinned where the reader is, which is something to receive. - fn pinned_here(&self, rx: &PipeRef) -> bool { - let guard = self.batches.lock(); - let Some(batches) = guard.as_ref() else { return false }; - batches.at(rx.read_position()).is_some() - } - - /// Drops the batches outside the lock: releasing a handle can run another object's hook. - pub(super) fn close_now(&self) { - let batches = self.batches.lock().take(); - drop(batches); - } -} - -/// One end of a connection: the queues are cross-wired, this end's `outbox` is the peer's `inbox`. +/// One end of a connection: it receives on `rx` and sends on `tx`, which are +/// the peer's `tx` and `rx`. pub struct ConnectionEnd { pub(super) core: ObjectCore, - rx: PipeRef, - tx: PipeRef, - inbox: Arc, - outbox: Arc, - reference: Held<(PipeReader, PipeWriter)>, + rx: QueueRef, + tx: QueueRef, + reference: Held<(Receiver, Sender)>, /// The server's end of a port's connection; `None` on a client's end. stamp: Option, } impl ConnectionEnd { - /// One call for both ends, so the cross-wiring can't be gotten wrong from two constructors. - pub fn pair_queues() -> Result<(Arc, Arc), Exhausted> { - Ok((HandleQueue::open()?, HandleQueue::open()?)) - } - - pub fn new( - rx: PipeReader, - tx: PipeWriter, - inbox: Arc, - outbox: Arc, - ) -> Result, Exhausted> { - Ok(Self::build(Self::new_core(0)?, rx, tx, inbox, outbox, None)) + pub fn new(rx: Receiver, tx: Sender) -> Result, Exhausted> { + Ok(Self::build(Self::new_core(0)?, rx, tx, None)) } /// What a server's end is charged, a stamp's badge at its bound @@ -153,65 +38,32 @@ impl ConnectionEnd { /// The server's end of a queued connection, over what /// [`accept_core`](Self::accept_core) charged, keeping its stamp. pub fn accepted(core: ObjectCore, conn: PendingConnection) -> Arc { - Self::build(core, conn.rx, conn.tx, conn.inbox, conn.outbox, Some(conn.stamp)) + Self::build(core, conn.rx, conn.tx, Some(conn.stamp)) } - fn build( - core: ObjectCore, - rx: PipeReader, - tx: PipeWriter, - inbox: Arc, - outbox: Arc, - stamp: Option, - ) -> Arc { - Arc::new(Self { - core, - rx: rx.pipe().clone(), - tx: tx.pipe().clone(), - inbox, - outbox, - reference: Held::new((rx, tx)), - stamp, - }) + fn build(core: ObjectCore, rx: Receiver, tx: Sender, stamp: Option) -> Arc { + Arc::new(Self { core, rx: rx.queue().clone(), tx: tx.queue().clone(), reference: Held::new((rx, tx)), stamp }) } pub fn stamp(&self) -> Option<&Stamp> { self.stamp.as_ref() } - pub fn rx(&self) -> &PipeRef { + /// What this end receives. + pub fn rx(&self) -> &QueueRef { &self.rx } - pub fn tx(&self) -> &PipeRef { + /// What this end sends to. + pub fn tx(&self) -> &QueueRef { &self.tx } - - /// See [`HandleQueue::send`]: a refusal comes back with the batch. - pub fn send(&self, bytes: &UserBytes, batch: Vec) -> Result<(), (Vec, SendRefused)> { - self.outbox.send(&self.tx, bytes, batch) - } - - /// See [`HandleQueue::read`]. - pub fn read(&self, buf: &mut UserBytesMut) -> Option> { - self.inbox.read(&self.rx, buf) - } - - /// See [`HandleQueue::take`]. - pub fn recv(&self, cap: usize, room: impl FnOnce(usize) -> bool) -> Result>, SyscallError> { - self.inbox.take(&self.rx, cap, room) - } - - /// Bytes to read, the peer's end, or a batch pinned where the reader is. - pub fn has_data(&self) -> bool { - self.rx.has_data() || self.inbox.pinned_here(&self.rx) - } } impl ZeroHandles for ConnectionEnd { fn on_zero_handles(&self) { - // Only the inbox: what this end sent is still the peer's to receive. - self.inbox.close_now(); + // The receiver's leaving gives back what was sent to this end; what it + // sent stays the peer's to receive. self.reference.release(); } } diff --git a/kernel/src/pipe.rs b/kernel/src/pipe.rs index b26a8ffaace..7b1882ccd8d 100644 --- a/kernel/src/pipe.rs +++ b/kernel/src/pipe.rs @@ -1,11 +1,7 @@ -use core::ops::Range; - +use crate::frame_bytes::FrameBytes; use crate::mm::charge::{self, Charge, Exhausted}; -use crate::mm::{pmm, DirectMap}; -use kernel::message::Window; -use kernel::pipe::{Bytes, End, Holders, Landing, Released, MOST_FRAMES}; -use kernel::sched::job::Job; +use kernel::pipe::{End, Holders, Released}; use alloc::sync::Arc; @@ -83,111 +79,6 @@ mod handle { } } - -/// What a pipe holds: its bytes, in frames counted to the job that made the -/// pipe, and the heap their list takes, charged to it. The frames go back -/// when it drops. -struct Held { - bytes: Bytes, - list: Charge, -} - -impl Held { - fn new(job: &Arc) -> Self { - Self { bytes: Bytes::new(), list: Charge::none(job) } - } - - /// Take the frames a write of `n` bytes wants, all of them or none. - fn take_frames(&mut self, n: usize) -> Result<(), Exhausted> { - let wants = self.bytes.wants(n); - if wants == 0 { - return Ok(()); - } - let frames = self.bytes.frames() + wants; - if frames > self.bytes.listed() { - let list = frames.next_power_of_two().min(MOST_FRAMES); - self.list.resize(charge::array::(list))?; - self.bytes.list(list); - } - let bytes = &mut self.bytes; - pmm::alloc_held(self.list.job(), wants, |frame| bytes.add(frame)).then_some(()).ok_or(Exhausted) - } - - /// Give back the spares a read left past those it keeps. - fn give_back_surplus(&mut self) { - // SAFETY: a spare holds no byte, and nothing but this list names it. - unsafe { pmm::free_held(self.list.job(), self.bytes.surplus()) }; - } -} - -impl Drop for Held { - fn drop(&mut self) { - // SAFETY: nothing but this list names its frames. - unsafe { pmm::free_held(self.list.job(), self.bytes.clear()) }; - } -} - -/// One copy gathered from runs that continue each other in both source and -/// destination, so a user page's piece over two physically adjacent frames -/// is one copy, not two. -struct Gather { - from: *const u8, - to: *mut u8, - len: usize, -} - -impl Gather { - const NONE: Self = Self { from: core::ptr::null(), to: core::ptr::null_mut(), len: 0 }; - - /// Add `len` bytes from `from` to `to`, copying what was gathered first - /// unless these continue it. - /// - /// # Safety - /// `from` is readable and `to` writable for `len` bytes, and stay so - /// until the next [`Self::flush`]; the two never overlap. - #[inline(always)] - unsafe fn add(&mut self, from: *const u8, to: *mut u8, len: usize) { - if self.len > 0 && self.from.wrapping_add(self.len) == from && self.to.wrapping_add(self.len) == to { - self.len += len; - return; - } - // SAFETY: what was gathered meets this function's contract. - unsafe { self.flush() }; - *self = Self { from, to, len }; - } - - /// Copy what was gathered. - /// - /// # Safety - /// As [`Self::add`] for every run gathered. - #[inline(always)] - unsafe fn flush(&mut self) { - if self.len > 0 { - // SAFETY: the runs gathered are contiguous on both sides and - // each met `add`'s contract. - unsafe { core::ptr::copy_nonoverlapping(self.from, self.to, self.len) }; - } - *self = Self::NONE; - } -} - -/// `within` of a frame a pipe holds, which no process maps and only that -/// pipe's lock reaches. -#[inline(always)] -fn frame_bytes(frame: &u64, within: Range) -> *mut u8 { - if within.end > kernel::pipe::FRAME { - past_a_frame(within); - } - // SAFETY: inside the frame, which the direct map covers. - unsafe { DirectMap::from_phys(*frame).as_mut_ptr::().add(within.start) } -} - -#[cold] -#[inline(never)] -fn past_a_frame(within: Range) -> ! { - panic!("pipe: bytes {within:?} past a frame") -} - struct Pipe { state: Lock, /// The read end's and the write end's watches. Held by `Arc` so a blocking site or a @@ -206,14 +97,10 @@ struct Pipe { /// What a pipe's lock holds. struct State { /// Given back once no reader is left to read it. - held: Held, + held: FrameBytes, holders: Holders, /// Set when a write should hand the next reader transient RT priority — covers readers already runnable, which the wake-time boost misses. rt_boost_pending: bool, - /// Bytes ever written and ever read: the stream positions a connection's - /// batches are pinned at (`kernel::message`), which never wrap. - written: u64, - read: u64, } impl Pipe { @@ -229,9 +116,9 @@ impl Pipe { /// what it needs, or is refused if none is left. pub fn create() -> Result<(PipeReader, PipeWriter), Exhausted> { let charge = Charge::current(Pipe::heap())?; - let held = Held::new(charge.job()); + let held = FrameBytes::new(charge.job()); let pipe = PipeRef(Arc::new(Pipe { - state: Lock::new(State { held, holders: Holders::new(), rt_boost_pending: false, written: 0, read: 0 }), + state: Lock::new(State { held, holders: Holders::new(), rt_boost_pending: false }), read_watch: Arc::new(Watch::new()), write_watch: Arc::new(Watch::new()), left: [Arc::new(Watch::new()), Arc::new(Watch::new())], @@ -240,14 +127,9 @@ pub fn create() -> Result<(PipeReader, PipeWriter), Exhausted> { Ok((PipeReader::acquire(&pipe), PipeWriter::acquire(&pipe))) } -/// A read stopped at a batch pinned where the reader is. -#[derive(Debug)] -pub struct AtBatch; - pub enum PipeWrite { - /// This many bytes, the stream ending at `end`. - Wrote { n: usize, end: u64 }, - /// Not the room its landing needs now: wait for `Landing::room`. + Wrote(usize), + /// Not the room the write needs now: wait for `kernel::pipe::room`. NoRoom, BrokenPipe, /// The memory its bytes need could not be taken; no wait for room makes @@ -261,50 +143,15 @@ impl PipeRef { } pub fn try_read(&self, buf: &mut UserBytesMut) -> Option { - self.try_read_within(buf, |_, available| Window::Bytes(available)) - .map(|read| read.expect("a read with no window stops nowhere")) - } - - /// [`Self::try_read`], taking only what `window` allows of the bytes the - /// pipe holds, given the reader's stream position: [`AtBatch`] where it - /// says the reader is at a pinned batch. - pub fn try_read_within( - &self, - buf: &mut UserBytesMut, - window: impl FnOnce(u64, usize) -> Window, - ) -> Option> { let (result, boost) = { let mut pipe = self.0.state.lock(); - let available = pipe.held.bytes.len(); - let limit = match window(pipe.read, available) { - Window::AtBatch => return Some(Err(AtBatch)), - Window::Bytes(limit) => limit, - }; - if available > 0 && limit > 0 { - let len = buf.len().min(limit); - let n = len.min(available); - let bytes = &mut pipe.held.bytes; - // One walk of the user window, each pinned piece copied out of - // the frames it covers. - buf.write_pieces(0, n, |to, at, piece| { - let mut gather = Gather::NONE; - bytes.peek(at, piece, |frame, within, off| { - // SAFETY: `to` is valid for `piece` bytes until this - // walk's callback returns, and `off + within.len()` is - // at most `piece`; the frame is this pipe's alone, - // reached only under its lock, and no process maps it. - unsafe { gather.add(frame_bytes(frame, within.clone()), to.add(off), within.len()) } - }); - // SAFETY: as above, before the callback returns. - unsafe { gather.flush() }; - }); - bytes.consume(n); - pipe.held.give_back_surplus(); - pipe.read += n as u64; - let boost = core::mem::take(&mut pipe.rt_boost_pending); - (Some(Ok(n)), boost) + let available = pipe.held.len(); + if available > 0 { + let n = buf.len().min(available); + pipe.held.read(buf, n); + (Some(n), core::mem::take(&mut pipe.rt_boost_pending)) } else if !pipe.holders.held(End::Write) && available == 0 { - (Some(Ok(0)), false) + (Some(0), false) } else { (None, false) } @@ -316,49 +163,29 @@ impl PipeRef { result } - /// The stream position the reader is at: every byte read so far. - pub fn read_position(&self) -> u64 { - self.0.state.lock().read - } - - /// Write `buf` as `landing` has it land, under the one lock hold. - pub fn write(&self, buf: &UserBytes, landing: Landing) -> PipeWrite { + /// Write `buf` as a pipe write lands (`kernel::pipe::takes`), under the + /// one lock hold. + pub fn write(&self, buf: &UserBytes) -> PipeWrite { let mut pipe = self.0.state.lock(); if !pipe.holders.held(End::Read) { return PipeWrite::BrokenPipe; } - let Some(n) = landing.takes(pipe.held.bytes.space(), buf.len()) else { return PipeWrite::NoRoom }; - if pipe.held.take_frames(n).is_err() { + let Some(n) = kernel::pipe::takes(pipe.held.space(), buf.len()) else { return PipeWrite::NoRoom }; + if pipe.held.write(buf, n).is_err() { return PipeWrite::NoMemory; } - let bytes = &mut pipe.held.bytes; - // One walk of the user window, each pinned piece copied into the - // frames it covers. - buf.read_pieces(0, n, |from, at, piece| { - let mut gather = Gather::NONE; - bytes.place(at, piece, |frame, within, off| { - // SAFETY: `from` is valid for `piece` bytes until this walk's - // callback returns, and `off + within.len()` is at most `piece`; - // the frame is this pipe's alone, as above. - unsafe { gather.add(from.add(off), frame_bytes(frame, within.clone()), within.len()) } - }); - // SAFETY: as above, before the callback returns. - unsafe { gather.flush() }; - }); - bytes.commit(n); - pipe.written += n as u64; - PipeWrite::Wrote { n, end: pipe.written } + PipeWrite::Wrote(n) } pub fn has_data(&self) -> bool { let pipe = self.0.state.lock(); - pipe.holders.readable(pipe.held.bytes.len()) + pipe.holders.readable(pipe.held.len()) } /// Whether a write waiting for `want` bytes of room answers now. pub fn has_space(&self, want: usize) -> bool { let pipe = self.0.state.lock(); - pipe.holders.writable(pipe.held.bytes.space(), want) + pipe.holders.writable(pipe.held.space(), want) } /// Whether the end opposite `end` of this pipe has no holder left. @@ -378,9 +205,9 @@ impl PipeRef { let (released, unread) = { let mut pipe = self.0.state.lock(); let released = pipe.holders.release(end); - let unread = (!pipe.holders.held(End::Read) && pipe.held.bytes.frames() > 0).then(|| { - let job = Arc::clone(pipe.held.list.job()); - core::mem::replace(&mut pipe.held, Held::new(&job)) + let unread = (!pipe.holders.held(End::Read) && pipe.held.frames() > 0).then(|| { + let job = Arc::clone(pipe.held.job()); + core::mem::replace(&mut pipe.held, FrameBytes::new(&job)) }); (released, unread) }; diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index d8f9eb89661..29edbd62adc 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -36,7 +36,8 @@ use super::io::{ sys_null, sys_random, sys_read, sys_read_nonblock, sys_write, sys_write_nonblock, }; use super::ipc::{ - sys_accept, sys_handle_recv, sys_inbox_setup, sys_message_send, + Incoming, Outgoing, + sys_accept, sys_inbox_setup, sys_message_recv, sys_message_send, sys_inbox_submit, sys_namespace_build, sys_namespace_open, sys_pipe, sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, sys_shm_sub, }; @@ -460,36 +461,15 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_ACCEPT => sys_accept(RawHandle(a1 as u32), a2), SYS_MESSAGE_SEND => { let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; - let unknown_flags = args.flags & !MESSAGE_NONBLOCK != 0; - if unknown_flags || args.count as usize > MAX_TRANSFER_HANDLES || args.len as usize > MAX_MESSAGE_LEN { - return SyscallError::InvalidArgument.to_u64(); - } - let Some(bytes) = ctx.user_bytes(UserAddr::new(args.bytes), args.len) else { return bad_addr }; - let Some(handles) = (args.count as usize) - .checked_mul(HANDLE_LEN) - .and_then(|len| ctx.user_bytes(UserAddr::new(args.handles), len as u64)) - else { - return bad_addr; - }; - sys_message_send( - RawHandle(a1 as u32), - &bytes, - &handles, - args.count as usize, - args.flags & MESSAGE_NONBLOCK != 0, - ) - } - SYS_HANDLE_RECV => { - if a3 as usize > MAX_TRANSFER_HANDLES { - return SyscallError::InvalidArgument.to_u64(); - } - let Some(mut out) = (a3 as usize) - .checked_mul(HANDLE_LEN) - .and_then(|len| ctx.user_bytes_mut(UserAddr::new(a2), len as u64)) - else { - return bad_addr; - }; - sys_handle_recv(RawHandle(a1 as u32), &mut out, a3 as usize) + let Ok(out) = outgoing(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; + let Some(out) = out else { return bad_addr }; + sys_message_send(RawHandle(a1 as u32), &out, args.flags & MESSAGE_NONBLOCK != 0) + } + SYS_MESSAGE_RECV => { + let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; + let Ok(into) = incoming(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; + let Some(mut into) = into else { return bad_addr }; + sys_message_recv(RawHandle(a1 as u32), &mut into, args.flags & MESSAGE_NONBLOCK != 0) } SYS_SHM_CREATE => sys_shm_create(a1, a2, RawHandle(a3 as u32)), SYS_SHM_MAP => sys_shm_map(RawHandle(a1 as u32), a2), @@ -695,3 +675,37 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> result } + +/// A send's buffers, bounded: `Err` for arguments past the ABI's bounds or an +/// unknown flag, `None` for an address the caller does not have. +fn outgoing<'a>(ctx: &SyscallContext<'a>, args: &MessageSend) -> Result>, ()> { + if args.flags & !MESSAGE_NONBLOCK != 0 || args.count as usize > MAX_TRANSFER_HANDLES || args.len as usize > MAX_MESSAGE_LEN { + return Err(()); + } + let count = args.count as usize; + Ok((|| { + Some(Outgoing { + bytes: ctx.user_bytes(UserAddr::new(args.bytes), args.len)?, + handles: ctx.user_bytes(UserAddr::new(args.handles), (count * HANDLE_LEN) as u64)?, + count, + }) + })()) +} + +/// A receive's buffers, bounded the same way. A buffer larger than any +/// message or batch holds every one, so only its first +/// [`MAX_MESSAGE_LEN`] bytes and [`MAX_TRANSFER_HANDLES`] slots are taken. +fn incoming<'a>(ctx: &SyscallContext<'a>, args: &MessageRecv) -> Result>, ()> { + if args.flags & !MESSAGE_NONBLOCK != 0 { + return Err(()); + } + let len = args.len.min(MAX_MESSAGE_LEN as u64); + let cap = (args.cap as usize).min(MAX_TRANSFER_HANDLES); + Ok((|| { + Some(Incoming { + bytes: ctx.user_bytes_mut(UserAddr::new(args.bytes), len)?, + handles: ctx.user_bytes_mut(UserAddr::new(args.handles), (cap * HANDLE_LEN) as u64)?, + cap, + }) + })()) +} diff --git a/kernel/src/syscall/io.rs b/kernel/src/syscall/io.rs index 3ce765d996b..57b81d8a182 100644 --- a/kernel/src/syscall/io.rs +++ b/kernel/src/syscall/io.rs @@ -22,8 +22,8 @@ use super::handles::with_object_ref; /// What `sys_write` does when the object took nothing. enum WriteBlock { - /// The pipe, the class of the wait, and the room the write waits for. - Pipe(pipe::PipeRef, WaitClass, usize), + /// The pipe, and the room the write waits for. + Pipe(pipe::PipeRef, usize), Refused(u64), /// Carried out of the process's lock: `HandleError::refuse` may take the /// process down and cannot run under a guard. @@ -32,7 +32,7 @@ enum WriteBlock { /// What `sys_read` parks on when the handle has nothing to give. enum ReadBlock { - Pipe(pipe::PipeRef, WaitClass), + Pipe(pipe::PipeRef), /// A notice with no reason in it, until a quit puts one there. Notice(alloc::sync::Arc), /// A console, which reads the serial line and nothing else: nothing posts @@ -55,7 +55,7 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { match ops::try_write(object, buf) { Some(n) => Ok((n, ops::written_pipe(object))), None => Err(match ops::pipe_write(object) { - Some((pipe, class, landing)) => WriteBlock::Pipe(pipe.clone(), class, landing.room(buf.len())), + Some(pipe) => WriteBlock::Pipe(pipe.clone(), kernel::pipe::room(buf.len())), None => WriteBlock::Refused(SyscallError::NotFound.to_u64()), }), } @@ -65,9 +65,9 @@ pub(super) fn sys_write(h: RawHandle, buf: &UserBytes) -> u64 { if let Some(pipe) = pipe { process::wake_pipe_readers(&pipe); } return n; } - Err(WriteBlock::Pipe(pipe, class, room)) => { + Err(WriteBlock::Pipe(pipe, room)) => { let parkable = crate::scheduler::Parkable::at_entry(); - if watch::wait_until(&parkable, pipe.write_watch(), 0, class, Deadline::never(), || pipe.has_space(room)) + if watch::wait_until(&parkable, pipe.write_watch(), 0, WaitClass::Pipe, Deadline::never(), || pipe.has_space(room)) .is_err() { return cancelled(); @@ -91,7 +91,7 @@ fn read_block(object: &KObjectRef) -> ReadBlock { ); ReadBlock::Console(Deadline::at(crate::clock::now() + CONSOLE_REPOLL.duration())) } - _ => match ops::pipe_read(object).map(|(pipe, class)| ReadBlock::Pipe(pipe.clone(), class)) { + _ => match ops::pipe_read(object).map(|pipe| ReadBlock::Pipe(pipe.clone())) { Some(block) => block, None => ReadBlock::Refused(SyscallError::NotFound.to_u64()), }, @@ -129,9 +129,9 @@ pub(super) fn sys_read(h: RawHandle, buf: &mut UserBytesMut) -> u64 { if let Some(pipe) = pipe { process::wake_pipe_writers(&pipe); } return n; } - Err(ReadBlock::Pipe(pipe, class)) => { + Err(ReadBlock::Pipe(pipe)) => { let parkable = crate::scheduler::Parkable::at_entry(); - if watch::wait_until(&parkable, pipe.read_watch(), 0, class, Deadline::never(), || pipe.has_data()) + if watch::wait_until(&parkable, pipe.read_watch(), 0, WaitClass::Pipe, Deadline::never(), || pipe.has_data()) .is_err() { return cancelled(); diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index e6a7e6a00d5..9390a3d02ac 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -11,11 +11,12 @@ use alloc::vec::Vec; use crate::watch; use crate::object::{ops, port, KObjectRef}; -use crate::object::service::SendRefused; +use crate::message::{SendRefused, Taken}; +use crate::object::service::ConnectionEnd; use crate::time::Deadline; use crate::user_ptr::SyscallContext; use crate::UserAddr; -use crate::{pipe, process}; +use crate::{message, pipe, process}; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::*; @@ -198,24 +199,11 @@ fn connect_through(connector: &port::Connector) -> u64 { return SyscallError::Gone.to_u64(); } let made = (|| { - let (cs_reader, cs_writer) = pipe::create()?; // client → server - let (sc_reader, sc_writer) = pipe::create()?; // server → client - // Cross-wired here only: the server's end is built from the same two queues when it accepts. - let (to_server, to_client) = crate::object::service::ConnectionEnd::pair_queues()?; - - let object = KObjectRef::Connection(crate::object::service::ConnectionEnd::new( - sc_reader, // client reads from server→client - cs_writer, // client writes to client→server - to_client.clone(), // and receives what the server sent - to_server.clone(), - )?); - let pending = port::PendingConnection { - rx: cs_reader, // server reads from client→server - tx: sc_writer, // server writes to server→client - inbox: to_server, - outbox: to_client, - stamp: connector.stamp(), - }; + let (to_server_rx, to_server_tx) = message::create()?; + let (to_client_rx, to_client_tx) = message::create()?; + // Cross-wired here only: the server's end is built from the pending half when it accepts. + let object = KObjectRef::Connection(ConnectionEnd::new(to_client_rx, to_server_tx)?); + let pending = port::PendingConnection { rx: to_server_rx, tx: to_client_tx, stamp: connector.stamp() }; Ok::<_, crate::mm::charge::Exhausted>((object, pending)) })(); let (object, pending) = match made { @@ -237,7 +225,7 @@ fn connect_through(connector: &port::Connector) -> u64 { }); let h = match installed { Ok(h) => h, - // Both halves of a refused connection drop here, with the hold given up: their pipes go with them. + // Both halves of a refused connection drop here, with the hold given up: their queues go with them. Err((e, _client, _server)) => return e.to_u64(), }; // The port's one watch carries both: the server blocked in `accept` and every ring polling it. @@ -260,13 +248,13 @@ pub(super) fn sys_accept(h: RawHandle, flags: u64) -> u64 { loop { // Charged before a connection is taken, so a refusal leaves it queued. - let core = match crate::object::service::ConnectionEnd::accept_core() { + let core = match ConnectionEnd::accept_core() { Ok(core) => core, Err(e) => return SyscallError::from(e).to_u64(), }; if let Some(conn) = acceptor.pop() { - // PipeReader/PipeWriter move from queue into connection: ownership transfers, no refcount change. - let object = KObjectRef::Connection(crate::object::service::ConnectionEnd::accepted(core, conn)); + // The ends move from the port's queue into the connection: ownership transfers, no count changes. + let object = KObjectRef::Connection(ConnectionEnd::accepted(core, conn)); let installed = process::with_process_data(|data| { ops::install(&mut data.handles, object) }); @@ -320,7 +308,7 @@ pub(super) fn sys_port_badge( ) -> u64 { let found = process::with_process_data(|data| { let acceptor = data.handles.get::(acceptor_h, Rights::READ)?; - let conn = data.handles.get::(conn_h, Rights::READ)?; + let conn = data.handles.get::(conn_h, Rights::READ)?; Ok::<_, crate::object::HandleError>((acceptor.port_id(), conn.stamp().cloned())) }); let (port, stamp) = match found { @@ -414,32 +402,49 @@ pub(super) fn sys_shm_map(h: RawHandle, prot: u64) -> u64 { at.raw() } -/// Write a message's bytes to a connection and move its handles to the peer, +/// A send's arguments as the dispatch bounded them. +pub(super) struct Outgoing<'a> { + pub bytes: crate::user_ptr::UserBytes<'a>, + pub handles: crate::user_ptr::UserBytes<'a>, + pub count: usize, +} + +/// A receive's buffers as the dispatch bounded them. +pub(super) struct Incoming<'a> { + pub bytes: crate::user_ptr::UserBytesMut<'a>, + pub handles: crate::user_ptr::UserBytesMut<'a>, + pub cap: usize, +} + +/// Queue a whole message on a connection and move its handles to the peer, /// together or not at all ([`toyos_abi::syscall::message_send`]). Rights, -/// `TRANSFER` included, travel unchanged. Waits for room in the stream unless -/// `nonblock`; the copy is the caller's, bounded by `MAX_MESSAGE_LEN`. -pub(super) fn sys_message_send( - conn_h: RawHandle, - bytes: &crate::user_ptr::UserBytes, - handles: &crate::user_ptr::UserBytes, - count: usize, - nonblock: bool, -) -> u64 { +/// `TRANSFER` included, travel unchanged. Waits for room unless `nonblock`. +pub(super) fn sys_message_send(conn_h: RawHandle, out: &Outgoing<'_>, nonblock: bool) -> u64 { + send(conn_h, out, nonblock).map_or_else(|word| word, |()| 0) +} + +/// Take a connection's next whole message ([`toyos_abi::syscall::message_recv`]). +pub(super) fn sys_message_recv(conn_h: RawHandle, into: &mut Incoming<'_>, nonblock: bool) -> u64 { + recv(conn_h, into, nonblock) +} + +/// [`sys_message_send`]'s work, or the word its refusal answers. +fn send(conn_h: RawHandle, out: &Outgoing<'_>, nonblock: bool) -> Result<(), u64> { let mut wanted = [RawHandle(0); MAX_TRANSFER_HANDLES]; - for (i, slot) in wanted.iter_mut().enumerate().take(count) { + for (i, slot) in wanted.iter_mut().enumerate().take(out.count) { let mut raw = [0u8; HANDLE_LEN]; - handles.read_at(i * HANDLE_LEN, &mut raw); + out.handles.read_at(i * HANDLE_LEN, &mut raw); *slot = RawHandle(u32::from_ne_bytes(raw)); } - let wanted = &wanted[..count]; + let wanted = &wanted[..out.count]; // `TRANSFER` on the connection only when it carries handles: bytes alone // are a write, which `WRITE` already allows. - let need = if count == 0 { Rights::WRITE } else { Rights::WRITE.union(Rights::TRANSFER) }; + let need = if out.count == 0 { Rights::WRITE } else { Rights::WRITE.union(Rights::TRANSFER) }; loop { let sent = process::with_process_data(|data| { - let conn = data.handles.get::(conn_h, need)?; - // Every handle is checked before anything is written or moved. + let conn = data.handles.get::(conn_h, need)?; + // Every handle is checked before anything is queued or moved. for (i, h) in wanted.iter().enumerate() { if *h == conn_h || wanted[..i].contains(h) { return Err(SyscallError::InvalidArgument.into()); @@ -450,55 +455,65 @@ pub(super) fn sys_message_send( } } // A refusal restores every handle at its own number, so it is invisible. - let moved = data.handles.transfer(wanted, |batch| conn.send(bytes, batch)); - Ok::<_, crate::object::Refusal>((conn.tx().clone(), moved)) + let tx = conn.tx().clone(); + let moved = data.handles.transfer(wanted, |batch| tx.send(&out.bytes, batch)); + Ok::<_, crate::object::Refusal>((tx, moved)) }); let tx = match sent { Ok((tx, Ok(()))) => { - process::wake_pipe_readers(&tx); - return 0; + crate::scheduler::post_lending(tx.read_watch(), || tx.set_rt_boost_pending()); + return Ok(()); } Ok((tx, Err(SendRefused::NoRoom))) if !nonblock => tx, - Ok((_, Err(SendRefused::NoRoom))) => return SyscallError::WouldBlock.to_u64(), - Ok((_, Err(SendRefused::Error(e)))) => return e.to_u64(), - Err(e) => return e.refuse(), + Ok((_, Err(SendRefused::NoRoom))) => return Err(SyscallError::WouldBlock.to_u64()), + Ok((_, Err(SendRefused::Gone))) => return Err(SyscallError::Gone.to_u64()), + Ok((_, Err(SendRefused::NoMemory))) => return Err(SyscallError::ResourceExhausted.to_u64()), + Err(e) => return Err(e.refuse()), }; - // Outside every lock: room appears as the reader drains, and its read posts this watch. - let len = bytes.len(); + // Outside every lock: room appears as the receiver takes messages, and each receive posts this watch. + let (len, count) = (out.bytes.len(), out.count); if watch::wait_until(&crate::scheduler::Parkable::at_entry(), tx.write_watch(), 0, WaitClass::Ipc, Deadline::never(), || { - tx.has_space(len) + tx.fits(len, count) }) .is_err() { - return cancelled(); + return Err(cancelled()); } } } -/// Take the batch pinned at the reader's place in the stream. Never blocks; -/// zero means no batch sits here. -pub(super) fn sys_handle_recv( - conn_h: RawHandle, - out: &mut crate::user_ptr::UserBytesMut, - cap: usize, -) -> u64 { - let taken = process::with_process_data(|data| { - let conn = data - .handles - .get::(conn_h, Rights::READ)?; - // Width and room are measured before the batch is taken: refusing after taking would drop handles nobody could ask for again. - let Some(batch) = conn.recv(cap, |width| data.handles.make_room(width))? else { return Ok(0) }; - let count = batch.len(); - for (i, entry) in batch.into_iter().enumerate() { - let h = data.handles.install(entry).expect("room was asked for first"); - out.write_at(i * HANDLE_LEN, &h.0.to_ne_bytes()); +fn recv(conn_h: RawHandle, into: &mut Incoming<'_>, nonblock: bool) -> u64 { + loop { + let taken = process::with_process_data(|data| { + let conn = data.handles.get::(conn_h, Rights::READ)?; + let rx = conn.rx().clone(); + let taken = rx.recv(&mut into.bytes, into.cap, &mut data.handles); + Ok::<_, crate::object::Refusal>((rx, taken)) + }); + let rx = match taken { + Ok((rx, Taken::Message(head, installed))) => { + for (i, h) in installed[..head.handles].iter().enumerate() { + into.handles.write_at(i * HANDLE_LEN, &h.0.to_ne_bytes()); + } + // Room for the sender. + rx.write_watch().post(); + return toyos_abi::syscall::Received::encode(head.len, head.handles); + } + Ok((_, Taken::Gone)) => return SyscallError::Gone.to_u64(), + Ok((_, Taken::Untaken(kernel::message::Untaken::TooSmall))) => return SyscallError::InvalidArgument.to_u64(), + Ok((_, Taken::Untaken(kernel::message::Untaken::NoRoom))) => return SyscallError::ResourceExhausted.to_u64(), + Ok((_, Taken::Empty)) if nonblock => return SyscallError::WouldBlock.to_u64(), + Ok((rx, Taken::Empty)) => rx, + // Refused outside this hold: ending the caller takes the same non-reentrant lock. + Err(e) => return e.refuse(), + }; + if watch::wait_until(&crate::scheduler::Parkable::at_entry(), rx.read_watch(), 0, WaitClass::Ipc, Deadline::never(), || { + rx.readable() + }) + .is_err() + { + return cancelled(); } - Ok::<_, crate::object::Refusal>(count as u64) - }); - // Refused outside this hold: ending the caller takes the same non-reentrant lock. - match taken { - Ok(n) => n, - Err(e) => e.refuse(), } } diff --git a/lib/link/src/driver.rs b/lib/link/src/driver.rs index 445fe45b1c4..371226d9fe0 100644 --- a/lib/link/src/driver.rs +++ b/lib/link/src/driver.rs @@ -18,10 +18,10 @@ use toyos::endow::{self, EndowError}; use toyos::ipc::{Connection, FrameRx, RxStep}; +use toyos::wake::{Bell, Waker}; use toyos::poller::{Poller, READABLE}; use toyos::shm::ReadBytes; use toyos::{say, PciDev}; -use toyos_abi::syscall::SyscallError; use crate::ends::{Driver, Fault}; use crate::layout::{RX_DEPTH, SLOT_BYTES, TX_DEPTH}; @@ -121,11 +121,16 @@ enum Ended { struct Session<'a> { label: &'a str, - conn: Connection, + /// The hello's connection, until the stack answers it. + conn: Option, region: Region, end: Driver, rx: FrameRx<4>, - opened: bool, + /// Rung for the stack; the stack's bell answers `Hungup` once it goes. + ring: Waker, + /// The stack's doorbell, once the link is open; its waker going is the + /// stack's end. + rung: Option, said: Option, /// Received frames longer than a slot, dropped. oversize: u64, @@ -141,24 +146,37 @@ impl<'a> Session<'a> { .unwrap_or_else(|| panic!("{label}: the device's address is a group's, which no link carries")); let hello = Hello { mac, medium: N::MEDIUM, label: label.into() }; let shared = region.share().unwrap_or_else(|why| panic!("{label}: the link's region would not share: {why:?}")); + let (ring, stack_hears) = toyos::wake::pair().unwrap_or_else(|why| panic!("{label}: no doorbell for a link: {why:?}")); // A connection that will not take the hello is one whose stack has // gone: its hang-up is read in the first pass. - let _ = conn.try_send_bytes_with(wire::MSG_HELLO, &hello.encode(), [shared]); - Self { label, conn, region, end, rx: FrameRx::new(), opened: false, said: None, oversize: 0, oversize_said: 0 } + let _ = conn.try_send_bytes_with(wire::MSG_HELLO, &hello.encode(), [shared, stack_hears.into()]); + Self { + label, + conn: Some(conn), + region, + end, + rx: FrameRx::new(), + ring, + rung: None, + said: None, + oversize: 0, + oversize_said: 0, + } } fn serve(&mut self, nic: &mut impl Nic) -> Ended { let poller = Poller::new(2); loop { let ring = self.pass(nic).unwrap_or_else(|why| panic!("{}: the stack wrote {why}", self.label)); - if ring && self.opened { - match self.conn.write_nonblock(&[1]) { - Ok(_) | Err(SyscallError::WouldBlock) => {} - Err(_) => return Ended::HungUp("it would not take a doorbell"), - } + if ring && self.rung.is_some() && self.ring.ring().is_err() { + return Ended::HungUp("its doorbell is gone"); } poller.watch(nic.claim(), READABLE, TOKEN_CLAIM); - poller.watch(&self.conn, READABLE, TOKEN_LINK); + match (&self.rung, &self.conn) { + (Some(rung), _) => poller.watch(rung, READABLE, TOKEN_LINK), + (None, Some(conn)) => poller.watch(conn, READABLE, TOKEN_LINK), + (None, None) => unreachable!("{}: a link neither opening nor open", self.label), + } poller.wait(1, nic.pass_due_in().unwrap_or(u64::MAX), |_| {}); if let Some(ended) = self.hear() { return ended; @@ -234,35 +252,31 @@ impl<'a> Session<'a> { Ok(ring) } - /// Read the stack's answer, then its doorbells; the end of the session + /// Read the stack's answer, then its doorbell; the end of the session /// if that is what came. fn hear(&mut self) -> Option { - if !self.opened { - return match self.rx.pump(&self.conn) { - RxStep::Idle => None, - RxStep::Eof => Some(Ended::HungUp("its process ended before it answered")), - RxStep::Frame { msg_type: wire::MSG_OPENED, payload_len: 0, handles: 0 } => { - self.opened = true; - say!("{}: the link is open", self.label); - self.hear() - } - RxStep::Frame { msg_type: wire::MSG_REFUSED, payload_len, handles: 0 } => { - match Refusal::decode(self.rx.payload(payload_len)) { - Some(why) => Some(Ended::Refused(why)), - None => panic!("{}: the stack refused this link with a word no stack writes", self.label), - } - } - RxStep::Frame { .. } | RxStep::Malformed => panic!("{}: the stack answered a hello with a frame no stack writes", self.label), - }; + if let Some(rung) = &self.rung { + return rung.try_take().err().map(|_| Ended::HungUp("its process ended")); } - let mut sink = [0u8; 64]; - loop { - match self.conn.read_nonblock(&mut sink) { - Ok(0) => return Some(Ended::HungUp("its process ended")), - Ok(_) => {} - Err(SyscallError::WouldBlock) => return None, - Err(_) => return Some(Ended::HungUp("it refused a read")), + let conn = self.conn.as_ref().expect("a link not open is opening"); + match self.rx.pump(conn) { + RxStep::Idle => None, + RxStep::Eof => Some(Ended::HungUp("its process ended before it answered")), + RxStep::Frame { msg_type: wire::MSG_OPENED, payload_len: 0, handles: 1 } => { + let [rung] = self.rx.take_handles::<1>().expect("counted above"); + self.rung = Some(Bell::from(rung)); + // Its work is done: the bells carry the link from here. + self.conn = None; + say!("{}: the link is open", self.label); + self.hear() + } + RxStep::Frame { msg_type: wire::MSG_REFUSED, payload_len, handles: 0 } => { + match Refusal::decode(self.rx.payload(payload_len)) { + Some(why) => Some(Ended::Refused(why)), + None => panic!("{}: the stack refused this link with a word no stack writes", self.label), + } } + RxStep::Frame { .. } | RxStep::Malformed => panic!("{}: the stack answered a hello with a frame no stack writes", self.label), } } } diff --git a/lib/link/src/lib.rs b/lib/link/src/lib.rs index fb6a452d8dd..59936db774a 100644 --- a/lib/link/src/lib.rs +++ b/lib/link/src/lib.rs @@ -8,10 +8,11 @@ //! [`state`]); its arena holds a slot per ring entry, which a frame is copied //! into and out of once on each side ([`ends`]). //! -//! **A doorbell is a byte on the connection**, written after what it -//! announces is in the region. The connection's hang-up is the link's end: -//! the stack keeps every connection it carries across it, and the driver -//! started again opens a new link. +//! **A doorbell is an event**, rung after what it announces is in the +//! region: each side holds the waker of the other's bell, sent with the +//! hello and its answer ([`wire`]). A bell whose waker is gone is the link's +//! end: the stack keeps every connection it carries across it, and the +//! driver started again opens a new link. //! //! The rings, the frames' lengths and every state word are pure and //! host-tested; [`region`] maps the region and [`driver`] carries a device's diff --git a/lib/link/src/wire.rs b/lib/link/src/wire.rs index fe3769dfae3..28d2367ab73 100644 --- a/lib/link/src/wire.rs +++ b/lib/link/src/wire.rs @@ -1,10 +1,10 @@ //! The frames a link is opened with, before the region carries anything. //! //! **One way in, one answer.** A driver says who it is ([`MSG_HELLO`]) and -//! sends the region it made with it; the stack answers [`MSG_OPENED`] or -//! [`MSG_REFUSED`] with a [`Refusal`]. After `MSG_OPENED` the connection -//! carries nothing but doorbell bytes, each way, and its hang-up ends the -//! link. +//! sends the region it made and the bell it rings with it; the stack answers +//! [`MSG_OPENED`] with the bell it rings, or [`MSG_REFUSED`] with a +//! [`Refusal`]. After `MSG_OPENED` the connection has nothing left to carry, +//! and either side's waker going ends the link. use alloc::string::String; use alloc::vec::Vec; @@ -25,9 +25,13 @@ impl Address { } } -/// Driver to stack: a [`Hello`]. Handles: the region. +/// Driver to stack: a [`Hello`]. Handles: the region, then the bell the +/// driver rings. pub const MSG_HELLO: u32 = 1; -/// Stack to driver: the link is open. No payload. +/// Driver to stack: the hello's handles. +pub const HELLO_HANDLES: usize = 2; +/// Stack to driver: the link is open. No payload. Handles: the bell the +/// stack rings. pub const MSG_OPENED: u32 = 2; /// Stack to driver: refused, a [`Refusal`]'s word. pub const MSG_REFUSED: u32 = 3; diff --git a/lib/logkeeper-api/src/lib.rs b/lib/logkeeper-api/src/lib.rs index 76147b05ed6..998b98b8191 100644 --- a/lib/logkeeper-api/src/lib.rs +++ b/lib/logkeeper-api/src/lib.rs @@ -18,13 +18,14 @@ pub struct Served { pub fn read() -> Result { let conn = endow::service(SERVICE).map_err(|e| format!("no `{SERVICE}` service: {e:?}"))?; conn.signal(READ).map_err(|e| format!("logkeeper would not take the request: {e:?}"))?; - let header = conn.recv_header().map_err(|e| format!("logkeeper did not answer: {e:?}"))?; - if header.msg_type != SERVED { - return Err(format!("logkeeper answered frame type {}", header.msg_type)); + let mut rx = toyos::ipc::FrameRx::<8>::new(); + let frame = rx.recv(&conn).map_err(|e| format!("logkeeper did not answer: {e:?}"))?; + if frame.msg_type != SERVED { + return Err(format!("logkeeper answered frame type {}", frame.msg_type)); } - let boot_so_far: u64 = - conn.recv_payload(&header).map_err(|e| format!("logkeeper's answer is not one `u64`: {e:?}"))?; - let [pipe] = conn.recv_handles_exact::<1>(&header).ok_or("logkeeper's answer carried no pipe")?; + let boot_so_far: u64 = toyos::ipc::decode_payload(rx.payload(frame.payload_len)) + .map_err(|e| format!("logkeeper's answer is not one `u64`: {e:?}"))?; + let [pipe] = rx.take_handles::<1>().ok_or("logkeeper's answer carried no pipe")?; let pipe = Pipe::from(pipe); Ok(Served { pipe, boot_so_far }) } diff --git a/lib/session/src/lib.rs b/lib/session/src/lib.rs index a5ec3ebdc61..58433c80d46 100644 --- a/lib/session/src/lib.rs +++ b/lib/session/src/lib.rs @@ -278,13 +278,13 @@ pub fn open( ) -> Result<(toyos::namespace::Namespace, toyos::ipc::Connection), String> { let conn = login.open(LOGIN_PORT).map_err(|e| format!("the supervisor's login port refused: {e:?}"))?; conn.signal(MSG_OPEN).map_err(|e| format!("the open was not sent: {e:?}"))?; - let header = conn.recv_header().map_err(|e| format!("the open was not answered: {e:?}"))?; - if header.msg_type != MSG_OPENED { - let mut why = vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut why).unwrap_or(0); - return Err(format!("the supervisor refused a session: {}", String::from_utf8_lossy(&why[..n]))); + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let frame = rx.recv(&conn).map_err(|e| format!("the open was not answered: {e:?}"))?; + if frame.msg_type != MSG_OPENED { + let why = String::from_utf8_lossy(rx.payload(frame.payload_len)); + return Err(format!("the supervisor refused a session: {why}")); } - let [launcher, controls] = conn.recv_handles_exact::<2>(&header).ok_or("the session came without its handles")?; + let [launcher, controls] = rx.take_handles::<2>().ok_or("the session came without its handles")?; let control = toyos::namespace::Namespace::received(controls) .open(CONTROL) .map_err(|e| format!("the session's controls refused its control connection: {e:?}"))?; diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 8b9a055af0e..0e7691b7641 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -164,9 +164,6 @@ pub const SYS_NAMESPACE_BUILD: u64 = 101; /// no other place to ask, and a name it was not given resolves to nothing. pub const SYS_NAMESPACE_OPEN: u64 = 102; -/// Take the batch of handles at the reader's place in the connection's bytes. -/// See [`handle_recv`]. -pub const SYS_HANDLE_RECV: u64 = 104; /// Make a shared-memory region, sized up to whole [`PAGE_SIZE`]s. /// See [`shm_create`]. pub const SYS_SHM_CREATE: u64 = 105; @@ -296,8 +293,8 @@ pub const SYS_NULL: u64 = 130; pub const SYS_DEVICE_DMA_LIMIT: u64 = 131; /// Let a claimed PCI function master the bus. See [`device_dma_start`]. pub const SYS_DEVICE_DMA_START: u64 = 132; -/// Write a message's bytes to a connection and move the handles it carries, -/// together or not at all. See [`message_send`]. +/// Queue a whole message on a connection, its bytes and the handles it +/// carries, or nothing of it. See [`message_send`]. pub const SYS_MESSAGE_SEND: u64 = 133; /// A handle to the job the caller runs in. See [`job_own`]. pub const SYS_JOB_OWN: u64 = 134; @@ -362,6 +359,8 @@ pub const SYS_PAGED_READ: u64 = 156; pub const SYS_PAGED_OP: u64 = 157; /// What a pager's objects ask of it. See [`paged_notices`]. pub const SYS_PAGED_NOTICES: u64 = 158; +/// Take a connection's next whole message. See [`message_recv`]. +pub const SYS_MESSAGE_RECV: u64 = 159; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -369,13 +368,13 @@ pub const SYS_PAGED_NOTICES: u64 = 158; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 160; +pub const SYSCALL_PROFILE_BINS: usize = 161; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_PAGED_NOTICES < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_MESSAGE_RECV < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -592,9 +591,6 @@ pub enum SyscallError { /// the same rule the storage layer already obeys applies here: a dead peer /// must not be indistinguishable from a handle that was never there. Gone = 10, - /// A read of a connection reached a batch of handles the peer sent with - /// the bytes before it: take the batch ([`handle_recv`]) before reading on. - HandlesPending = 11, } impl SyscallError { @@ -619,7 +615,6 @@ impl SyscallError { 8 => Some(Self::NotSupported), 9 => Some(Self::Io), 10 => Some(Self::Gone), - 11 => Some(Self::HandlesPending), _ => Some(Self::Unknown), } } @@ -639,7 +634,6 @@ impl core::fmt::Display for SyscallError { Self::NotSupported => f.write_str("not supported"), Self::Io => f.write_str("the device did not complete the transfer"), Self::Gone => f.write_str("the other end is gone"), - Self::HandlesPending => f.write_str("handles the peer sent wait at this point of the stream"), } } } @@ -2107,15 +2101,20 @@ pub fn accept_nonblock(acceptor: RawHandle) -> Result { /// and never truncated to fit. pub const MAX_TRANSFER_HANDLES: usize = 8; -/// Batches one direction of a connection may hold unreceived. -pub const MAX_QUEUED_BATCHES: usize = 16; +/// Messages one direction of a connection may hold unreceived. +pub const MAX_QUEUED_MESSAGES: usize = 256; -/// The most bytes one [`message_send`] writes. Policy on the primitive: the -/// bytes are copied whole under the connection's locks, so the copy is -/// bounded, and every frame a protocol here sends is far smaller. +/// Handles one direction of a connection may hold unreceived, across its +/// messages. +pub const MAX_QUEUED_HANDLES: usize = 64; + +/// The most bytes one message carries. Policy on the primitive: the bytes are +/// copied whole under the connection's lock, so the copy is bounded, and bulk +/// data travels in shared memory. pub const MAX_MESSAGE_LEN: usize = 16 * 1024; -/// [`message_send`] answers `WouldBlock` rather than waiting for room. +/// [`message_send`] or [`message_recv`] answers `WouldBlock` rather than +/// waiting. pub const MESSAGE_NONBLOCK: u64 = 1; crate::user_safe! { @@ -2131,21 +2130,33 @@ crate::user_safe! { } } -/// Write `bytes` to `conn` and move `handles` to its peer, as one message. +impl MessageSend { + fn new(bytes: &[u8], handles: &[RawHandle], nonblock: bool) -> Self { + Self { + bytes: bytes.as_ptr() as u64, + len: bytes.len() as u64, + handles: handles.as_ptr() as u64, + count: handles.len() as u64, + flags: if nonblock { MESSAGE_NONBLOCK } else { 0 }, + } + } +} + +/// Queue `bytes` and `handles` on `conn` as one message for its peer. /// -/// **All or nothing, and in order.** The bytes go into the connection's -/// stream whole, and the handles are queued at the byte just past them: the -/// peer reads the bytes, then takes the batch ([`handle_recv`]), and no read -/// of its crosses the batch before it is taken. Nothing is written and no -/// handle moves on a refusal: +/// **All or nothing, and in order.** The peer takes each message whole, its +/// bytes and exactly its handles, in the order they were sent +/// ([`message_recv`]). Nothing is queued and no handle moves on a refusal: /// - every handle must carry [`Rights::TRANSFER`], and so must `conn` when /// there are any; `conn` itself and a handle named twice are refused; -/// - handles need bytes (`InvalidArgument`), and past [`MAX_MESSAGE_LEN`] -/// bytes or [`MAX_TRANSFER_HANDLES`] handles is `InvalidArgument`; -/// - a peer already holding [`MAX_QUEUED_BATCHES`] is `ResourceExhausted`, and -/// a peer that is gone is `Gone`; -/// - a stream without room for the whole message waits for it, or with -/// `nonblock` answers `WouldBlock`. +/// - past [`MAX_MESSAGE_LEN`] bytes or [`MAX_TRANSFER_HANDLES`] handles is +/// `InvalidArgument`; +/// - a peer that is gone is `Gone`, and memory for the bytes the machine +/// does not have is `ResourceExhausted`; +/// - without room for the whole message beside the peer's unreceived ones +/// (at most [`MAX_QUEUED_MESSAGES`], [`MAX_QUEUED_HANDLES`] and +/// [`PIPE_CAPACITY`] bytes) it waits for room, or with `nonblock` answers +/// `WouldBlock`. `conn` polls writable once the largest message fits. /// /// [`Rights::TRANSFER`]: crate::handle::Rights::TRANSFER /// @@ -2153,32 +2164,69 @@ crate::user_safe! { /// Each of `handles` is the caller's to give away: a sent handle leaves this /// process, so nothing may answer for it afterwards ([`crate::handle`]). pub unsafe fn message_send(conn: RawHandle, bytes: &[u8], handles: &[RawHandle], nonblock: bool) -> Result<(), SyscallError> { - let args = MessageSend { - bytes: bytes.as_ptr() as u64, - len: bytes.len() as u64, - handles: handles.as_ptr() as u64, - count: handles.len() as u64, - flags: if nonblock { MESSAGE_NONBLOCK } else { 0 }, - }; + let args = MessageSend::new(bytes, handles, nonblock); check_unit(syscall(SYS_MESSAGE_SEND, conn.0 as u64, &args as *const MessageSend as u64, 0, 0)) } -/// Take the batch the peer sent at this reader's place in the connection's -/// bytes, into `out`. Answers how many it wrote. +crate::user_safe! { + /// Arguments for `SYS_MESSAGE_RECV`, passed as a single pointer: where + /// the message's bytes and handles go. + #[derive(Clone, Copy)] + pub struct MessageRecv { + pub bytes: u64, + pub len: u64, + pub handles: u64, + pub cap: u64, + /// [`MESSAGE_NONBLOCK`] or none; any other bit is `InvalidArgument`. + pub flags: u64, + } +} + +impl MessageRecv { + fn new(bytes: &mut [u8], handles: &mut [RawHandle], nonblock: bool) -> Self { + Self { + bytes: bytes.as_mut_ptr() as u64, + len: bytes.len() as u64, + handles: handles.as_mut_ptr() as u64, + cap: handles.len() as u64, + flags: if nonblock { MESSAGE_NONBLOCK } else { 0 }, + } + } +} + +/// A message taken: how many bytes and handles it carried. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Received { + pub len: usize, + pub handles: usize, +} + +impl Received { + /// The answer's word: the length low, the handle count above it. + fn decode(word: u64) -> Self { + Self { len: (word & 0xFFFF_FFFF) as usize, handles: (word >> 32) as usize } + } + + /// The word the kernel answers. + pub const fn encode(len: usize, handles: usize) -> u64 { + len as u64 | (handles as u64) << 32 + } +} + +const _: () = assert!(Received::encode(MAX_MESSAGE_LEN, MAX_TRANSFER_HANDLES) < u64::MAX - 255); + +/// Take `conn`'s next message whole: its bytes into `bytes` and its handles, +/// each installed in this process's table, into `handles`. /// -/// Never blocks: zero means no batch sits here, which for a reader that has -/// just read a message carrying handles is a peer that broke its protocol. A -/// batch larger than `out` is `InvalidArgument` and stays queued — `out` -/// should be [`MAX_TRANSFER_HANDLES`] long. -pub fn handle_recv(conn: RawHandle, out: &mut [RawHandle]) -> Result { - check(syscall( - SYS_HANDLE_RECV, - conn.0 as u64, - out.as_mut_ptr() as u64, - out.len() as u64, - 0, - )) - .map(|n| n as usize) +/// Waits for one unless `nonblock`, which answers `WouldBlock`. Once the peer +/// has gone and nothing is queued, `Gone`. **A message is never cut**: one +/// larger than `bytes` or carrying more handles than `handles` holds is +/// `InvalidArgument`, and a table without room for its handles +/// `ResourceExhausted`, both leaving it queued. A protocol bounds its own +/// messages, so either is the peer breaking it. +pub fn message_recv(conn: RawHandle, bytes: &mut [u8], handles: &mut [RawHandle], nonblock: bool) -> Result { + let args = MessageRecv::new(bytes, handles, nonblock); + check(syscall(SYS_MESSAGE_RECV, conn.0 as u64, &args as *const MessageRecv as u64, 0, 0)).map(Received::decode) } /// Make a shared-memory region of zeroed memory and answer a handle to it. diff --git a/sdk/filepicker/src/lib.rs b/sdk/filepicker/src/lib.rs index c69dff01fae..0ea0143c2d1 100644 --- a/sdk/filepicker/src/lib.rs +++ b/sdk/filepicker/src/lib.rs @@ -72,14 +72,13 @@ pub fn pick_file(mode: PickerMode, start_dir: &str) -> Result, Pi conn.send_bytes(MSG_FILEPICKER_REQUEST, &data[..1 + len]) .map_err(|_| PickError::PickerGone)?; - let header = conn.recv_header().map_err(|_| PickError::PickerGone)?; - if header.msg_type != MSG_FILEPICKER_RESULT { - return Err(PickError::Protocol(header.msg_type)); + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let frame = rx.recv(&conn).map_err(|_| PickError::PickerGone)?; + if frame.msg_type != MSG_FILEPICKER_RESULT { + return Err(PickError::Protocol(frame.msg_type)); } - if header.len() == 0 { + if frame.payload_len == 0 { return Ok(None); } - let mut buf = [0u8; 4096]; - let n = conn.recv_bytes(&header, &mut buf).map_err(|_| PickError::PickerGone)?; - String::from_utf8(buf[..n].to_vec()).map(Some).map_err(|_| PickError::NotUtf8) + String::from_utf8(rx.payload(frame.payload_len).to_vec()).map(Some).map_err(|_| PickError::NotUtf8) } diff --git a/sdk/libc/src/errcode.rs b/sdk/libc/src/errcode.rs index d4ad92a6e2d..ebc2c8d9300 100644 --- a/sdk/libc/src/errcode.rs +++ b/sdk/libc/src/errcode.rs @@ -44,7 +44,6 @@ pub(crate) const ENOPROTOOPT: i32 = 92; pub(crate) const EOPNOTSUPP: i32 = 95; pub(crate) const EAFNOSUPPORT: i32 = 97; pub(crate) const EADDRINUSE: i32 = 98; -pub(crate) const EPROTO: i32 = 71; pub(crate) const ECONNABORTED: i32 = 103; pub(crate) const ECONNRESET: i32 = 104; pub(crate) const EISCONN: i32 = 106; @@ -69,8 +68,6 @@ pub(crate) fn of(e: SyscallError) -> i32 { // The kernel's word for a full handle table and for no memory alike. SyscallError::ResourceExhausted => ENOMEM, SyscallError::NotSupported => EOPNOTSUPP, - // A read of a connection met handles no frame announced: the peer broke its protocol. - SyscallError::HandlesPending => EPROTO, _ => EINVAL, } } diff --git a/sdk/std/sys/pal/mod.rs b/sdk/std/sys/pal/mod.rs index bd87cda9a27..e2d59454146 100644 --- a/sdk/std/sys/pal/mod.rs +++ b/sdk/std/sys/pal/mod.rs @@ -32,8 +32,6 @@ pub fn to_io_error(e: toyos_abi::syscall::SyscallError) -> crate::io::Error { SyscallError::NotSupported => ErrorKind::Unsupported, SyscallError::Io => ErrorKind::Other, SyscallError::Gone => ErrorKind::BrokenPipe, - // A read of a connection met handles the peer sent with no frame announcing them. - SyscallError::HandlesPending => ErrorKind::InvalidData, }; crate::io::Error::from(kind) } diff --git a/sdk/toyos/src/audio.rs b/sdk/toyos/src/audio.rs index 6798c5a8d36..e902ada00ae 100644 --- a/sdk/toyos/src/audio.rs +++ b/sdk/toyos/src/audio.rs @@ -2,7 +2,7 @@ use core::sync::atomic::Ordering; use toyos_abi::audio::AudioSlotHeader; -use crate::ipc::IpcError; +use crate::ipc::{self, FrameRx, IpcError}; use crate::shm::SharedMemory; pub const MSG_STREAM_OPEN: u32 = 1; @@ -250,9 +250,10 @@ pub struct AudioStream { pub fn describe_device() -> Result { let control = AudioStream::connect_soundserver()?; control.signal(MSG_DEVICE_DESCRIBE).map_err(AudioError::Ipc)?; - let header = control.recv_header().map_err(AudioError::Ipc)?; - match header.msg_type { - MSG_DEVICE_DESCRIBED => control.recv_payload(&header).map_err(AudioError::Ipc), + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&control).map_err(AudioError::Ipc)?; + match frame.msg_type { + MSG_DEVICE_DESCRIBED => ipc::decode_payload(rx.payload(frame.payload_len)).map_err(AudioError::Ipc), other => Err(AudioError::Protocol(other)), } } @@ -263,16 +264,15 @@ impl AudioStream { let req = StreamOpenRequest { sample_rate, channels, format }; control.send(MSG_STREAM_OPEN, &req).map_err(AudioError::Ipc)?; - let header = control.recv_header().map_err(AudioError::Ipc)?; - let resp: StreamOpenResponse = match header.msg_type { - MSG_STREAM_OPENED => control.recv_payload(&header).map_err(AudioError::Ipc)?, + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&control).map_err(AudioError::Ipc)?; + let resp: StreamOpenResponse = match frame.msg_type { + MSG_STREAM_OPENED => ipc::decode_payload(rx.payload(frame.payload_len)).map_err(AudioError::Ipc)?, MSG_STREAM_ERROR => return Err(AudioError::Rejected), other => return Err(AudioError::Protocol(other)), }; - let [shm, signal] = control - .recv_handles_exact::(&header) - .ok_or(AudioError::MissingHandles)?; + let [shm, signal] = rx.take_handles::().ok_or(AudioError::MissingHandles)?; let signal = crate::wake::Bell::from(signal); let slot_count = resp.slot_count as u32; diff --git a/sdk/toyos/src/device.rs b/sdk/toyos/src/device.rs index c3cf0933ae1..2912a78e293 100644 --- a/sdk/toyos/src/device.rs +++ b/sdk/toyos/src/device.rs @@ -34,7 +34,7 @@ device_info!( /// /// Bounded by [`IpcPayload`] and not by `Copy`: the answer's bytes are /// *reinterpreted* as a `T`, so what the bound has to promise is that every bit -/// pattern of that width is one — the contract [`crate::ipc::recv_payload`] +/// pattern of that width is one — the contract [`crate::ipc::decode_payload`] /// reads on, held here by the kernel rather than by a peer. pub(crate) fn read_info(dev: &Device) -> Result { let size = core::mem::size_of::(); diff --git a/sdk/toyos/src/fs.rs b/sdk/toyos/src/fs.rs index ccb2703747b..8a415f418f9 100644 --- a/sdk/toyos/src/fs.rs +++ b/sdk/toyos/src/fs.rs @@ -51,7 +51,7 @@ use alloc::vec::Vec; use toyos_abi::syscall::{SyscallError, MAX_BADGE, MAX_SERVICE_NAME}; -use crate::ipc::{Connection, IpcError}; +use crate::ipc::{Connection, FrameRx, IpcError}; use crate::ipc_payload; use crate::namespace::Namespace; use crate::shm::{Bytes, SharedMemory}; @@ -831,14 +831,14 @@ impl Dir { self.tag = self.tag.wrapping_add(1); request.tag = self.tag; let conn = self.conn.as_ref().expect("connected above"); + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); let answered = conn.send(op, &request).map_err(transport).and_then(|()| { - let header = conn.recv_header().map_err(transport)?; - let reply: Reply = conn.recv_payload(&header).map_err(transport)?; - let carried = conn.recv_handles(&header).map_err(transport)?; + let frame = rx.recv(conn).map_err(transport)?; + let reply: Reply = crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(transport)?; if reply.tag != request.tag { return Err(Error::Malformed); } - Ok((header.msg_type, reply, carried)) + Ok((frame.msg_type, reply, rx.take_batch())) }); if answered.is_err() { self.conn = None; @@ -1292,9 +1292,10 @@ fn hello(conn: &Connection, window: &SharedMemory) -> Result { Ok(()) | Err(Error::Gone) => {} Err(e) => return Err(e), } - let header = conn.recv_header().map_err(transport)?; - let reply = conn.recv_payload(&header).map_err(transport)?; - answer(header.msg_type, reply) + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(conn).map_err(transport)?; + let reply = crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(transport)?; + answer(frame.msg_type, reply) } /// A connection's transport refused, as the protocol names it. diff --git a/sdk/toyos/src/ipc.rs b/sdk/toyos/src/ipc.rs index 9aca638ec48..cb44491c914 100644 --- a/sdk/toyos/src/ipc.rs +++ b/sdk/toyos/src/ipc.rs @@ -1,16 +1,12 @@ -//! Framed IPC over connections (bidirectional pipe pairs). +//! Framed IPC over connections. //! -//! Wire format: `[u32 msg_type][u32 len][u32 handles][len bytes payload]`. -//! -//! **A frame and the handles it announces travel as one message** -//! (`SYS_MESSAGE_SEND`): the kernel writes the frame whole and pins its batch -//! at the byte just past it, so the receiver reads the frame and takes exactly -//! the batch it announces, there. A read that reaches a batch no frame -//! announced is answered `HandlesPending`, which is the peer breaking the -//! protocol; the connection is dropped and the batch goes with it. +//! A frame is `[u32 msg_type][payload]` and the handles it carries, sent as +//! one kernel message (`SYS_MESSAGE_SEND`) and taken whole +//! (`SYS_MESSAGE_RECV`): a receiver never sees part of a frame, and a +//! frame's handles arrive with it and with no other. use toyos_abi::RawHandle; -use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::syscall::{self, Received, SyscallError}; use crate::{AsHandle, BorrowedHandle, OwnedHandle}; /// A type that may cross an IPC boundary as a payload. @@ -20,7 +16,7 @@ use crate::{AsHandle, BorrowedHandle, OwnedHandle}; /// pointers, and have no invalid bit patterns. All three are load-bearing: /// [`send`] publishes `size_of::()` bytes of the sender's memory to a peer, /// so a padding byte no field owns is whatever the sender's stack held; and -/// [`recv_payload`] reinterprets bytes a *peer* chose, so a `bool`, a +/// [`decode_payload`] reinterprets bytes a *peer* chose, so a `bool`, a /// fieldless enum or a `NonZeroU32` would be UB the moment it arrives. /// /// Declare payload structs with [`ipc_payload!`], which proves the first two @@ -85,85 +81,36 @@ macro_rules! ipc_payload { /// clipboard set and the compositor's paste both switch to shared memory /// instead. 8192 is one doubling of headroom, so a protocol can grow its /// largest inline message without the SDK becoming the thing that refuses it. -/// -/// What the bound buys is a refusal in O(1). A peer's `len` sizes a -/// [`recv_bytes`] tail-skip, so `len = u32::MAX` obliges the reader to consume -/// every byte that peer will ever send before it returns to its event loop. -/// Bounded, that frame is an error at the header and the peer can be dropped. +/// It sizes the buffer every send frames in, and the receive buffer of every +/// protocol that takes frames up to it. pub const MAX_FRAME_LEN: u32 = 8192; -/// A frame header. `len` and `handles` are private because they carry -/// invariants a peer does not: every `IpcHeader` in existence has a length -/// within [`MAX_FRAME_LEN`] and a count within -/// [`MAX_TRANSFER_HANDLES`](syscall::MAX_TRANSFER_HANDLES), because -/// [`IpcHeader::from_wire`] is the only way to make one out of bytes somebody -/// else chose. -#[derive(Clone, Copy)] -pub struct IpcHeader { - pub msg_type: u32, - len: u32, - handles: u32, -} - -impl IpcHeader { - pub const WIRE_SIZE: usize = 12; - - /// Build a header from words that have not been trusted yet. - pub fn from_wire(msg_type: u32, len: u32, handles: u32) -> Result { - if len > MAX_FRAME_LEN || handles as usize > syscall::MAX_TRANSFER_HANDLES { - return Err(IpcError::Malformed); - } - Ok(Self { msg_type, len, handles }) - } - - fn decode(bytes: &[u8; Self::WIRE_SIZE]) -> Result { - let word = |at: usize| u32::from_ne_bytes(bytes[at..at + 4].try_into().expect("four bytes")); - Self::from_wire(word(0), word(4), word(8)) - } - - pub fn len(&self) -> u32 { - self.len - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } +/// The message type's bytes in front of every payload. +const TYPE_LEN: usize = 4; - /// How many handles travel with this frame. - pub fn handles(&self) -> usize { - self.handles as usize - } +const _: () = assert!(TYPE_LEN + MAX_FRAME_LEN as usize <= syscall::MAX_MESSAGE_LEN); - fn to_wire(self) -> [u8; Self::WIRE_SIZE] { - let mut bytes = [0u8; Self::WIRE_SIZE]; - bytes[..4].copy_from_slice(&self.msg_type.to_ne_bytes()); - bytes[4..8].copy_from_slice(&self.len.to_ne_bytes()); - bytes[8..].copy_from_slice(&self.handles.to_ne_bytes()); - bytes - } +/// A frame as the kernel carries it: the message type, then its payload. +/// Bytes only, so no padding. +#[repr(C)] +struct Wire

{ + msg_type: [u8; TYPE_LEN], + payload: P, +} - /// The same header, built from a length and count this endpoint chose itself. - fn frame(msg_type: u32, len: usize, handles: usize) -> Result { - if len > MAX_FRAME_LEN as usize || handles > syscall::MAX_TRANSFER_HANDLES { - return Err(IpcError::TooLarge); - } - Ok(Self { msg_type, len: len as u32, handles: handles as u32 }) +impl Wire<[u8; MAX]> { + fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: a `Wire` of initialised bytes, whole. + unsafe { core::slice::from_raw_parts_mut(self as *mut Self as *mut u8, core::mem::size_of::()) } } } -const _: () = assert!(IpcHeader::WIRE_SIZE + MAX_FRAME_LEN as usize <= syscall::MAX_MESSAGE_LEN); - -/// A payload of at most this many bytes, such as a signal or a small typed -/// payload sent per input event, is framed in a buffer this size rather than -/// one of [`MAX_FRAME_LEN`], which would be zeroed on every send. -const SMALL_PAYLOAD: usize = 64; - #[derive(Debug)] pub enum IpcError { Disconnected, - /// The peer's frame does not describe a message this endpoint can read: - /// a length past [`MAX_FRAME_LEN`], or one other than the payload the - /// caller asked for. + /// The peer's frame is not one this endpoint can read: larger than the + /// receiver holds, too short for a message type, or another payload than + /// the caller asked for. Malformed, /// This endpoint will not frame a payload that large. Distinct from /// [`IpcError::Malformed`] because the two blame different sides. @@ -174,9 +121,9 @@ pub enum IpcError { /// Why a whole frame could not be handed over without blocking. #[derive(Debug)] pub enum TrySendError { - /// The peer's stream has no room for the whole frame now. **Nothing was - /// sent**: a frame goes in whole or not at all. A peer that cannot take - /// one frame has a stream of unread messages behind it, and is dropped. + /// The peer has no room for the whole frame now. **Nothing was sent**: a + /// frame goes in whole or not at all. A peer that cannot take one frame + /// has a queue of unread messages behind it, and is dropped. Full, /// This endpoint will not frame a payload that large. TooLarge, @@ -259,7 +206,7 @@ impl Connection { message(self.as_handle(), msg_type, data, handles, false).map_err(blocking) } - /// [`send_with`](Self::send_with) for a bare header. + /// [`send_with`](Self::send_with) for a bare message type. pub fn signal_with(&self, msg_type: u32, handles: impl IntoIterator) -> Result<(), IpcError> { message(self.as_handle(), msg_type, &[], handles, false).map_err(blocking) } @@ -285,7 +232,7 @@ impl Connection { message(self.as_handle(), msg_type, data, handles, true).map_err(nonblocking) } - /// [`try_send_with`](Self::try_send_with) for a bare header. + /// [`try_send_with`](Self::try_send_with) for a bare message type. pub fn try_signal_with( &self, msg_type: u32, @@ -294,47 +241,6 @@ impl Connection { message(self.as_handle(), msg_type, &[], handles, true).map_err(nonblocking) } - pub fn recv_header(&self) -> Result { - recv_header(self.as_handle()) - } - - pub fn recv_payload(&self, header: &IpcHeader) -> Result { - recv_payload(self.as_handle(), header) - } - - pub fn recv(&self) -> Result<(u32, T), IpcError> { - recv(self.as_handle()) - } - - /// The handles `header`'s frame announced, read after its payload: exactly - /// that many, or [`IpcError::Malformed`] with whatever arrived closed. The - /// kernel pins a frame's batch just past its bytes, so they are there. - pub fn recv_handles(&self, header: &IpcHeader) -> Result { - recv_handles(self.as_handle(), header) - } - - /// The `N` handles a frame of this message type carries. - /// - /// `None` for any other count, with whatever did arrive closed rather than - /// leaked: a frame announcing another number speaks a protocol this - /// endpoint does not. - pub fn recv_handles_exact(&self, header: &IpcHeader) -> Option<[OwnedHandle; N]> { - self.recv_handles(header).ok()?.exact() - } - - pub fn recv_bytes(&self, header: &IpcHeader, buf: &mut [u8]) -> Result { - recv_bytes(self.as_handle(), header, buf) - } - - pub fn read_nonblock(&self, buf: &mut [u8]) -> Result { - self.0.read_nonblock(buf) - } - - /// Raw bytes, for a connection a protocol has made a doorbell: never a - /// frame, which [`send`](Self::send) and its kin write whole. - pub fn write_nonblock(&self, buf: &[u8]) -> Result { - self.0.write_nonblock(buf) - } } // Free functions, for a connection a caller holds by something other than a @@ -360,9 +266,9 @@ pub(crate) fn signal_with(conn: BorrowedHandle<'_>, msg_type: u32, handles: impl /// Send a framed payload without ever blocking, or refuse the whole frame. /// /// A server writing to a client it does not trust to read cannot use [`send`]: -/// a blocking send parks in the kernel until the peer drains, which is a -/// client deciding when the server runs again. This is the same frame, and the -/// peer's backlog is an answer rather than a wait. +/// a blocking send parks in the kernel until the peer takes its messages, +/// which is a client deciding when the server runs again. This is the same +/// frame, and the peer's backlog is an answer rather than a wait. pub fn try_send(conn: BorrowedHandle<'_>, msg_type: u32, payload: &T) -> Result<(), TrySendError> { message(conn, msg_type, as_bytes(payload), none(), true).map_err(nonblocking) } @@ -402,7 +308,28 @@ fn nonblocking(e: Unsent) -> TrySendError { } } -/// The one way a frame leaves this endpoint: its header and `payload` in one +/// `msg_type` and `payload` laid out in `frame`, answering the bytes the +/// kernel sends, or `None` for a payload past [`MAX_FRAME_LEN`]. Only those +/// bytes are written: nothing else of the buffer is read. +fn framed<'a>( + frame: &'a mut core::mem::MaybeUninit>, + msg_type: u32, + payload: &[u8], +) -> Option<&'a [u8]> { + if payload.len() > MAX_FRAME_LEN as usize { + return None; + } + let base = frame.as_mut_ptr() as *mut u8; + // SAFETY: the buffer holds `TYPE_LEN + MAX_FRAME_LEN` bytes, the payload + // fits behind the type, and the slice covers only the bytes written here. + unsafe { + core::ptr::copy_nonoverlapping(msg_type.to_ne_bytes().as_ptr(), base, TYPE_LEN); + core::ptr::copy_nonoverlapping(payload.as_ptr(), base.add(TYPE_LEN), payload.len()); + Some(core::slice::from_raw_parts(base, TYPE_LEN + payload.len())) + } +} + +/// The one way a frame leaves this endpoint: its type and `payload` in one /// buffer, and `handles`, handed to the kernel as one message. Every handle is /// consumed whatever the answer. fn message( @@ -412,30 +339,18 @@ fn message( handles: impl IntoIterator, nonblock: bool, ) -> Result<(), Unsent> { - let mut too_large = false; - let sent = move_batch(handles, |batch| { - let Ok(header) = IpcHeader::frame(msg_type, payload.len(), batch.len()) else { - too_large = true; - return Err(SyscallError::InvalidArgument); - }; - let send = |frame: &mut [u8]| { - let end = IpcHeader::WIRE_SIZE + payload.len(); - frame[..IpcHeader::WIRE_SIZE].copy_from_slice(&header.to_wire()); - frame[IpcHeader::WIRE_SIZE..end].copy_from_slice(payload); - // SAFETY: `move_batch` hands over only handles it was given to - // consume, and forgets them once they are sent. - unsafe { syscall::message_send(conn.raw(), &frame[..end], batch, nonblock) } - }; - match payload.len() <= SMALL_PAYLOAD { - true => send(&mut [0u8; IpcHeader::WIRE_SIZE + SMALL_PAYLOAD]), - false => send(&mut [0u8; IpcHeader::WIRE_SIZE + MAX_FRAME_LEN as usize]), - } - }); - match sent { - Ok(()) => Ok(()), - Err(_) if too_large => Err(Unsent::TooLarge), - Err(e) => Err(Unsent::Kernel(e)), - } + let mut frame = core::mem::MaybeUninit::uninit(); + let Some(bytes) = framed(&mut frame, msg_type, payload) else { + // Consumed all the same. + handles.into_iter().for_each(drop); + return Err(Unsent::TooLarge); + }; + move_batch(handles, |batch| { + // SAFETY: `move_batch` hands over only handles it was given to + // consume, and forgets them once they are sent. + unsafe { syscall::message_send(conn.raw(), bytes, batch, nonblock) } + }) + .map_err(Unsent::Kernel) } /// Hand `handles` to `send` as one batch, consuming every one: what `send` @@ -460,156 +375,96 @@ fn move_batch( Ok(()) } -pub fn recv_header(conn: BorrowedHandle<'_>) -> Result { - let mut bytes = [0u8; IpcHeader::WIRE_SIZE]; - read_exact(conn.raw(), &mut bytes)?; - IpcHeader::decode(&bytes) -} - -/// The frame's payload as one `T`. A payload of any other length is read -/// past, so the connection stays framed, and refused [`IpcError::Malformed`]. -pub fn recv_payload(conn: BorrowedHandle<'_>, header: &IpcHeader) -> Result { - if let Err(e) = exactly::(header.len as usize) { - skip(conn.raw(), header.len as usize)?; - return Err(e); - } - let mut val = core::mem::MaybeUninit::::uninit(); - // SAFETY: the slice covers exactly the `T` being filled, and `IpcPayload` - // promises every bit pattern `read_exact` can write is a valid `T`. - read_exact(conn.raw(), unsafe { - core::slice::from_raw_parts_mut(val.as_mut_ptr() as *mut u8, core::mem::size_of::()) - })?; - Ok(unsafe { val.assume_init() }) -} - -/// Receive header + typed payload in one call, for a frame that carries no -/// handles: one that does is [`IpcError::Malformed`], its batch closed. -pub fn recv(conn: BorrowedHandle<'_>) -> Result<(u32, T), IpcError> { - let header = recv_header(conn)?; - let payload = recv_payload(conn, &header)?; - if header.handles() != 0 { - // Dropped, so closed. - let _ = recv_handles(conn, &header); - return Err(IpcError::Malformed); - } - Ok((header.msg_type, payload)) -} - -/// Receive raw bytes. Returns the number of valid bytes read. -pub fn recv_bytes(conn: BorrowedHandle<'_>, header: &IpcHeader, buf: &mut [u8]) -> Result { - let count = (header.len as usize).min(buf.len()); - if count > 0 { - read_exact(conn.raw(), &mut buf[..count])?; - } - skip(conn.raw(), header.len as usize - count)?; - Ok(count) -} - -/// The batch `header` announced, taken where the kernel pinned it: exactly -/// [`IpcHeader::handles`] of them, or [`IpcError::Malformed`] with any that -/// arrived closed. -pub fn recv_handles(conn: BorrowedHandle<'_>, header: &IpcHeader) -> Result { - let want = header.handles(); - if want == 0 { - return Ok(Batch::EMPTY); - } - let batch = Batch::take(conn).map_err(IpcError::Syscall)?; - if batch.len() != want { - return Err(IpcError::Malformed); - } - Ok(batch) -} - -/// The memory twin of [`recv_payload`], for a reader that already holds the -/// bytes. -/// -/// Buffering a whole frame before acting on it is what lets a server stay -/// non-blocking against a peer that stops mid-message, and a buffered payload -/// has no handle left to read it from. +/// One payload as a `T`, for a reader holding a frame's bytes +/// ([`FrameRx::payload`]): exactly one `T`, or [`IpcError::Malformed`]. pub fn decode_payload(bytes: &[u8]) -> Result { - exactly::(bytes.len())?; - let mut val = core::mem::MaybeUninit::::uninit(); - // SAFETY: the copy fills exactly the `T` being built from a source that - // long, and `IpcPayload` promises every bit pattern is a valid `T` — the - // same contract `recv_payload` reads on. - unsafe { - core::ptr::copy_nonoverlapping(bytes.as_ptr(), val.as_mut_ptr() as *mut u8, bytes.len()); - Ok(val.assume_init()) + if bytes.len() != core::mem::size_of::() { + return Err(IpcError::Malformed); } -} - -/// A typed payload is exactly one `T`: a shorter one is not a `T`, and the -/// bytes past a longer one are no protocol's. -fn exactly(len: usize) -> Result<(), IpcError> { - if len == core::mem::size_of::() { Ok(()) } else { Err(IpcError::Malformed) } + // SAFETY: the source is exactly one `T` long, and `IpcPayload` promises + // every bit pattern is a valid `T`. + Ok(unsafe { core::ptr::read_unaligned(bytes.as_ptr() as *const T) }) } /// What one [`FrameRx::pump`] found. #[derive(Clone, Copy, Debug)] pub enum RxStep { - /// The handle has nothing more right now. + /// The connection has nothing more right now. Idle, - /// A whole frame is buffered; its payload is [`FrameRx::payload`] and its + /// A whole frame was taken; its payload is [`FrameRx::payload`] and its /// `handles` are [`FrameRx::take_handles`]'s. Frame { msg_type: u32, payload_len: usize, handles: usize }, /// The peer hung up, or its handle is gone. Eof, - /// The peer broke framing: a length past [`MAX_FRAME_LEN`] or the - /// receiver's `MAX`, a count past the bound, a batch other than its frame - /// announced, or one no frame announced. Nothing after it can be located, - /// so there is nothing to resynchronise to. + /// A frame this receiver cannot take: too short for a message type, or + /// larger than its `MAX` or carrying handles this process's table has no + /// room for, either of which stays queued in front of every later frame. + /// The peer is dropped. Malformed, } -/// One peer's inbound framing, over non-blocking reads only. +/// A frame [`FrameRx::recv`] took. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Frame { + pub msg_type: u32, + pub payload_len: usize, + pub handles: usize, +} + +/// One peer's inbound frames: a buffer for the widest payload the caller's +/// protocol carries, `MAX`, and the handles the last frame brought. /// -/// **A server never reads a client with a blocking read.** That is the whole -/// point of this type: [`recv_header`] and [`recv_payload`] park the caller -/// until the peer sends the bytes it promised, which hands a client the -/// decision of when the server runs again. Here a peer that stops halfway -/// through a frame costs a buffer instead of the event loop. +/// **A server never reads a client with a blocking receive** — that hands a +/// client the decision of when the server runs again — so it [`pump`]s, +/// which never blocks. A client waiting on its server's answer [`recv`]s. +/// A frame wider than `MAX` is refused whole and never cut short. A protocol +/// whose messages are all bare types takes `MAX = 0`. /// -/// `MAX` is the widest payload the caller's protocol carries. A frame -/// declaring more is [`RxStep::Malformed`] at its header, so a peer cannot -/// make the server hold what it will not look at, and no frame arrives cut -/// short. A protocol whose messages are all bare headers takes `MAX = 0`. +/// **A frame's handles come with it.** What the caller does not take with +/// [`take_handles`](Self::take_handles) is closed before the next frame is +/// taken or when this is dropped. /// -/// **A frame's handles come with it.** They are taken when the frame is, and -/// what the caller does not take with [`take_handles`](Self::take_handles) is -/// closed before the next frame is read or when this is dropped. +/// [`pump`]: Self::pump +/// [`recv`]: Self::recv pub struct FrameRx { - header: [u8; IpcHeader::WIRE_SIZE], - buf: [u8; MAX], - len: usize, - state: RxState, + wire: Wire<[u8; MAX]>, held: Batch, } -enum RxState { - Header, - Payload { msg_type: u32, len: usize, handles: usize }, -} - impl Default for FrameRx { fn default() -> Self { Self::new() } } +/// Why a receive took no frame. +enum Refused { + Malformed, + Kernel(SyscallError), +} + +impl Refused { + fn blocking(self) -> IpcError { + match self { + // A frame larger than the receiver holds, or with more handles, + // is the peer's: it stays queued, as `step` says. + Refused::Malformed | Refused::Kernel(SyscallError::InvalidArgument) => IpcError::Malformed, + Refused::Kernel(SyscallError::Gone) => IpcError::Disconnected, + // The rest are this side's, a table without room for the frame's + // handles (`ResourceExhausted`) among them. + Refused::Kernel(e) => IpcError::Syscall(e), + } + } +} + impl FrameRx { pub const fn new() -> Self { - Self { - header: [0; IpcHeader::WIRE_SIZE], - buf: [0; MAX], - len: 0, - state: RxState::Header, - held: Batch::EMPTY, - } + Self { wire: Wire { msg_type: [0; TYPE_LEN], payload: [0; MAX] }, held: Batch::EMPTY } } /// The payload of the frame the last [`RxStep::Frame`] announced. pub fn payload(&self, len: usize) -> &[u8] { - &self.buf[..len] + &self.wire.payload[..len] } /// The handles the last frame carried, now the caller's to claim. @@ -623,56 +478,54 @@ impl FrameRx { self.take_batch().exact::() } - /// Read what is there, and report the first thing that happened. - /// - /// At most one frame per call: the caller has other peers to serve, and a - /// peer that always has another frame ready must not be able to keep the - /// rest from being served. - pub fn pump(&mut self, conn: &Connection) -> RxStep { + /// Take the next frame if one is queued, never waiting: at most one per + /// call, so a peer that always has another ready cannot keep a server's + /// other peers from being served. + pub fn pump(&mut self, conn: &impl AsHandle) -> RxStep { + step(self.take(conn.as_handle(), true)) + } + + /// Wait for the next frame: [`IpcError::Disconnected`] once the peer has + /// gone with nothing left, and [`IpcError::Malformed`] for one this + /// receiver cannot take, which stays queued. + pub fn recv(&mut self, conn: &impl AsHandle) -> Result { + self.take(conn.as_handle(), false).map_err(Refused::blocking) + } + + fn take(&mut self, conn: BorrowedHandle<'_>, nonblock: bool) -> Result { self.held = Batch::EMPTY; - loop { - match self.state { - RxState::Header => { - match fill(conn, &mut self.header, &mut self.len) { - Fill::Idle => return RxStep::Idle, - Fill::Eof => return RxStep::Eof, - Fill::Malformed => return RxStep::Malformed, - Fill::Filled => {} - } - let Some((msg_type, len, handles)) = Self::announced(&self.header) else { - return RxStep::Malformed; - }; - self.len = 0; - self.state = RxState::Payload { msg_type, len, handles }; - } - RxState::Payload { msg_type, len, handles } => { - match fill(conn, &mut self.buf[..len], &mut self.len) { - Fill::Idle => return RxStep::Idle, - Fill::Eof => return RxStep::Eof, - Fill::Malformed => return RxStep::Malformed, - Fill::Filled => {} - } - self.len = 0; - self.state = RxState::Header; - if handles > 0 { - // Pinned just past the bytes now read, so it is there or never sent. - self.held = Batch::take(conn.as_handle()).unwrap_or(Batch::EMPTY); - if self.held.len() != handles { - return RxStep::Malformed; - } - } - return RxStep::Frame { msg_type, payload_len: len, handles }; - } - } - } + let mut handles = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; + let got = syscall::message_recv(conn.raw(), self.wire.bytes_mut(), &mut handles, nonblock); + self.taken(got, handles) } - /// The message type, payload length and handle count `header` announces, - /// or `None` for a frame this receiver cannot hold whole. - fn announced(header: &[u8; IpcHeader::WIRE_SIZE]) -> Option<(u32, usize, usize)> { - let header = IpcHeader::decode(header).ok()?; - let len = header.len() as usize; - (len <= MAX).then_some((header.msg_type, len, header.handles())) + /// A receive's answer, the handles it installed now this receiver's. + fn taken( + &mut self, + got: Result, + handles: [RawHandle; syscall::MAX_TRANSFER_HANDLES], + ) -> Result { + let got = got.map_err(Refused::Kernel)?; + // The kernel installed each number in this process's table for this + // call, so this batch is the only thing that answers for them. + self.held = Batch { handles, next: 0, len: got.handles }; + let payload_len = got.len.checked_sub(TYPE_LEN).ok_or(Refused::Malformed)?; + let msg_type = u32::from_ne_bytes(self.wire.msg_type); + Ok(Frame { msg_type, payload_len, handles: got.handles }) + } +} + +/// What a non-blocking receive's answer is to a server. +fn step(taken: Result) -> RxStep { + match taken { + Ok(Frame { msg_type, payload_len, handles }) => RxStep::Frame { msg_type, payload_len, handles }, + Err(Refused::Kernel(SyscallError::WouldBlock)) => RxStep::Idle, + Err(Refused::Malformed | Refused::Kernel(SyscallError::InvalidArgument | SyscallError::ResourceExhausted)) => { + RxStep::Malformed + } + // The handle itself is gone, which is the same hang-up seen from + // the other side of the same race. + Err(Refused::Kernel(_)) => RxStep::Eof, } } @@ -687,15 +540,6 @@ pub struct Batch { impl Batch { const EMPTY: Batch = Batch { handles: [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES], next: 0, len: 0 }; - /// The batch the kernel pinned at `conn`'s place in its stream. - fn take(conn: BorrowedHandle<'_>) -> Result { - let mut handles = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; - // The kernel installed each number in this process's table for this - // call, so this batch is the only thing that answers for them. - let len = syscall::handle_recv(conn.raw(), &mut handles)?; - Ok(Self { handles, next: 0, len }) - } - /// The handles not yet taken. pub fn len(&self) -> usize { self.len - self.next @@ -731,63 +575,11 @@ impl Drop for Batch { } } -enum Fill { - Filled, - Idle, - Eof, - /// A batch no frame announced sits where the next byte would be. - Malformed, -} - -/// Fill `buf` from `*len` onwards without blocking, carrying `*len` forward so -/// the next call resumes where this one stopped. -fn fill(conn: &Connection, buf: &mut [u8], len: &mut usize) -> Fill { - while *len < buf.len() { - match conn.read_nonblock(&mut buf[*len..]) { - Ok(0) => return Fill::Eof, - Ok(n) => *len += n, - Err(SyscallError::WouldBlock) => return Fill::Idle, - Err(SyscallError::HandlesPending) => return Fill::Malformed, - // The handle itself is gone, which is the same hang-up seen from the - // other side of the same race. - Err(_) => return Fill::Eof, - } - } - Fill::Filled -} - /// A payload's bytes, as a frame carries them. pub(crate) fn as_bytes(val: &T) -> &[u8] { unsafe { core::slice::from_raw_parts(val as *const T as *const u8, core::mem::size_of::()) } } -fn skip(handle: RawHandle, mut remaining: usize) -> Result<(), IpcError> { - let mut buf = [0u8; 128]; - while remaining > 0 { - let chunk = remaining.min(buf.len()); - read_exact(handle, &mut buf[..chunk])?; - remaining -= chunk; - } - Ok(()) -} - -/// Fill `buf` with blocking reads. A batch no frame announced where the next -/// byte would be is the peer breaking framing. -fn read_exact(handle: RawHandle, buf: &mut [u8]) -> Result<(), IpcError> { - let mut offset = 0; - while offset < buf.len() { - let n = syscall::read(handle, &mut buf[offset..]).map_err(|e| match e { - SyscallError::HandlesPending => IpcError::Malformed, - e => IpcError::Syscall(e), - })?; - if n == 0 { - return Err(IpcError::Disconnected); - } - offset += n; - } - Ok(()) -} - #[cfg(test)] mod tests { use std::cell::RefCell; @@ -828,36 +620,88 @@ mod tests { assert!(matches!(decode_payload::<[u32; 2]>(&[]), Err(IpcError::Malformed))); } - fn header(len: u32, handles: u32) -> [u8; IpcHeader::WIRE_SIZE] { - let mut bytes = [0u8; IpcHeader::WIRE_SIZE]; - bytes[..4].copy_from_slice(&7u32.to_ne_bytes()); - bytes[4..8].copy_from_slice(&len.to_ne_bytes()); - bytes[8..].copy_from_slice(&handles.to_ne_bytes()); - bytes + /// A frame is its type's four bytes and then its payload, and nothing + /// else of the buffer goes to the kernel. + #[test] + fn a_frame_is_its_type_then_exactly_its_payload() { + let mut frame = core::mem::MaybeUninit::uninit(); + let bytes = framed(&mut frame, 0x0102_0304, &[9, 8, 7]).unwrap(); + let mut want = 0x0102_0304u32.to_ne_bytes().to_vec(); + want.extend([9, 8, 7]); + assert_eq!(bytes, &want[..]); + let mut frame = core::mem::MaybeUninit::uninit(); + assert_eq!(framed(&mut frame, 5, &[]).unwrap(), &5u32.to_ne_bytes()); + } + + /// The largest payload is framed, and one byte more is refused before + /// the kernel is asked. + #[test] + fn a_payload_past_the_frame_bound_is_refused() { + let most = vec![0u8; MAX_FRAME_LEN as usize]; + let mut frame = core::mem::MaybeUninit::uninit(); + assert_eq!(framed(&mut frame, 1, &most).map(<[u8]>::len), Some(TYPE_LEN + most.len())); + let over = vec![0u8; MAX_FRAME_LEN as usize + 1]; + let mut frame = core::mem::MaybeUninit::uninit(); + assert!(framed(&mut frame, 1, &over).is_none()); } - /// A frame is held whole up to the receiver's `MAX` and refused at its - /// header past it, never cut down to what fits. + /// A refused frame consumes its handles all the same. #[test] - fn a_frame_past_the_receivers_max_is_refused_at_its_header() { - assert_eq!(FrameRx::<8>::announced(&header(0, 0)), Some((7, 0, 0))); - assert_eq!(FrameRx::<8>::announced(&header(8, 0)), Some((7, 8, 0))); - assert_eq!(FrameRx::<8>::announced(&header(9, 0)), None); - assert_eq!(FrameRx::<0>::announced(&header(1, 0)), None); - assert_eq!(FrameRx::<{ MAX_FRAME_LEN as usize + 1 }>::announced(&header(MAX_FRAME_LEN + 1, 0)), None); + fn a_payload_too_large_closes_its_handles() { + let closed = Rc::default(); + let over = vec![0u8; MAX_FRAME_LEN as usize + 1]; + // SAFETY: a number no syscall is made on: the frame is refused first. + let conn = unsafe { BorrowedHandle::borrow_raw(RawHandle(77)) }; + assert!(matches!(message(conn, 1, &over, probes(2, &closed), false), Err(Unsent::TooLarge))); + assert_eq!(*closed.borrow(), [1, 2]); + } + + /// A message is its type and then its payload: one too short for a type + /// is no frame, and the type is read from the front. + #[test] + fn a_message_shorter_than_a_type_is_malformed() { + let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; + let mut rx = FrameRx::<8>::new(); + rx.wire.msg_type = 7u32.to_ne_bytes(); + assert!(matches!(rx.taken(Ok(Received { len: TYPE_LEN - 1, handles: 0 }), none), Err(Refused::Malformed))); + let frame = rx.taken(Ok(Received { len: TYPE_LEN, handles: 0 }), none).ok().unwrap(); + assert_eq!(frame, Frame { msg_type: 7, payload_len: 0, handles: 0 }); + let frame = rx.taken(Ok(Received { len: TYPE_LEN + 8, handles: 0 }), none).ok().unwrap(); + assert_eq!(frame.payload_len, 8); + } + + /// The kernel's refusals reach a server as the step it acts on. + #[test] + fn a_receive_refused_is_the_step_a_server_acts_on() { + let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; + let mut rx = FrameRx::<0>::new(); + for (e, idle, eof) in [ + (SyscallError::WouldBlock, true, false), + (SyscallError::Gone, false, true), + (SyscallError::InvalidArgument, false, false), + (SyscallError::ResourceExhausted, false, false), + ] { + let step = step(rx.taken(Err(e), none)); + assert_eq!(matches!(step, RxStep::Idle), idle, "{e:?} answered {step:?}"); + assert_eq!(matches!(step, RxStep::Eof), eof, "{e:?} answered {step:?}"); + } } - /// A frame announces at most the batch one message can carry: one more - /// is refused at its header, as the kernel would refuse the send. + /// The same refusals reach a waiting client as the error it acts on: a + /// frame too large for it is the peer's protocol broken, never a + /// kernel fault. #[test] - fn a_frame_announcing_more_handles_than_a_message_carries_is_refused() { - let most = syscall::MAX_TRANSFER_HANDLES as u32; - assert_eq!(FrameRx::<8>::announced(&header(4, most)), Some((7, 4, most as usize))); - assert_eq!(FrameRx::<8>::announced(&header(4, most + 1)), None); - assert!(matches!(IpcHeader::frame(7, 0, most as usize + 1), Err(IpcError::TooLarge))); - let built = IpcHeader::frame(7, 3, 2).unwrap().to_wire(); - let back = IpcHeader::decode(&built).unwrap(); - assert_eq!((back.msg_type, back.len(), back.handles()), (7, 3, 2)); + fn a_receive_refused_is_the_error_a_client_acts_on() { + let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; + let mut rx = FrameRx::<0>::new(); + let mut blocking = |got| rx.taken(got, none).map_err(Refused::blocking); + assert!(matches!(blocking(Err(SyscallError::InvalidArgument)), Err(IpcError::Malformed))); + assert!(matches!(blocking(Ok(Received { len: TYPE_LEN - 1, handles: 0 })), Err(IpcError::Malformed))); + assert!(matches!(blocking(Err(SyscallError::Gone)), Err(IpcError::Disconnected))); + assert!(matches!( + blocking(Err(SyscallError::ResourceExhausted)), + Err(IpcError::Syscall(SyscallError::ResourceExhausted)) + )); } /// The kernel's refusal leaves every handle at its own number, and the diff --git a/sdk/toyos/src/launch.rs b/sdk/toyos/src/launch.rs index 8aace1298a4..6a742bf6def 100644 --- a/sdk/toyos/src/launch.rs +++ b/sdk/toyos/src/launch.rs @@ -380,9 +380,10 @@ pub fn launch(conn: &Connection, request: Launch<'_>) -> Result match conn.recv_handles_exact::<1>(&header) { + let mut rx = crate::ipc::FrameRx::<0>::new(); + let frame = rx.recv(&conn).map_err(LaunchError::Sent)?; + match frame.msg_type { + MSG_LAUNCHED => match rx.take_handles::<1>() { Some([process]) => Ok(Outcome::Started(process)), None => Err(LaunchError::Sent(IpcError::Malformed)), }, diff --git a/sdk/toyos/src/net.rs b/sdk/toyos/src/net.rs index 4fa74c521b7..0c8f7417739 100644 --- a/sdk/toyos/src/net.rs +++ b/sdk/toyos/src/net.rs @@ -48,7 +48,7 @@ impl Drop for Exporter { } } -use crate::ipc::{IpcError, IpcHeader, IpcPayload}; +use crate::ipc::{Frame, FrameRx, IpcError, IpcPayload}; use crate::ipc_payload; use crate::{Connection, OwnedHandle}; @@ -316,7 +316,7 @@ const MAX_ADOPT: usize = { /// process takes up no holder of it. pub fn adopt_exported(end: OwnedHandle) -> Option { let connection = Connection::received(end); - let mut frames: crate::ipc::FrameRx = crate::ipc::FrameRx::new(); + let mut frames: FrameRx = FrameRx::new(); let first = match frames.pump(&connection) { crate::ipc::RxStep::Frame { msg_type, payload_len, .. } => { let handles = frames.take_batch(); @@ -335,31 +335,14 @@ pub fn adopt_exported(end: OwnedHandle) -> Option { /// A netstack that hung up mid-exchange is a netstack that is not there. /// -/// **[`NetstackConn::connect`] already says so and the exchange did not, which is -/// a distinction this architecture removed.** A connector is in the namespace -/// from a program's first instruction, so connecting to a netstack that has -/// already exited *succeeds* — the connection queues on a port nobody will -/// ever accept from — and the hang-up arrives at the first send or the first -/// read instead. Reporting that as [`NetError::Io`] left every caller unable -/// to tell "this machine has no network" from "netstack failed", and `sshserver` -/// panicked across the boot of every NIC-less machine that lost the race -/// rather than exiting with the line it has for exactly this. -/// -/// **`Disconnected` is only the word for a hang-up this endpoint *read*, and a -/// request is sent before anything is read.** `IpcError::Disconnected` is -/// raised in one place — `ipc::read_exact`, on a `read` that answered zero — so -/// it is what a peer that left while this endpoint was waiting for the response -/// looks like. A peer that left *before* the request went out is refused by the -/// kernel at the send instead, which answers `Gone` on a connection whose -/// handle is still live and still this process's: when the server end's last -/// handle goes, `port::Acceptor::on_zero_handles` closes every queued -/// connection's inbox — this end's *outbox*, so `SYS_MESSAGE_SEND` refuses a -/// message rather than filling a queue nobody will drain — and drops the -/// connection, which drops the server's read end, so the stream has no reader. -/// -/// `Gone` cannot mean anything else here. This is applied only to `read` and -/// `message_send` on a connection this process holds, and a handle a process -/// does not hold ends it at the kernel rather than answering a word. +/// A connector is in the namespace from a program's first instruction, so +/// connecting to a netstack that has already exited *succeeds*: the +/// connection queues on a port nobody will accept from, and the hang-up +/// arrives at the first send or receive instead. The kernel answers a send +/// `Gone` once no receiver is left, and a receive `Gone` (`Disconnected`) +/// once the peer has gone with nothing queued. Either is +/// [`NetError::NetstackNotFound`], so a caller can tell "this machine has no +/// network" from "netstack failed". pub(crate) fn hangup(e: IpcError) -> NetError { match e { IpcError::Disconnected @@ -368,56 +351,63 @@ pub(crate) fn hangup(e: IpcError) -> NetError { } } -/// The header of netstack's answer on `connection`: a [`RespType::Result`]'s, -/// whose payload and handles are the caller's to read, or the error a +/// netstack's answer on `connection`, taken into `rx`: a [`RespType::Result`], +/// whose payload and handles are the caller's, or the error a /// [`RespType::Error`] carries. -pub(crate) fn answer(connection: crate::BorrowedHandle<'_>) -> Result { - let header = crate::ipc::recv_header(connection).map_err(hangup)?; - if header.msg_type == RespType::Error as u32 { - let err: ErrorResponse = crate::ipc::recv_payload(connection, &header).map_err(hangup)?; +pub(crate) fn answer( + connection: &impl crate::AsHandle, + rx: &mut FrameRx, +) -> Result { + let frame = rx.recv(connection).map_err(hangup)?; + if frame.msg_type == RespType::Error as u32 { + let err: ErrorResponse = crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup)?; return Err(NetError::from_error_code(err.code)); } - if header.msg_type != RespType::Result as u32 { - return Err(NetError::Protocol(header.msg_type)); + if frame.msg_type != RespType::Result as u32 { + return Err(NetError::Protocol(frame.msg_type)); } - Ok(header) + Ok(frame) } +/// The widest typed answer netstack sends. +pub(crate) const MAX_ANSWER: usize = 64; + pub struct PendingResponse(NetstackConn); impl PendingResponse { fn conn(&self) -> &Connection { &(self.0).0 } - fn recv_checked_header(&self) -> Result { - answer(crate::AsHandle::as_handle(self.conn())) - } - pub fn response(self) -> Result { - let header = self.recv_checked_header()?; - self.conn().recv_payload(&header).map_err(hangup) + const { assert!(core::mem::size_of::() <= MAX_ANSWER && core::mem::size_of::() <= MAX_ANSWER) }; + let mut rx = FrameRx::::new(); + let frame = answer(self.conn(), &mut rx)?; + crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup) } + /// The answer's bytes, as many as `buf` holds. pub fn response_bytes(self, buf: &mut [u8]) -> Result { - let header = self.recv_checked_header()?; - self.conn().recv_bytes(&header, buf).map_err(hangup) + let mut rx = FrameRx::<{ crate::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = answer(self.conn(), &mut rx)?; + let n = frame.payload_len.min(buf.len()); + buf[..n].copy_from_slice(&rx.payload(frame.payload_len)[..n]); + Ok(n) } /// The answer, the `N` handles it hands over, and the connection, which /// is then the caller's. pub fn answer_with_handles(self) -> Result<(Resp, [OwnedHandle; N], Connection), NetError> { - let header = self.recv_checked_header()?; - let answer = self.conn().recv_payload(&header).map_err(hangup)?; - let handles = self.conn().recv_handles_exact::(&header).ok_or(NetError::Protocol(header.msg_type))?; + const { assert!(core::mem::size_of::() <= MAX_ANSWER && core::mem::size_of::() <= MAX_ANSWER) }; + let mut rx = FrameRx::::new(); + let frame = answer(self.conn(), &mut rx)?; + let answer = crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup)?; + let handles = rx.take_handles::().ok_or(NetError::Protocol(frame.msg_type))?; Ok((answer, handles, (self.0).0)) } + /// An answer whose payload says nothing past its type. pub fn status(self) -> Result<(), NetError> { - let header = self.recv_checked_header()?; - if !header.is_empty() { - let mut skip = [0u8; 128]; - let _ = self.conn().recv_bytes(&header, &mut skip); - } - Ok(()) + let mut rx = FrameRx::::new(); + answer(self.conn(), &mut rx).map(drop) } } diff --git a/sdk/toyos/src/net/socket.rs b/sdk/toyos/src/net/socket.rs index 2fb8ce2c99b..eae1046b436 100644 --- a/sdk/toyos/src/net/socket.rs +++ b/sdk/toyos/src/net/socket.rs @@ -36,13 +36,13 @@ use toyos_abi::handle::Rights; use toyos_abi::syscall; use toyos_abi::RawHandle; -use crate::ipc::{self, Connection}; +use crate::ipc::{self, Connection, FrameRx}; use crate::sync::Lock; use crate::{AsHandle, BorrowedHandle, OwnedHandle}; use super::ends::{close_kept, kept, narrowed, Ends, HELD_BELL, HELD_CONNECTION}; use super::stream::{self, Wait}; -use super::{answer, hangup, BoundResponse, MsgType, NetError, NetstackConn, TcpListenRequest, TcpOptions, UdpBindRequest}; +use super::{answer, hangup, BoundResponse, MAX_ANSWER, MsgType, NetError, NetstackConn, TcpListenRequest, TcpOptions, UdpBindRequest}; pub use super::stream::{EXPORT, EXPORTED, REFUSED}; @@ -437,10 +437,9 @@ pub fn recv_from(handle: BorrowedHandle<'_>, buf: &mut [u8], wait: Wait) -> Resu call_on(handle, Kind::Datagram, |slot, end| { look(&EndBell { slot, bell: end.bell }, wait, |drain| { drain.now().receive(end.connection).map_err(hangup)?; - let header = answer(end.connection)?; - let mut frame = [0u8; MAX_FRAME]; - let n = ipc::recv_bytes(end.connection, &header, &mut frame).map_err(hangup)?; - let (from, data) = datagram_of(&frame[..n]).filter(|_| n == header.len() as usize).ok_or(NetError::Protocol(RECEIVE))?; + let mut rx = FrameRx::::new(); + let frame = answer(&end.connection, &mut rx)?; + let (from, data) = datagram_of(rx.payload(frame.payload_len)).ok_or(NetError::Protocol(RECEIVE))?; let len = data.len().min(buf.len()); buf[..len].copy_from_slice(&data[..len]); Ok(Received { len, addr: from.addr, port: from.port }) @@ -463,9 +462,10 @@ pub fn peer(handle: BorrowedHandle<'_>) -> Result, NetErr call_on(handle, Kind::Datagram, |slot, end| { let _turn = TURNS[slot].lock(); ipc::signal(end.connection, PEER).map_err(hangup)?; - match answer(end.connection) { - Ok(header) => { - let peer: Datagram = ipc::recv_payload(end.connection, &header).map_err(|_| NetError::Protocol(PEER))?; + let mut rx = FrameRx::::new(); + match answer(&end.connection, &mut rx) { + Ok(frame) => { + let peer: Datagram = ipc::decode_payload(rx.payload(frame.payload_len)).map_err(|_| NetError::Protocol(PEER))?; Ok(Some((peer.addr, peer.port))) } Err(NetError::NotConnected) => Ok(None), @@ -503,17 +503,19 @@ pub fn option(handle: BorrowedHandle<'_>, option: u32) -> Result call(handle, |slot, end| { let _turn = TURNS[slot].lock(); ipc::send(end.connection, OPTION, &option).map_err(hangup)?; - let header = answer(end.connection)?; - ipc::recv_payload(end.connection, &header).map_err(|_| NetError::Protocol(OPTION)) + let mut rx = FrameRx::::new(); + let frame = answer(&end.connection, &mut rx)?; + ipc::decode_payload(rx.payload(frame.payload_len)).map_err(|_| NetError::Protocol(OPTION)) }) } /// An answer that carries nothing. fn bare(connection: BorrowedHandle<'_>) -> Result<(), NetError> { - let header = answer(connection)?; - match header.is_empty() && header.handles() == 0 { + let mut rx = FrameRx::::new(); + let frame = answer(&connection, &mut rx)?; + match frame.payload_len == 0 && frame.handles == 0 { true => Ok(()), - false => Err(NetError::Protocol(header.msg_type)), + false => Err(NetError::Protocol(frame.msg_type)), } } diff --git a/sdk/toyos/src/net/stream.rs b/sdk/toyos/src/net/stream.rs index 454b391e415..ca6e068f75f 100644 --- a/sdk/toyos/src/net/stream.rs +++ b/sdk/toyos/src/net/stream.rs @@ -51,7 +51,7 @@ use toyos_abi::RawHandle; use super::ends::{close_kept, kept, narrowed, Ends, HELD_BELL, HELD_CONNECTION, HELD_RUNG}; -use crate::ipc::{self, Connection}; +use crate::ipc::{self, Connection, FrameRx}; use crate::shm::{Bytes, SharedMemory}; use crate::sync::Lock; use crate::transport::session::{self, Bells, Doorbell, Holder, HungUp, Layout, Mode, Polled, Receiving, Refused as Answer, Sending, Took, Waited}; @@ -757,10 +757,10 @@ pub struct Accepted { pub(super) fn accept(listener: BorrowedHandle<'_>, drain: super::socket::Drain<'_>) -> Result { let opened = open(false, ([0; 4], 0), |region| { drain.now().accept(listener, region).map_err(hangup)?; - let header = answer(listener)?; - let accepted: TcpAcceptResponse = ipc::recv_payload(listener, &header).map_err(hangup)?; - let handles = ipc::recv_handles(listener, &header).map_err(hangup)?; - let [connection, receive, send, rung] = handles.exact::<{ BELLS + 1 }>().ok_or(NetError::Protocol(header.msg_type))?; + let mut rx = FrameRx::<{ super::MAX_ANSWER }>::new(); + let frame = answer(&listener, &mut rx)?; + let accepted: TcpAcceptResponse = ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup)?; + let [connection, receive, send, rung] = rx.take_handles::<{ BELLS + 1 }>().ok_or(NetError::Protocol(frame.msg_type))?; Ok((accepted, [receive, send, rung], Connection::received(connection))) })?; let answer = opened.answer; @@ -950,17 +950,15 @@ pub(super) fn ask_export(connection: BorrowedHandle<'_>, rung: Option { - let handles = ipc::recv_handles(connection, &header).map_err(hangup)?; - match handles.exact::<1>() { - Some([end]) if header.is_empty() => Ok(end), - _ => Err(NetError::Protocol(EXPORTED)), - } - } + let mut rx = FrameRx::<4>::new(); + let frame = rx.recv(&connection).map_err(hangup)?; + match frame.msg_type { + EXPORTED => match rx.take_handles::<1>() { + Some([end]) if frame.payload_len == 0 => Ok(end), + _ => Err(NetError::Protocol(EXPORTED)), + }, REFUSED => { - let code: u32 = ipc::recv_payload(connection, &header).map_err(hangup)?; + let code: u32 = ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup)?; Err(NetError::from_error_code(code)) } other => Err(NetError::Protocol(other)), diff --git a/sdk/toyos/src/power.rs b/sdk/toyos/src/power.rs index 071d7458af9..ef1110a7334 100644 --- a/sdk/toyos/src/power.rs +++ b/sdk/toyos/src/power.rs @@ -324,11 +324,9 @@ fn asked(msg_type: u32, payload: &[u8]) -> Result<(), Refused> { /// One whole frame off `conn`, waiting for it. fn frame(conn: &Connection) -> Result<(u32, Vec), ()> { - let header = conn.recv_header().map_err(drop)?; - let mut payload = alloc::vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut payload).map_err(drop)?; - payload.truncate(n); - Ok((header.msg_type, payload)) + let mut rx = crate::ipc::FrameRx::<{ crate::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(conn).map_err(drop)?; + Ok((frame.msg_type, rx.payload(frame.payload_len).to_vec())) } #[cfg(test)] diff --git a/sdk/toyos/src/surface.rs b/sdk/toyos/src/surface.rs index 554b6ffc662..f4ade8f110b 100644 --- a/sdk/toyos/src/surface.rs +++ b/sdk/toyos/src/surface.rs @@ -476,8 +476,8 @@ impl Keys { pub fn grab() -> Result { let conn = crate::endow::service(SERVICE).map_err(|_| GrabError::HostGone)?; conn.signal(MSG_GRAB_KEYS).map_err(|_| GrabError::HostGone)?; - let header = conn.recv_header().map_err(|_| GrabError::HostGone)?; - match header.msg_type { + let frame = FrameRx::<0>::new().recv(&conn).map_err(|_| GrabError::HostGone)?; + match frame.msg_type { MSG_GRAB_GRANTED => Ok(Self { conn, poller: Poller::new(1) }), MSG_GRAB_REFUSED => Err(GrabError::Busy), other => Err(GrabError::Protocol(other)), @@ -496,9 +496,10 @@ impl Keys { if !ready { return None; } - let header = self.conn.recv_header().ok()?; - match header.msg_type { - MSG_KEY => return self.conn.recv_payload::(&header).ok(), + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&self.conn).ok()?; + match frame.msg_type { + MSG_KEY => return ipc::decode_payload::(rx.payload(frame.payload_len)).ok(), MSG_LAYOUT_CHANGED => {} _ => return None, } diff --git a/sdk/window/src/display.rs b/sdk/window/src/display.rs index 52b87ece9f3..4bdb49c983e 100644 --- a/sdk/window/src/display.rs +++ b/sdk/window/src/display.rs @@ -6,7 +6,7 @@ //! answered: the compositor drops the connection by name. use toyos::endow::{self, EndowError}; -use toyos::ipc::{self, IpcError}; +use toyos::ipc::{self, FrameRx, IpcError}; use toyos::Connection; use toyos_abi::syscall::SyscallError; @@ -235,14 +235,16 @@ pub fn set_scale(port: &str, setting: Setting) -> Result { fn ask(port: &str, request: impl FnOnce(&Connection) -> Result<(), IpcError>) -> Result { let conn = endow::service(port)?; request(&conn)?; - let header = conn.recv_header()?; - match header.msg_type { + let mut rx = FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&conn)?; + let payload = rx.payload(frame.payload_len); + match frame.msg_type { MSG_STATE => { - let state: State = conn.recv_payload(&header)?; + let state: State = ipc::decode_payload(payload)?; Shown::from_wire(state).ok_or(Error::Protocol(MSG_STATE)) } MSG_REFUSED => { - let refusal: Refusal = conn.recv_payload(&header)?; + let refusal: Refusal = ipc::decode_payload(payload)?; Err(Error::Refused(Refused::from_wire(refusal.reason))) } other => Err(Error::Protocol(other)), diff --git a/sdk/window/src/lib.rs b/sdk/window/src/lib.rs index e8bcf08901d..d7e23c0ea4e 100644 --- a/sdk/window/src/lib.rs +++ b/sdk/window/src/lib.rs @@ -524,11 +524,12 @@ fn copy(bytes: &[u8]) -> Result<(), CreateError> { return Ok(conn.send_bytes(MSG_CLIPBOARD_SET, bytes)?); } conn.send(MSG_COPY_BEGIN, &ClipboardShmMsg { len: bytes.len() as u32 })?; - let header = conn.recv_header()?; - if header.msg_type != MSG_COPY_REGION || !header.is_empty() { - return Err(CreateError::Protocol(header.msg_type)); + let mut rx = ipc::FrameRx::<0>::new(); + let frame = rx.recv(&conn)?; + if frame.msg_type != MSG_COPY_REGION { + return Err(CreateError::Protocol(frame.msg_type)); } - let [region] = conn.recv_handles_exact::<1>(&header).ok_or(CreateError::NoHandle)?; + let [region] = rx.take_handles::<1>().ok_or(CreateError::NoHandle)?; let region = SharedMemory::lent(region, bytes.len()).map_err(CreateError::Lent)?; region.bytes().write(0, bytes); Ok(conn.signal(MSG_COPY_COMMIT)?) @@ -546,6 +547,8 @@ fn buffer_bytes(width: u32, stride: u32, height: u32) -> Option { pub struct Window { conn: Connection, + /// The compositor's events, each taken whole. + rx: Box>, poller: Poller, /// The buffer, whose one writable handle the compositor sent; the /// compositor maps its own read-only. @@ -607,19 +610,20 @@ impl Window { req.title_len = len as u8; conn.send(MSG_CREATE_WINDOW, &req)?; - // Header first, then the payload the message type calls for: the two - // answers carry different structs, and a payload of another length than - // the type it was asked for is a refusal from `recv_payload`, not a window. - let header = conn.recv_header()?; - match header.msg_type { + // The two answers carry different structs, and a payload of another + // length than its type's is a refusal from `decode_payload`, not a window. + let mut rx = ipc::FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&conn)?; + let payload = rx.payload(frame.payload_len); + match frame.msg_type { MSG_WINDOW_CREATED => {} MSG_WINDOW_REFUSED => { - let refused: WindowRefused = conn.recv_payload(&header)?; + let refused: WindowRefused = ipc::decode_payload(payload)?; return Err(CreateError::from_wire(refused.reason)); } other => return Err(CreateError::Protocol(other)), } - let info: WindowInfo = conn.recv_payload(&header)?; + let info: WindowInfo = ipc::decode_payload(payload)?; let scale = Scale::from_wire(info.scale).ok_or(CreateError::Protocol(MSG_WINDOW_CREATED))?; let Some(buf_size) = buffer_bytes(info.width, info.stride, info.height) else { return Err(CreateError::Protocol(MSG_WINDOW_CREATED)); @@ -627,12 +631,13 @@ impl Window { // The buffer travels in the frame. A compositor that announced a // window and sent nothing with it is not serving this client, whatever // else it is doing. - let [buffer] = conn.recv_handles_exact::<1>(&header).ok_or(CreateError::NoHandle)?; + let [buffer] = rx.take_handles::<1>().ok_or(CreateError::NoHandle)?; let shm = SharedMemory::lent(buffer, buf_size).map_err(CreateError::Lent)?; let poller = Poller::new(1); Ok(Self { conn, + rx: Box::new(ipc::FrameRx::new()), poller, shm: Rc::new(shm), width: info.width, @@ -672,10 +677,10 @@ impl Window { } pub fn recv_event(&mut self) -> Event { - let Ok(header) = self.conn.recv_header() else { + let Ok(frame) = self.rx.recv(&self.conn) else { return Event::Close; }; - self.decode_event(&header) + self.decode_event(frame) } /// The next event, or `None` when `timeout_nanos` passes — and `None` @@ -709,9 +714,10 @@ impl Window { /// A message the compositor cannot have meant closes the window rather /// than killing the client: this side is a library inside somebody else's /// program, and it has a way to say "the session is over". - fn decode_event(&mut self, header: &ipc::IpcHeader) -> Event { - match header.msg_type { - MSG_KEY_INPUT => match self.conn.recv_payload::(header) { + fn decode_event(&mut self, frame: ipc::Frame) -> Event { + let payload = self.rx.payload(frame.payload_len); + match frame.msg_type { + MSG_KEY_INPUT => match ipc::decode_payload::(payload) { Ok(key) => Event::KeyInput(key), Err(_) => Event::Close, }, @@ -719,12 +725,12 @@ impl Window { load_layout(&mut self.translator); Event::LayoutChanged } - MSG_MOUSE_INPUT => match self.conn.recv_payload(header) { + MSG_MOUSE_INPUT => match ipc::decode_payload(payload) { Ok(ev) => Event::MouseInput(ev), Err(_) => Event::Close, }, MSG_WINDOW_RESIZED => { - let Ok(info) = self.conn.recv_payload::(header) else { + let Ok(info) = ipc::decode_payload::(payload) else { return Event::Close; }; // A scale outside the protocol is a compositor this client @@ -738,7 +744,7 @@ impl Window { // The compositor shows the old buffer only until this // window's next present, which it reads from the new one, so a // window that cannot map the new one has nothing to present. - let Some([buffer]) = self.conn.recv_handles_exact::<1>(header) else { + let Some([buffer]) = self.rx.take_handles::<1>() else { return Event::Close; }; let Ok(shm) = SharedMemory::lent(buffer, buf_size) else { @@ -753,18 +759,12 @@ impl Window { self.generation = info.generation; Event::Resized } - MSG_CLIPBOARD_PASTE => { - let mut buf = [0u8; MAX_INLINE_PAYLOAD]; - let Ok(n) = self.conn.recv_bytes(header, &mut buf) else { - return Event::Close; - }; - Event::ClipboardPaste(buf[..n].to_vec()) - } + MSG_CLIPBOARD_PASTE => Event::ClipboardPaste(payload.to_vec()), MSG_CLIPBOARD_PASTE_SHM => { - let Ok(info) = self.conn.recv_payload::(header) else { + let Ok(info) = ipc::decode_payload::(payload) else { return Event::Close; }; - let Some([buffer]) = self.conn.recv_handles_exact::<1>(header) else { + let Some([buffer]) = self.rx.take_handles::<1>() else { return Event::ClipboardPaste(Vec::new()); }; let Ok(shm) = ReadOnly::lent(buffer, info.len as usize) else { diff --git a/system/CLAUDE.md b/system/CLAUDE.md index 1f43e64ad23..6856d5d11d2 100644 --- a/system/CLAUDE.md +++ b/system/CLAUDE.md @@ -2,7 +2,7 @@ The module header at the site owns its subject — surfaces, translators and the channel in `sdk/toyos/`'s surface modules, soundserver whole under `system/soundserver/`, what a process holds and how it got it at `kernel/src/object/` and `system/supervisor/`. The compositor's decisions are `system/compositor/desktop/`, pure and host-tested; `system/compositor/` is devices, handles, shared memory and the panel. -**A server never blocks on a client** — the doctrine no single site owns. Accept and the first frame are two events; a frame is buffered until whole before anything acts on it; a write is one `try_send` whose refusal drops the peer by name; a blocking read or write of a pipe the client owns is the same bug. The supervisor, the compositor, netstack, soundserver and every surface host use `ipc::FrameRx`. +**A server never blocks on a client** — the doctrine no single site owns. Accept and the first frame are two events; a frame is taken whole with `ipc::FrameRx::pump`, which never waits; a write is one `try_send` whose refusal drops the peer by name; a blocking read or write of a pipe the client owns is the same bug. The supervisor, the compositor, netstack, soundserver and every surface host use `ipc::FrameRx`. ## Caveats that bite every agent diff --git a/system/compositor/src/client.rs b/system/compositor/src/client.rs index 301609e4d87..f744150ee55 100644 --- a/system/compositor/src/client.rs +++ b/system/compositor/src/client.rs @@ -1,12 +1,10 @@ //! Everything between a client's pipe and a decision. //! -//! **The compositor never reads a client with a blocking read**, and never -//! writes one with a blocking write. `ipc::recv_header` and `ipc::recv_payload` -//! park the caller until the peer sends the bytes it promised, which hands a -//! client the decision of when the desktop runs again; a blocking `send` hands -//! it the same decision by not reading. Here a peer that stops halfway through -//! a frame costs a buffer and a deadline, and one that will not take a message -//! costs itself. +//! **The compositor never waits on a client**, to receive or to send. A +//! blocking receive hands a client the decision of when the desktop runs +//! again, and a blocking `send` hands it the same decision by not reading. +//! Frames are taken with `ipc::FrameRx::pump`, and one a client will not take +//! costs that client itself. //! //! **The compositor never receives a handle from a client.** A received handle's //! kind is unknowable, and using one of the wrong kind ends the process that @@ -111,9 +109,8 @@ impl ClientFrame { /// A connection that has been accepted and has not yet said what it is for. /// -/// It exists because `accept` and the first frame are two events, and the -/// compositor used to fuse them with a blocking `recv_header` — so a client -/// that connected and sent four bytes owned the desktop until it disconnected. +/// It exists because `accept` and the first frame are two events: a client +/// that connects and sends nothing must not hold the desktop. pub struct PendingConn { pub conn: Connection, pub rx: ClientRx, @@ -170,14 +167,13 @@ pub enum DropReason { /// `window::MSG_RETIRED_CLIPBOARD_SET_SHM`, whose handle this compositor /// never takes. Retired, - /// Its pipe would not take a whole frame — an entire pipe of messages it - /// has not read. + /// Its connection would not take a whole frame — a full queue of messages + /// it has not taken. NotReading, /// Its other end closed: the one send error that is a client's routine end. Gone, - /// The kernel refused a send to it for any other reason — a handle send - /// to a peer that leaves `MAX_QUEUED_BATCHES` unreceived is - /// `ResourceExhausted`. + /// The kernel refused a send to it for any other reason, such as no memory + /// for the frame's bytes (`ResourceExhausted`). Refused(SyscallError), /// A message to it that this end will not frame. TooLarge, @@ -195,7 +191,7 @@ impl fmt::Display for DropReason { f.write_str("it sent the retired clipboard region, whose handle is never taken") } Self::NotReading => { - f.write_str("its pipe will not take another message and it is not reading") + f.write_str("its connection will not take another message and it is not reading") } Self::Gone => f.write_str("its connection is gone"), Self::Refused(e) => write!(f, "the kernel refused a send to it: {e}"), diff --git a/system/diskserver/src/session.rs b/system/diskserver/src/session.rs index f2e55a2a7fb..99a3e7f8270 100644 --- a/system/diskserver/src/session.rs +++ b/system/diskserver/src/session.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; -use toyos::ipc::Connection; +use toyos::ipc::{Connection, Frame, FrameRx}; use toyos::namespace::Namespace; use toyos::poller::READABLE; use toyos::wake::{Bell, Waker}; @@ -395,13 +395,16 @@ pub(crate) fn connect(names: &Namespace, service: &str) -> Result Result { +pub(crate) fn answer(conn: &Connection, rx: &mut FrameRx) -> Result { if !toyos::poller::wait_one(conn, READABLE, wire::ANSWER_BOUND_NS).map_err(Error::Kernel)? { return Err(Error::Unanswered); } - conn.recv_header().map_err(|_| Error::Ended) + rx.recv(conn).map_err(|e| match e { + toyos::ipc::IpcError::Malformed => Error::Protocol, + _ => Error::Ended, + }) } /// An open session's two doorbells, which the server made and sent with its @@ -423,16 +426,15 @@ fn handshake( ) -> Result<(Bells, Opened), Error> { let conn = connect(names, service)?; conn.send_bytes_with(wire::MSG_OPEN, &guid, [region.share().map_err(Error::Kernel)?]).map_err(|_| Error::Ended)?; - let header = answer(&conn)?; - let mut payload = [0u8; Opened::BYTES]; - let len = conn.recv_bytes(&header, &mut payload).map_err(|_| Error::Ended)?; - match header.msg_type { + let mut rx = Box::new(FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let frame = answer(&conn, &mut rx)?; + match frame.msg_type { wire::MSG_OPENED => { - let [ring, answered] = conn.recv_handles_exact::<{ wire::OPENED_HANDLES }>(&header).ok_or(Error::Protocol)?; - let opened = Opened::decode(&payload[..len]).ok_or(Error::Protocol)?; + let [ring, answered] = rx.take_handles::<{ wire::OPENED_HANDLES }>().ok_or(Error::Protocol)?; + let opened = Opened::decode(rx.payload(frame.payload_len)).ok_or(Error::Protocol)?; Ok((Bells { ring: Waker::from(ring), answered: Bell::from(answered) }, opened)) } - wire::MSG_REFUSED => Err(Refusal::decode(&payload[..len]).map_or(Error::Protocol, Error::Refused)), + wire::MSG_REFUSED => Err(Refusal::decode(rx.payload(frame.payload_len)).map_or(Error::Protocol, Error::Refused)), _ => Err(Error::Protocol), } } @@ -442,14 +444,12 @@ fn handshake( pub fn list(names: &Namespace, service: &str) -> Result, Error> { let conn = connect(names, service)?; conn.signal(wire::MSG_LIST).map_err(|_| Error::Ended)?; - let header = answer(&conn)?; - let mut payload = vec![0u8; header.len() as usize]; - let len = conn.recv_bytes(&header, &mut payload).map_err(|_| Error::Ended)?; - match header.msg_type { - wire::MSG_LISTED => { - wire::Listed::decode_all(&payload[..len]).map(Iterator::collect).ok_or(Error::Protocol) - } - wire::MSG_REFUSED => Err(Refusal::decode(&payload[..len]).map_or(Error::Protocol, Error::Refused)), + let mut rx = Box::new(FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let frame = answer(&conn, &mut rx)?; + let payload = rx.payload(frame.payload_len); + match frame.msg_type { + wire::MSG_LISTED => wire::Listed::decode_all(payload).map(Iterator::collect).ok_or(Error::Protocol), + wire::MSG_REFUSED => Err(Refusal::decode(payload).map_or(Error::Protocol, Error::Refused)), _ => Err(Error::Protocol), } } diff --git a/system/diskserver/src/tables.rs b/system/diskserver/src/tables.rs index 48da7223035..a199b3e96e0 100644 --- a/system/diskserver/src/tables.rs +++ b/system/diskserver/src/tables.rs @@ -13,11 +13,9 @@ const FRAME: usize = 4 + 2 * (2 + wire::MAX_STATE) + 64; /// One frame's type and payload on `conn`, within the answer bound. fn frame(conn: &toyos::ipc::Connection) -> Result<(u32, Vec), Error> { - let header = answer(conn)?; - let mut payload = vec![0u8; (header.len() as usize).min(FRAME)]; - let len = conn.recv_bytes(&header, &mut payload).map_err(|_| Error::Ended)?; - payload.truncate(len); - Ok((header.msg_type, payload)) + let mut rx = toyos::ipc::FrameRx::::new(); + let frame = answer(conn, &mut rx)?; + Ok((frame.msg_type, rx.payload(frame.payload_len).to_vec())) } fn refused(payload: &[u8]) -> Error { diff --git a/system/grants/src/main.rs b/system/grants/src/main.rs index 90e4ddb0594..4699632e8a9 100644 --- a/system/grants/src/main.rs +++ b/system/grants/src/main.rs @@ -64,15 +64,12 @@ fn ask(request: &Request) -> (bool, String) { if let Err(e) = supervisor.send_bytes(msg_type, &payload) { return (false, format!("the supervisor did not take the request: {e:?}")); } - let answer = match supervisor.recv_header() { + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let answer = match rx.recv(&supervisor) { Ok(answer) => answer, Err(e) => return (false, format!("the supervisor did not answer: {e:?}")), }; - let mut text = vec![0u8; answer.len() as usize]; - let said = match supervisor.recv_bytes(&answer, &mut text) { - Ok(n) => String::from_utf8_lossy(&text[..n]).into_owned(), - Err(e) => return (false, format!("the supervisor's answer did not arrive whole: {e:?}")), - }; + let said = String::from_utf8_lossy(rx.payload(answer.payload_len)).into_owned(); match answer.msg_type { grants::MSG_DONE => (true, said), grants::MSG_REFUSED => (false, said), diff --git a/system/hda/src/main.rs b/system/hda/src/main.rs index 67df408d48f..dde97a4cec6 100644 --- a/system/hda/src/main.rs +++ b/system/hda/src/main.rs @@ -444,20 +444,17 @@ fn offered(control: &Connection, offer: &Offer) -> Result if !answered { return Err(refused(format!("it did not answer the offer within {OFFER_ANSWER:?}"))); } - let header = control.recv_header().map_err(|why| refused(format!("no answer to the offer ({why:?})")))?; - let mut payload = [0u8; MAX_REQUEST]; - let n = control - .recv_bytes(&header, &mut payload) - .map_err(|why| refused(format!("an answer of {} bytes ({why:?})", header.len())))?; - match ToDriver::decode(header.msg_type, &payload[..n]) { + let mut rx = FrameRx::::new(); + let frame = rx.recv(control).map_err(|why| refused(format!("no answer to the offer ({why:?})")))?; + match ToDriver::decode(frame.msg_type, rx.payload(frame.payload_len)) { Some(ToDriver::Accepted) => { - let [ring] = control - .recv_handles_exact::<{ output::ACCEPTED_HANDLES }>(&header) + let [ring] = rx + .take_handles::<{ output::ACCEPTED_HANDLES }>() .ok_or_else(|| refused("its acceptance carried no ring".into()))?; Ok(ring) } Some(ToDriver::Refused) => Err(Refusal::Declined), - _ => Err(refused(format!("it answered the offer with frame {}", header.msg_type))), + _ => Err(refused(format!("it answered the offer with frame {}", frame.msg_type))), } } diff --git a/system/logout/src/main.rs b/system/logout/src/main.rs index b25b43bae90..5224809d441 100644 --- a/system/logout/src/main.rs +++ b/system/logout/src/main.rs @@ -60,15 +60,12 @@ fn ask(msg: u32, payload: &[u8]) -> (bool, String) { if let Err(e) = supervisor.send_bytes(msg, payload) { return (false, format!("logout: the request was not sent: {e:?}")); } - let answer = match supervisor.recv_header() { + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let answer = match rx.recv(&supervisor) { Ok(answer) => answer, Err(e) => return (false, format!("logout: the supervisor did not answer: {e:?}")), }; - let mut text = vec![0u8; answer.len() as usize]; - let said = match supervisor.recv_bytes(&answer, &mut text) { - Ok(n) => String::from_utf8_lossy(&text[..n]).into_owned(), - Err(e) => return (false, format!("logout: the supervisor's answer did not arrive whole: {e:?}")), - }; + let said = String::from_utf8_lossy(rx.payload(answer.payload_len)).into_owned(); match answer.msg_type { wire::MSG_SESSIONS | wire::MSG_DONE => (true, said), wire::MSG_REFUSED => (false, said), diff --git a/system/netstack/src/client.rs b/system/netstack/src/client.rs index 8f0ec6a9099..dad43411be1 100644 --- a/system/netstack/src/client.rs +++ b/system/netstack/src/client.rs @@ -37,18 +37,17 @@ impl Client { /// carries one request, so hanging up is the one thing it may say while /// it waits; a client that says more is dropped by name. pub fn gone(&self) -> bool { - let mut byte = [0u8; 1]; - match self.conn.read_nonblock(&mut byte) { - Err(toyos_abi::syscall::SyscallError::WouldBlock) => false, - Ok(0) | Err(_) => true, - Ok(_) => { + match ipc::FrameRx::<0>::new().pump(&self.conn) { + ipc::RxStep::Idle => false, + ipc::RxStep::Eof => true, + ipc::RxStep::Frame { .. } | ipc::RxStep::Malformed => { say!("netstack: dropping client {} — it spoke again before its answer", self.conn.as_handle().raw().0); true } } } - /// **The answer goes out in one non-blocking write, and a refusal is not + /// **The answer goes out in one non-blocking send, and a refusal is not /// retried.** `ipc::send` parks in `SYS_MESSAGE_SEND` until the client /// drains, which is a client deciding when the network stack runs again. /// The connection closes either way. @@ -58,7 +57,7 @@ impl Client { if let Err(e) = sent { let why = match e { ipc::TrySendError::Full => { - "its pipe will not take the answer and it is not reading" + "its connection has no room for the answer and it is not reading" } ipc::TrySendError::TooLarge => "the answer netstack built is larger than a frame", ipc::TrySendError::Syscall(_) => "its connection is gone", @@ -84,12 +83,8 @@ pub const MAX_PENDING_CONNS: u32 = 32; /// bounds is the one that never sends it. pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); -/// One client's inbound framing. -/// -/// **netstack never reads a client with a blocking read.** That is the whole point -/// of [`ipc::FrameRx`]: `ipc::recv_header` and `ipc::recv_payload` park the -/// caller until the peer sends the bytes it promised. Here a peer that stops halfway -/// through a frame costs a buffer and a deadline instead of the event loop. +/// One client's inbound frames, taken without waiting +/// ([`ipc::FrameRx::pump`]): netstack never waits on a client. pub type ClientRx = ipc::FrameRx; /// A connection that has been accepted and has not yet said what it wants. diff --git a/system/netstack/src/link.rs b/system/netstack/src/link.rs index 30fdb000755..f8c05b409ea 100644 --- a/system/netstack/src/link.rs +++ b/system/netstack/src/link.rs @@ -1,5 +1,5 @@ -//! The stack's side of a link: the driver's connection, the region it sent, -//! and the stack's end of the region's rings (`toyos-link`). +//! The stack's side of a link: the region a driver sent with its hello, the +//! stack's end of the region's rings (`toyos-link`) and the two doorbells. //! //! **What the driver writes is believed only as `toyos-link`'s ends bound //! it**: a frame's length by its slot, the link by its word, and the counters @@ -9,7 +9,7 @@ use toyos::ipc::{Connection, FrameRx, RxStep}; use toyos::poller::READABLE; use toyos::say; -use toyos_abi::syscall::SyscallError; +use toyos::wake::{Bell, Waker}; use toyos_inspect::Snapshot; use toyos_link::ends::{Fault, Stack}; use toyos_link::layout::{RX_DEPTH, SLOT_BYTES}; @@ -36,9 +36,10 @@ pub enum Heard { Nothing, /// The driver went, or said what no driver says: it was refused, or let go. Gone, - /// The driver said who it is, its address as the node takes one: its - /// connection is [`Pending::into_conn`]. - Hello(Hello, IndividualMac, Region), + /// The driver said who it is, its address as the node takes one, with + /// its region and the bell it rings: its connection is + /// [`Pending::into_conn`]. + Hello(Hello, IndividualMac, Region, Bell), } impl Pending { @@ -58,12 +59,12 @@ impl Pending { match self.rx.pump(&self.conn) { RxStep::Idle => Heard::Nothing, RxStep::Eof => Heard::Gone, - RxStep::Frame { msg_type: wire::MSG_HELLO, payload_len, handles: 1 } => { + RxStep::Frame { msg_type: wire::MSG_HELLO, payload_len, handles: wire::HELLO_HANDLES } => { let hello = Hello::decode(self.rx.payload(payload_len)); let mac = hello.as_ref().and_then(|hello| IndividualMac::new(MacAddr(hello.mac.bytes()))); - let region = self.rx.take_handles::<1>().and_then(|[region]| Region::lent(region).ok()); - match (hello, mac, region) { - (Some(hello), Some(mac), Some(region)) => Heard::Hello(hello, mac, region), + let [region, rung] = self.rx.take_handles::<{ wire::HELLO_HANDLES }>().expect("counted above"); + match (hello, mac, Region::lent(region).ok()) { + (Some(hello), Some(mac), Some(region)) => Heard::Hello(hello, mac, region, Bell::from(rung)), _ => { say!("netstack: a NIC driver's hello is not one this protocol makes, or its region is none"); refuse(&self.conn, Refusal::Malformed); @@ -92,7 +93,10 @@ pub enum Ended { /// A link the stack opened. pub struct Session { - conn: Connection, + /// Rung for the driver. + ring_driver: Waker, + /// The driver's doorbell; its waker going is the driver's end. + rung: Bell, region: Region, end: Stack, hello: Hello, @@ -110,14 +114,31 @@ pub struct Session { impl Session { /// Open the link `hello` asked for over `region` as the node's `iface`, - /// and say so to the driver; `None` for a driver gone before it heard. - pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, groups: &[Group]) -> Option { + /// and say so to the driver with the bell this end rings, its `conn` let + /// go then; `None` for a driver gone before it heard. + pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, rung: Bell, groups: &[Group]) -> Option { let mut end = Stack::new(region.words()); for group in groups { assert!(end.join(region.words(), *group), "netstack: more groups than a link holds"); } - conn.try_send_bytes(wire::MSG_OPENED, &[]).ok()?; - Some(Self { conn, region, end, hello, iface, sent: 0, said: Link::Down, frame: Box::new([0; SLOT_BYTES]), asked: false, ring: true }) + let Ok((ring_driver, driver_hears)) = toyos::wake::pair() else { + say!("netstack: no doorbell for {}'s link", hello.label); + return None; + }; + conn.try_signal_with(wire::MSG_OPENED, [driver_hears.into()]).ok()?; + Some(Self { + ring_driver, + rung, + region, + end, + hello, + iface, + sent: 0, + said: Link::Down, + frame: Box::new([0; SLOT_BYTES]), + asked: false, + ring: true, + }) } pub fn iface(&self) -> IfIndex { @@ -135,7 +156,7 @@ impl Session { } pub fn watch(&self, poller: &mut Watches, token: u64) { - poller.watch(&self.conn, READABLE, token); + poller.watch(&self.rung, READABLE, token); } /// The link's state, the frames the driver received, and what is due @@ -219,29 +240,17 @@ impl Session { Some(counters) } - /// Ring the driver if this pass gave it anything to do; a refusal is a - /// full pipe, a doorbell already rung. + /// Ring the driver if this pass gave it anything to do. pub fn ring(&mut self) -> Result<(), Ended> { if std::mem::take(&mut self.ring) { - match self.conn.write_nonblock(&[1]) { - Ok(_) | Err(SyscallError::WouldBlock) => {} - Err(_) => return Err(Ended::HungUp), - } + self.ring_driver.ring().map_err(|_| Ended::HungUp)?; } Ok(()) } - /// Consume the driver's doorbell bytes; its hang-up is the link's end. + /// Take the driver's doorbell; its waker gone is the link's end. fn hear(&self) -> Result<(), Ended> { - let mut sink = [0u8; 64]; - loop { - match self.conn.read_nonblock(&mut sink) { - Ok(0) => return Err(Ended::HungUp), - Ok(_) => {} - Err(SyscallError::WouldBlock) => return Ok(()), - Err(_) => return Err(Ended::HungUp), - } - } + self.rung.try_take().map(drop).map_err(|_| Ended::HungUp) } /// What `inspect` reads about the link: its driver, its address, its diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index 65702ac6162..2be79753eb0 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -89,19 +89,20 @@ const LEASE_BOUND: Duration = Duration::from_millis(toyos_tco::LEASE_BOUND_MS); /// keeps up (`toyos-net-tcp`'s `rx`); a send buffer is the region's send ring. const TCP_BUFFER: u32 = toyos::net::stream::RECEIVE_BYTES; -/// The most a kernel pipe holds unread; the frames it takes for that add a -/// frame and their list's few KiB (`kernel::pipe::MOST_FRAMES`). A client's -/// pipes are counted to it, and netstack holding their far ends keeps what is -/// in them alive. -const PIPE_BYTES: u64 = PIPE_CAPACITY as u64; +/// The most bytes one direction of a connection holds unreceived; the frames +/// it takes for that add a frame and their list's few KiB +/// (`kernel::pipe::MOST_FRAMES`). A connection is counted to the client that +/// made it, and netstack holding its far end keeps what is in it alive. +const DIRECTION_BYTES: u64 = PIPE_CAPACITY as u64; -/// What one holder of a stream can make this machine hold: the two pipes of -/// its connection, each full. Its bells are events, which hold no bytes. -const HOLDER_BYTES: u64 = 2 * PIPE_BYTES; +/// What one holder of a stream can make this machine hold: the two +/// directions of its connection, each full. Its bells are events, which hold +/// no bytes. +const HOLDER_BYTES: u64 = 2 * DIRECTION_BYTES; /// What one holder of a listener or a datagram socket can make it hold: the -/// two pipes of its connection, its bell being an event. -const SOCKET_HOLDER_BYTES: u64 = 2 * PIPE_BYTES; +/// two directions of its connection, its bell being an event. +const SOCKET_HOLDER_BYTES: u64 = 2 * DIRECTION_BYTES; /// What one place can make this machine hold, at the largest of the things a /// place is (`toyos-net-node`'s `places`): a stream, its region and its first @@ -347,7 +348,7 @@ fn answer_inspect(request: &Request, links: &[Held], counters: &[(IfIndex, Count /// the boot, and say so. fn open(asked: Asked, iface: IfIndex, n: usize, groups: &[Group]) -> Option { let label = asked.hello.label.clone(); - let opened = Session::open(asked.hello, iface, asked.conn, asked.region, groups); + let opened = Session::open(asked.hello, iface, asked.conn, asked.region, asked.rung, groups); match &opened { Some(_) => say!("netstack: {label}'s link is open, as link {n}"), None => say!("netstack: {label} went before its link {n} was opened"), @@ -374,9 +375,9 @@ fn hear_links(pending: &mut Vec) -> Vec { Heard::Gone => { pending.remove(i); } - Heard::Hello(hello, mac, region) => { + Heard::Hello(hello, mac, region, rung) => { let conn = pending.remove(i).into_conn(); - asked.push(Asked { hello, mac, conn, region }); + asked.push(Asked { hello, mac, conn, region, rung }); } } } @@ -462,6 +463,8 @@ struct Asked { mac: IndividualMac, conn: toyos::Connection, region: toyos_link::region::Region, + /// The bell the driver rings. + rung: toyos::wake::Bell, } /// An `inspect` waiting for the drivers' counters. diff --git a/system/soundserver/src/control.rs b/system/soundserver/src/control.rs index 75eac14885c..0a7e783f1a3 100644 --- a/system/soundserver/src/control.rs +++ b/system/soundserver/src/control.rs @@ -47,12 +47,10 @@ const _: () = assert!( "a control payload wider than the frame buffer would be refused as malformed", ); -/// One client's inbound framing. +/// One client's inbound frames. /// -/// **soundserver never reads a client with a blocking read.** `ipc::recv_header` and -/// `ipc::recv_payload` park the caller until the peer sends the bytes it -/// promised, which hands a client the decision of when the control thread runs -/// again: one client parking a partial header would wedge accept and +/// **soundserver never waits on a client's receive**: one client deciding +/// when the control thread runs again would wedge accept and /// volume/close/disconnect handling for every other client (the mix thread is /// unaffected either way). The doctrine is `system/CLAUDE.md`'s. type ControlRx = ipc::FrameRx; diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index aca951cd0ef..3fc6a17f8aa 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -170,13 +170,9 @@ const MAX_PENDING_LAUNCHES: usize = 32; /// sends it, and it is what guarantees the table above drains. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); -/// One caller's inbound framing. -/// -/// **The supervisor never reads a client with a blocking read.** `recv_header` and -/// `recv_bytes` park the caller until the peer sends the bytes it promised. The -/// whole request blob is kept because a launch carries the child's argv, -/// environment and working directory, and a truncated one is a launch refused -/// for a reason the caller did not cause. +/// One caller's inbound frames, taken without waiting: the supervisor never +/// waits on a client. Room for the largest frame, because a launch carries +/// the child's argv, environment and working directory. type LaunchRx = ipc::FrameRx<{ ipc::MAX_FRAME_LEN as usize }>; /// A connection that has been accepted and has not yet said what to start. diff --git a/system/swap/src/main.rs b/system/swap/src/main.rs index bc07efa7edd..e6b472f8eff 100644 --- a/system/swap/src/main.rs +++ b/system/swap/src/main.rs @@ -118,15 +118,12 @@ fn ask(asked: &Asked) -> (Answer, Option) { let _ = std::fs::remove_file(&request.staged); return unasked(format!("the supervisor did not take the request: {e:?}")); } - let answer = match supervisor.recv_header() { + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let answer = match rx.recv(&supervisor) { Ok(answer) => answer, Err(e) => return (Answer::Refused(format!("the supervisor did not answer: {e:?}")), None), }; - let mut text = vec![0u8; answer.len() as usize]; - let said = match supervisor.recv_bytes(&answer, &mut text) { - Ok(n) => String::from_utf8_lossy(&text[..n]).into_owned(), - Err(e) => return (Answer::Refused(format!("the supervisor's answer did not arrive whole: {e:?}")), None), - }; + let said = String::from_utf8_lossy(rx.payload(answer.payload_len)).into_owned(); match answer.msg_type { toyos_swap::MSG_ACCEPTED => (Answer::Accepted(said), Some(supervisor)), toyos_swap::MSG_REFUSED => (Answer::Refused(said), None), diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index d8d47990231..5c3891b66f4 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -84,8 +84,10 @@ pub mod ipc { use std::os::toyos::process::CommandExt; use std::process::{Child, Command, Stdio}; + use std::cell::RefCell; + use toyos::endow::{Endowments, FromHandle}; - use toyos::ipc::{Connection, MAX_FRAME_LEN}; + use toyos::ipc::{Connection, FrameRx, MAX_FRAME_LEN}; use toyos::port::{self, Acceptor}; use toyos::{OwnedHandle, Pipe}; @@ -96,7 +98,7 @@ pub mod ipc { /// One end of a connection. A message longer than one frame /// ([`MAX_FRAME_LEN`]) goes as consecutive frames: a ToyOS connection /// frames each message, and a frame carries at most that much. - pub struct Peer(Connection); + pub struct Peer(Connection, RefCell>>); /// What a message carries beside its bytes: one end of a pipe. pub type Token = Pipe; @@ -124,28 +126,46 @@ pub mod ipc { pub fn recv(&self, buf: &mut [u8]) -> io::Result<()> { let mut at = 0; loop { - let header = self.0.recv_header().map_err(|e| failed("receive", e))?; - at += self.0.recv_bytes(&header, &mut buf[at..]).map_err(|e| failed("receive", e))?; + at += self.take(&mut buf[at..])?; if at >= buf.len() { return Ok(()); } } } + /// One frame into the front of `buf`: the bytes it held. + fn take(&self, buf: &mut [u8]) -> io::Result { + let mut rx = self.1.borrow_mut(); + let frame = rx.recv(&self.0).map_err(|e| failed("receive", e))?; + Ok(copy(rx.payload(frame.payload_len), buf)) + } + /// One frame of `data` (at most one frame's worth) and `token`, moved. pub fn send_with(&self, data: &[u8], token: Token) -> io::Result<()> { self.0.send_bytes_with(1, data, [OwnedHandle::from(token)]).map_err(|e| failed("send", e)) } pub fn recv_with(&self, buf: &mut [u8]) -> io::Result { - let header = self.0.recv_header().map_err(|e| failed("receive", e))?; - self.0.recv_bytes(&header, buf).map_err(|e| failed("receive", e))?; - let [handle] = self.0.recv_handles_exact::<1>(&header).ok_or_else(|| io::Error::other("no handle came"))?; + let mut rx = self.1.borrow_mut(); + let frame = rx.recv(&self.0).map_err(|e| failed("receive", e))?; + copy(rx.payload(frame.payload_len), buf); + let [handle] = rx.take_handles::<1>().ok_or_else(|| io::Error::other("no handle came"))?; // The peer sends only pipe ends. Ok(Pipe::from_handle(handle)) } } + /// As much of `from` as `to` holds, at its front. + fn copy(from: &[u8], to: &mut [u8]) -> usize { + let n = from.len().min(to.len()); + to[..n].copy_from_slice(&from[..n]); + n + } + + fn peer(conn: Connection) -> Peer { + Peer(conn, RefCell::new(Box::new(FrameRx::new()))) + } + pub fn token() -> io::Result { let (read, _write) = toyos::pipe_pair().map_err(|e| failed("pipe", e))?; Ok(read) @@ -162,14 +182,14 @@ pub mod ipc { .endow(LABEL, acceptor.into()) .stdin(Stdio::null()) .spawn()?; - Ok((child, Peer(conn))) + Ok((child, peer(conn))) } /// The child's end: the connection its parent made to the port it was /// endowed. pub fn served() -> io::Result { let acceptor: Acceptor = Endowments::get().take(LABEL).ok_or_else(|| io::Error::other("no port endowed"))?; - acceptor.accept().map(Peer).map_err(|e| failed("accept", e)) + acceptor.accept().map(peer).map_err(|e| failed("accept", e)) } /// A port this process both connects to and accepts on, for timing a diff --git a/tests/guest/rust-tests/src/bin/abuse_connect_flood.rs b/tests/guest/rust-tests/src/bin/abuse_connect_flood.rs index 8aad797f2c6..e5916415c81 100644 --- a/tests/guest/rust-tests/src/bin/abuse_connect_flood.rs +++ b/tests/guest/rust-tests/src/bin/abuse_connect_flood.rs @@ -97,19 +97,19 @@ fn main() { // A depth, not a permanent refusal: draining one must let one more in. let accepted = syscall::accept(acceptor.as_handle().raw()).expect("accept a queued connection"); - // The other direction, without which "costs nothing" could be a pipe that - // never works: one byte on a live connection has to buy a frame. This is - // the negative control for the ceiling above — a kernel that allocated - // nothing ever would pass that assertion and fail this one. + // The other direction, without which "costs nothing" could be a + // connection that never works: one byte on a live connection has to buy a + // frame. This is the negative control for the ceiling above — a kernel + // that allocated nothing ever would pass that assertion and fail this one. let before_write = used_bytes(); - let wrote = syscall::write(accepted, b"x").expect("write one byte to an accepted socket"); - assert_eq!(wrote, 1); + // SAFETY: no handles travel. + unsafe { syscall::message_send(accepted, b"x", &[], false) }.expect("send one byte on an accepted connection"); let took = used_bytes().saturating_sub(before_write); assert!( took >= syscall::PAGE_SIZE as u64, - "the first write to a connection grew physical memory by {took} bytes; a pipe's bytes take a frame" + "the first message on a connection grew physical memory by {took} bytes; its bytes take a frame" ); - println!(" the first write on a connection allocated {} KiB", took / 1024); + println!(" the first message on a connection allocated {} KiB", took / 1024); // SAFETY: no type holds this number. unsafe { syscall::close(accepted) }; diff --git a/tests/guest/rust-tests/src/bin/abuse_shared_grant.rs b/tests/guest/rust-tests/src/bin/abuse_shared_grant.rs index 7f6324a340b..b4e66cb65fd 100644 --- a/tests/guest/rust-tests/src/bin/abuse_shared_grant.rs +++ b/tests/guest/rust-tests/src/bin/abuse_shared_grant.rs @@ -167,8 +167,9 @@ fn marker(role: &str) { fn invited() { let conn = toyos::endow::service(SERVICE).expect("the invited peer holds a connector"); - let header = conn.recv_header().expect("the owner's announcement"); - let [handle] = conn.recv_handles_exact::<1>(&header).expect("the region the owner sent"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + rx.recv(&conn).expect("the owner's announcement"); + let [handle] = rx.take_handles::<1>().expect("the region the owner sent"); let region = SharedMemory::lent(handle, REGION).expect("take the region"); println!("read {}", String::from_utf8_lossy(®ion.bytes().sub(0, SECRET.len()).to_vec())); std::io::stdout().flush().expect("invited: flush the report"); diff --git a/tests/guest/rust-tests/src/bin/connect_before_serve.rs b/tests/guest/rust-tests/src/bin/connect_before_serve.rs index c5efb62e717..61a45a8f2f9 100644 --- a/tests/guest/rust-tests/src/bin/connect_before_serve.rs +++ b/tests/guest/rust-tests/src/bin/connect_before_serve.rs @@ -7,18 +7,18 @@ //! each with a bound and a sleep, each turning "the server is not up yet" into a //! duration. A port exists before either end's process does, so there is no //! "not yet" to retry — a connection made against an acceptor nobody holds yet -//! is queued on the port, and the bytes written into it are in the ring waiting -//! when the server first looks. +//! is queued on the port, and the message sent on it is waiting when the +//! server first looks. //! //! Two arms, and they must answer differently: //! -//! 1. The client connects and writes **before the server exists**. The server is -//! started afterwards, and at its very first look — a *non-blocking* read -//! taken the instruction after `accept` returns — the client's frame is -//! already there. It replies, and the client reads the reply. +//! 1. The client connects and sends **before the server exists**. The server +//! is started afterwards, and at its very first look — a *non-blocking* +//! receive taken the instruction after `accept` returns — the client's +//! message is already there. It replies, and the client takes the reply. //! 2. The same client, and a server that takes the acceptor and exits. The -//! client's read answers `0` and its next write is refused. Both are facts -//! the kernel states, not conclusions a clock reached. +//! client's receive answers `Gone` and its next send is refused. Both are +//! facts the kernel states, not conclusions a clock reached. //! //! **No wall clock is asserted anywhere here, deliberately.** A tree that put a //! retry back would not fail an arm slowly, it would *hang* — the client would @@ -107,11 +107,11 @@ fn left() { let heard = line(&mut from_client); assert_eq!( - heard, "read 0 then refused", + heard, "received gone then refused", "a client of a server that left was told something else", ); assert!(client.wait().expect("wait the client").success(), "the client exited nonzero"); - println!(" left: the client read 0 and its next write was refused"); + println!(" left: the client's receive answered gone and its next send was refused"); } /// Spawn the client and return once its frame is provably in the ring. @@ -143,29 +143,31 @@ fn say(what: &str) { std::io::stdout().flush().expect("flush"); } +fn send(conn: &impl AsHandle, bytes: &[u8], nonblock: bool) -> Result<(), SyscallError> { + // SAFETY: no handles travel. + unsafe { syscall::message_send(conn.as_handle().raw(), bytes, &[], nonblock) } +} + fn client() -> ! { let expect = std::env::args().nth(2).expect("the client needs to know what to expect"); let conn = toyos::endow::service(SERVICE).expect("the client holds a connector"); - conn.write_nonblock(QUESTION).expect("write into a port nobody has accepted from"); + send(&conn, QUESTION, true).expect("send into a port nobody has accepted from"); say("sent"); let mut buf = [0u8; 64]; - let n = syscall::read(conn.as_handle().raw(), &mut buf).expect("read the connection"); + let got = syscall::message_recv(conn.as_handle().raw(), &mut buf, &mut [], false); match expect.as_str() { "reply" => { + let n = got.expect("the reply").len; assert_eq!(&buf[..n], ANSWER, "the reply was not the server's"); say(&format!("reply: {}", String::from_utf8_lossy(&buf[..n]))); } "gone" => { - assert_eq!(n, 0, "a connection whose server left returned {n} bytes"); // The word this arm was specified for: the peer end has closed, and // the client is told so by the kernel without a timer. - assert_eq!( - conn.write_nonblock(QUESTION), - Err(SyscallError::Gone), - "a write into a port whose acceptor is gone was taken", - ); - say("read 0 then refused"); + assert_eq!(got, Err(SyscallError::Gone), "a connection whose server left answered a receive"); + assert_eq!(send(&conn, QUESTION, true), Err(SyscallError::Gone), "a send into a port whose acceptor is gone was taken"); + say("received gone then refused"); } other => panic!("unknown expectation {other:?}"), } @@ -176,17 +178,16 @@ fn server() -> ! { let acceptor: Acceptor = Endowments::get().take(ACCEPTOR_LABEL).expect("the server was endowed an acceptor"); let conn = acceptor.accept().expect("accept the queued connection"); - // The instruction after `accept`, and it must not block: the client wrote - // before this process existed, so the bytes are in the ring already. + // The instruction after `accept`, and it must not wait: the client sent + // before this process existed, so the message is queued already. let mut buf = [0u8; 64]; - let n = conn - .read_nonblock(&mut buf) - .expect("the client's frame was not in the ring at the server's first look"); + let n = syscall::message_recv(conn.as_handle().raw(), &mut buf, &mut [], true) + .expect("the client's message was not queued at the server's first look") + .len; say(&format!("already here: {}", String::from_utf8_lossy(&buf[..n]))); - conn.write_nonblock(ANSWER).expect("reply"); - // Held until the client has read the reply, which its own exit reports. - let mut wait = [0u8; 1]; - let _ = syscall::read(conn.as_handle().raw(), &mut wait); + send(&conn, ANSWER, false).expect("reply"); + // Held until the client has taken the reply, which its own exit reports. + let _ = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut [], false); std::process::exit(0); } diff --git a/tests/guest/rust-tests/src/bin/cpal_drop_unreleased.rs b/tests/guest/rust-tests/src/bin/cpal_drop_unreleased.rs index 8cb9392c5af..80f98cf4cb1 100644 --- a/tests/guest/rust-tests/src/bin/cpal_drop_unreleased.rs +++ b/tests/guest/rust-tests/src/bin/cpal_drop_unreleased.rs @@ -81,11 +81,12 @@ fn serve() { // it, and the stream is the connection whose first frame is an open. let conn = loop { let conn = acceptor.accept().expect("the client's connection"); - let header = conn.recv_header().expect("the client's first frame"); - match header.msg_type { + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("the client's first frame"); + match frame.msg_type { MSG_DEVICE_DESCRIBE => conn.send(MSG_DEVICE_DESCRIBED, &DEVICE).expect("the description"), MSG_STREAM_OPEN => { - let _: StreamOpenRequest = conn.recv_payload(&header).expect("the client's open"); + let _: StreamOpenRequest = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the client's open"); break conn; } other => panic!("the client's first frame is message {other}, neither a describe nor an open"), @@ -99,7 +100,8 @@ fn serve() { conn.send_with(MSG_STREAM_OPENED, &OPENED, [ring.share().expect("the ring to share"), signal_bell.into()]) .expect("the open answered with its handles"); - let close = conn.recv_header().expect("the client's close"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let close = rx.recv(&conn).expect("the client's close"); assert_eq!(close.msg_type, MSG_STREAM_CLOSE, "the dropped stream did not close"); // The signal side is held until the client has gone, so nothing but its @@ -113,11 +115,11 @@ fn serve() { print!("{said}"); signal.ring().expect("a signal to the stream thread"); - match conn.recv_header() { + match rx.recv(&conn) { Err(IpcError::Disconnected) => {} other => panic!( "the stream's connection gave {:?} after its close, not its end", - other.map(|h| h.msg_type) + other.map(|f| f.msg_type) ), } // The client exits once its stdin closes, so the connection's end above diff --git a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs index c82c7bef7f8..a1f72929169 100644 --- a/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs +++ b/tests/guest/rust-tests/src/bin/libc_recv_beside_close.rs @@ -127,10 +127,11 @@ fn test(role: &str) { let mut out = BufReader::new(child.stdout.take().expect("the child's stdout")); let conn = acceptor.accept().expect("the child's connect"); - let header = conn.recv_header().expect("a request"); - assert_eq!(header.msg_type, MsgType::TcpConnect as u32, "a stream's connect"); - let _: TcpConnectRequest = conn.recv_payload(&header).expect("the connect's request"); - let [region] = conn.recv_handles_exact::<1>(&header).expect("the stream's region"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("a request"); + assert_eq!(frame.msg_type, MsgType::TcpConnect as u32, "a stream's connect"); + let _: TcpConnectRequest = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the connect's request"); + let [region] = rx.take_handles::<1>().expect("the stream's region"); let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); // SAFETY: the mapping is `REGION_BYTES`, 2 MiB aligned, and its header is reached only as atomics. let words = unsafe { &*(region.as_ptr() as *const Words) }; diff --git a/tests/guest/rust-tests/src/bin/memory_floor.rs b/tests/guest/rust-tests/src/bin/memory_floor.rs index 1b509a8f40c..1178cf848d2 100644 --- a/tests/guest/rust-tests/src/bin/memory_floor.rs +++ b/tests/guest/rust-tests/src/bin/memory_floor.rs @@ -8,8 +8,8 @@ //! while that holds them: //! //! - a server in an ordinary job answers this job's connection, before any of -//! that, and its account does not move: a connection's pipes are its -//! client's; +//! that, and its account does not move: a connection's two directions are +//! its client's; //! - an ordinary job's spawn is refused, so the exhaustion is real; //! - the supervisor answers a stop's request on a new connection, which takes //! nothing of the supervisor's own; @@ -101,7 +101,7 @@ fn control(input: bool) { let mut log = Log { pipe: logkeeper_api::read().expect("the log, before anything is taken").pipe, bytes: vec![0; 64 << 10], held: 0 }; log.drain(); - pipes_are_the_clients(&cap); + directions_are_the_clients(&cap); let job = cap.new_job().expect("an ordinary job for the greedy one"); let mut greedy = Command::new(ME) @@ -162,7 +162,7 @@ fn control(input: bool) { }; println!(" memory_floor: the greedy job took back {} KiB, starveling having given back {} KiB", taken >> 10, given >> 10); starved.signal(END).expect("tell starveling to end"); - assert!(starved.recv_header().is_err(), "starveling answered, and was to end"); + assert!(toyos::ipc::FrameRx::<0>::new().recv(&starved).is_err(), "starveling answered, and was to end"); log.await_line(&["supervisor: starveling (pid ", REFUSED_RESTART]); println!(" PASS: starveling ended, and its restart was refused for memory"); // Asked of a port no process serves now: it waits for the next. @@ -188,10 +188,10 @@ fn control(input: bool) { } /// A server in an ordinary job of its own accepts this job's connection and -/// answers it: both of the connection's pipes are its client's, so the +/// answers it: both of the connection's directions are its client's, so the /// server's reply costs its job nothing, and no client can spend a server's /// floor by connecting to it. -fn pipes_are_the_clients(cap: &SysCap) { +fn directions_are_the_clients(cap: &SysCap) { let (acceptor, connector) = toyos::port::create().expect("a port"); let job = cap.new_job().expect("a job for the server"); let mut server = Command::new(ME) @@ -217,7 +217,7 @@ fn pipes_are_the_clients(cap: &SysCap) { assert_eq!(after.kernel, before.kernel, "the server's reply was charged to the server"); server.kill().expect("end the server"); server.wait().expect("the server's end"); - println!(" PASS: a connection's pipes are its client's: the server's reply cost its job nothing"); + println!(" PASS: a connection's directions are its client's: the server's reply cost its job nothing"); } /// Accept one connection on the endowed acceptor and answer its [`PING`]. @@ -292,11 +292,9 @@ fn ping(conn: &Connection) -> Result<(), String> { /// One whole frame off `conn`, waiting for it. fn frame(conn: &Connection) -> (u32, Vec) { - let header = conn.recv_header().expect("a frame's header"); - let mut payload = vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut payload).expect("a frame's payload"); - payload.truncate(n); - (header.msg_type, payload) + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("a frame"); + (frame.msg_type, rx.payload(frame.payload_len).to_vec()) } /// Wait until `conn` has something to read, within [`CEILING`]. diff --git a/tests/guest/rust-tests/src/bin/message_carries_its_handles.rs b/tests/guest/rust-tests/src/bin/message_carries_its_handles.rs deleted file mode 100644 index afffa1f4a0a..00000000000 --- a/tests/guest/rust-tests/src/bin/message_carries_its_handles.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! A connection carries handles only inside messages: the batch is pinned -//! just past its frame's bytes, so it arrives with them and with nothing else. -//! -//! - A frame's handles are taken with the frame. -//! - Handles with no bytes are refused whole: a poll-driven server sees -//! nothing to read, and the sender still holds what it tried to send. -//! - A batch its frame did not announce stops the reader at it, and dies with -//! the connection the reader then drops. - -use toyos::ipc::{Connection, FrameRx, RxStep}; -use toyos::poller::{Poller, OTHER_END_GONE, READABLE}; -use toyos::{namespace, pipe_pair, port, AsHandle, OwnedHandle, Pipe}; -use toyos_abi::syscall::{self, SyscallError}; - -/// A failure ceiling only: an answer that never comes is the defect. -const CEILING_NANOS: u64 = 10_000_000_000; - -fn main() { - let (acceptor, connector) = port::create().expect("a port"); - let names = namespace::build().add("conn", &connector).finish().expect("a namespace"); - let pair = || { - let client = names.open("conn").expect("connect"); - (client, acceptor.accept().expect("accept")) - }; - - // A frame's handles arrive with it. - let (client, server) = pair(); - let (read_end, write_end) = pipe_pair().expect("a pipe"); - client.signal_with(7, [OwnedHandle::from(write_end)]).expect("a frame carrying a handle"); - let header = server.recv_header().expect("the frame"); - assert_eq!((header.msg_type, header.handles()), (7, 1), "the frame does not announce its handle"); - let [received] = server.recv_handles_exact::<1>(&header).expect("the handle, with its frame"); - drop(received); - assert_eq!(gone(&read_end), Ok(OTHER_END_GONE), "the handle taken with its frame is not the one sent"); - println!("message_carries_its_handles: a frame's handle arrived with it"); - - // Handles with no bytes are refused, and the server has nothing to read. - let (stray_read, stray_write) = pipe_pair().expect("a pipe"); - assert_eq!( - // SAFETY: no type holds the handles sent. - unsafe { syscall::message_send(client.as_handle().raw(), &[], &[stray_write.as_handle().raw()], false) }, - Err(SyscallError::InvalidArgument), - "handles were sent with no bytes" - ); - assert!(!readable(&server, 0), "a refused send left the server something to read"); - assert_eq!(stray_write.write(b"x"), Ok(1), "a refused handle left its sender"); - drop(stray_write); - assert_eq!(gone(&stray_read), Ok(OTHER_END_GONE), "the refused handle stayed anywhere but here"); - println!("message_carries_its_handles: handles alone were refused, and the server stayed idle"); - - // A batch its frame did not announce stops the reader, and goes with the connection. - let (client, server) = pair(); - let (lied_read, lied_write) = pipe_pair().expect("a pipe"); - let mut frame = [0u8; 12]; - frame[..4].copy_from_slice(&9u32.to_ne_bytes()); - // Given up before the send, which moves it: this process no longer holds the number. - let lied_write = lied_write.into_raw(); - // SAFETY: no type holds the handles sent. - unsafe { syscall::message_send(client.as_handle().raw(), &frame, &[lied_write], false) } - .expect("a message whose frame announces no handle"); - let mut rx = FrameRx::<0>::new(); - assert!(readable(&server, CEILING_NANOS), "the message did not make the server readable"); - assert!( - matches!(rx.pump(&server), RxStep::Frame { msg_type: 9, handles: 0, .. }), - "the frame was not read as it was written" - ); - assert!(matches!(rx.pump(&server), RxStep::Malformed), "the reader went past a batch no frame announced"); - drop(rx); - drop(server); - assert_eq!(gone(&lied_read), Ok(OTHER_END_GONE), "the unannounced batch outlived its connection"); - drop(client); - println!("message_carries_its_handles: ok"); -} - -/// Whether `conn` answers readable within `timeout`. -fn readable(conn: &Connection, timeout: u64) -> bool { - let poller = Poller::new(1); - poller.watch(conn, READABLE, 1); - let mut answered = false; - poller.wait(u32::from(timeout != 0), timeout, |_| answered = true); - answered -} - -/// What a watch for `end`'s other end leaving is answered, within the ceiling. -fn gone(end: &Pipe) -> Result { - let poller = Poller::new(1); - poller.watch(&end, OTHER_END_GONE, 1); - let mut answer = None; - poller.wait_answers(1, CEILING_NANOS, |_, said| answer = Some(said)); - answer.expect("the other end never left") -} diff --git a/tests/guest/rust-tests/src/bin/messages_are_whole.rs b/tests/guest/rust-tests/src/bin/messages_are_whole.rs new file mode 100644 index 00000000000..935c776d9ba --- /dev/null +++ b/tests/guest/rust-tests/src/bin/messages_are_whole.rs @@ -0,0 +1,176 @@ +//! A connection carries whole messages: each is taken whole, in the order it +//! was sent, with exactly the handles it carried, or refused with nothing of +//! it moved. What a host test of the queue cannot show is the kernel's own +//! boundary: the syscalls' refusals, the handles moving between tables and +//! the bounds as a sender meets them. +//! +//! - Three messages arrive in order, each with its own bytes and handles. +//! - A buffer a byte or a handle short is refused, and the message stays +//! queued to be taken whole. +//! - Each bound (256 messages, 64 handles, 2 MiB of bytes) refuses the send +//! past it with nothing moved, and room comes back as messages are taken. +//! - Plain reads and writes of a connection are refused. +//! - A receive after the peer has gone takes what it sent, then answers +//! `Gone`; a send to a peer gone answers `Gone`. + +use toyos::ipc::Connection; +use toyos::poller::OTHER_END_GONE; +use toyos::{namespace, pipe_pair, port, AsHandle, OwnedHandle, Pipe}; +use toyos_abi::syscall::{ + self, Received, SyscallError, MAX_MESSAGE_LEN, MAX_QUEUED_HANDLES, MAX_QUEUED_MESSAGES, MAX_TRANSFER_HANDLES, + PIPE_CAPACITY, +}; +use toyos_abi::RawHandle; + +/// A failure ceiling only: an answer that never comes is the defect. +const CEILING_NANOS: u64 = 10_000_000_000; + +fn main() { + let (acceptor, connector) = port::create().expect("a port"); + let names = namespace::build().add("conn", &connector).finish().expect("a namespace"); + let pair = || { + let client = names.open("conn").expect("connect"); + (client, acceptor.accept().expect("accept")) + }; + + in_order_with_their_own_handles(pair()); + println!("messages_are_whole: three messages arrived in order, each with its own handles"); + a_short_buffer_leaves_it_queued(pair()); + println!("messages_are_whole: a short buffer took nothing, and the message came whole after"); + each_bound_refuses_the_send_past_it(pair()); + println!("messages_are_whole: each bound refused the send past it and moved nothing"); + bytes_alone_are_refused(pair()); + println!("messages_are_whole: plain reads and writes were refused"); + gone_after_what_was_sent(pair()); + println!("messages_are_whole: ok"); +} + +fn send(conn: &Connection, bytes: &[u8], handles: Vec, nonblock: bool) -> Result<(), SyscallError> { + let raw: Vec = handles.iter().map(|h| h.as_handle().raw()).collect(); + // SAFETY: the handles are this process's; on success they are the peer's + // and forgotten here, and on a refusal they are still here and close as + // they drop. + let sent = unsafe { syscall::message_send(conn.as_handle().raw(), bytes, &raw, nonblock) }; + if sent.is_ok() { + for h in handles { + // The number is the peer's now: given up, never closed here. + let _ = h.into_raw(); + } + } + sent +} + +/// The next message, its bytes and its handles, without waiting. +fn recv(conn: &Connection) -> Result<(Vec, Vec), SyscallError> { + let mut bytes = vec![0u8; MAX_MESSAGE_LEN]; + let mut handles = [RawHandle(0); MAX_TRANSFER_HANDLES]; + let got = syscall::message_recv(conn.as_handle().raw(), &mut bytes, &mut handles, true)?; + bytes.truncate(got.len); + // SAFETY: the kernel installed these numbers in this process for this call. + Ok((bytes, handles[..got.handles].iter().map(|&h| unsafe { OwnedHandle::from_raw(h) }).collect())) +} + +/// A pipe end to send, and the end that sees it go. +fn token() -> (OwnedHandle, Pipe) { + let (read, write) = pipe_pair().expect("a pipe"); + (OwnedHandle::from(write), read) +} + +/// Whether `end`'s other end has gone, within the ceiling. +fn gone(end: &Pipe) -> bool { + toyos::poller::wait_one(end, OTHER_END_GONE, CEILING_NANOS) == Ok(true) +} + +fn in_order_with_their_own_handles((client, server): (Connection, Connection)) { + let (a, a_seen) = token(); + let (c1, c1_seen) = token(); + let (c2, c2_seen) = token(); + send(&client, b"a", vec![a], false).expect("the first message"); + send(&client, b"bb", vec![], false).expect("the second message"); + send(&client, b"ccc", vec![c1, c2], false).expect("the third message"); + + let (bytes, handles) = recv(&server).expect("the first message"); + assert_eq!((bytes.as_slice(), handles.len()), (&b"a"[..], 1), "the first message came cut or with another's handles"); + drop(handles); + assert!(gone(&a_seen), "the first message's handle is not the one it carried"); + let (bytes, handles) = recv(&server).expect("the second message"); + assert_eq!((bytes.as_slice(), handles.len()), (&b"bb"[..], 0), "the second message came cut or with another's handles"); + let (bytes, mut handles) = recv(&server).expect("the third message"); + assert_eq!((bytes.as_slice(), handles.len()), (&b"ccc"[..], 2), "the third message came cut or with another's handles"); + drop(handles.remove(0)); + assert!(gone(&c1_seen), "the third message's handles came out of order"); + drop(handles); + assert!(gone(&c2_seen), "the third message's second handle is not the one it carried"); + assert_eq!(recv(&server).err(), Some(SyscallError::WouldBlock), "a fourth message where three were sent"); +} + +fn a_short_buffer_leaves_it_queued((client, server): (Connection, Connection)) { + let (h1, _) = token(); + let (h2, _) = token(); + send(&client, &[7; 8], vec![h1, h2], false).expect("a message of 8 bytes and two handles"); + let raw = server.as_handle().raw(); + let mut handles = [RawHandle(0); MAX_TRANSFER_HANDLES]; + assert_eq!( + syscall::message_recv(raw, &mut [0; 7], &mut handles, true), + Err(SyscallError::InvalidArgument), + "a buffer a byte short" + ); + assert_eq!( + syscall::message_recv(raw, &mut [0; 8], &mut handles[..1], true), + Err(SyscallError::InvalidArgument), + "a handle buffer one short" + ); + let mut bytes = [0u8; 8]; + let got = syscall::message_recv(raw, &mut bytes, &mut handles, true).expect("the message, whole"); + assert_eq!((got, bytes), (Received { len: 8, handles: 2 }, [7; 8]), "the refused receives took part of it"); + // SAFETY: installed for this call. + handles[..2].iter().for_each(|&h| drop(unsafe { OwnedHandle::from_raw(h) })); +} + +fn each_bound_refuses_the_send_past_it((client, server): (Connection, Connection)) { + for i in 0..MAX_QUEUED_MESSAGES { + send(&client, &[], vec![], true).unwrap_or_else(|e| panic!("message {i} refused {e:?} under the count bound")); + } + assert_eq!(send(&client, &[], vec![], true), Err(SyscallError::WouldBlock), "a message past the count bound"); + recv(&server).expect("one message taken"); + send(&client, &[], vec![], true).expect("room for one after one was taken"); + while recv(&server).is_ok() {} + + let mut seen = Vec::new(); + for _ in 0..MAX_QUEUED_HANDLES / MAX_TRANSFER_HANDLES { + let batch = (0..MAX_TRANSFER_HANDLES).map(|_| token()).map(|(h, s)| { + seen.push(s); + h + }); + send(&client, &[], batch.collect(), true).expect("handles under the bound"); + } + let (one, one_seen) = token(); + assert_eq!(send(&client, &[], vec![one], true), Err(SyscallError::WouldBlock), "a handle past the handle bound"); + assert!(gone(&one_seen), "the refused handle was not given back to close"); + send(&client, &[], vec![], true).expect("a message without handles waits on no handle room"); + while recv(&server).is_ok() {} + + let full = vec![0u8; MAX_MESSAGE_LEN]; + for i in 0..PIPE_CAPACITY / MAX_MESSAGE_LEN { + send(&client, &full, vec![], true).unwrap_or_else(|e| panic!("message {i} refused {e:?} under the byte bound")); + } + assert_eq!(send(&client, &[0], vec![], true), Err(SyscallError::WouldBlock), "a byte past the byte bound"); + recv(&server).expect("one full message taken"); + send(&client, &full, vec![], true).expect("room for one full message after one was taken"); + while recv(&server).is_ok() {} +} + +fn bytes_alone_are_refused((client, server): (Connection, Connection)) { + assert_eq!(syscall::write(client.as_handle().raw(), b"x"), Err(SyscallError::PermissionDenied), "a plain write"); + send(&client, b"y", vec![], false).expect("a message"); + assert_eq!(syscall::read(server.as_handle().raw(), &mut [0; 1]), Err(SyscallError::PermissionDenied), "a plain read"); + assert_eq!(recv(&server).map(|(b, _)| b), Ok(b"y".to_vec()), "the refused read took the message"); +} + +fn gone_after_what_was_sent((client, server): (Connection, Connection)) { + send(&client, b"last", vec![], false).expect("a message before leaving"); + drop(client); + assert_eq!(recv(&server).map(|(b, _)| b), Ok(b"last".to_vec()), "the message sent before leaving"); + assert_eq!(recv(&server).err(), Some(SyscallError::Gone), "after the last message, the leaving"); + assert_eq!(send(&server, b"x", vec![], false), Err(SyscallError::Gone), "a send to a peer gone"); +} diff --git a/tests/guest/rust-tests/src/bin/mutual_kill.rs b/tests/guest/rust-tests/src/bin/mutual_kill.rs index a0d71e92527..e45a71820f9 100644 --- a/tests/guest/rust-tests/src/bin/mutual_kill.rs +++ b/tests/guest/rust-tests/src/bin/mutual_kill.rs @@ -120,9 +120,10 @@ fn word(go: &SharedMemory) -> &AtomicU32 { fn killer() -> ! { let ns = endow::namespace().expect("the parent endowed a namespace"); let conn = ns.open(SERVICE).expect("connect through the endowed connector"); - let header = conn.recv_header().expect("the arming frame"); - assert_eq!(header.msg_type, ARM, "an unexpected frame"); - let [word_handle, victim] = conn.recv_handles_exact::<2>(&header).expect("the word and the victim"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("the arming frame"); + assert_eq!(frame.msg_type, ARM, "an unexpected frame"); + let [word_handle, victim] = rx.take_handles::<2>().expect("the word and the victim"); let go = SharedMemory::lent(word_handle, WORD_BYTES).expect("map the word"); let victim = Process::from_handle(victim); diff --git a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs index c504b944a14..5e7a91617c3 100644 --- a/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs +++ b/tests/guest/rust-tests/src/bin/netstack_connect_killed.rs @@ -91,11 +91,10 @@ fn until(wanted: impl Fn(&BTreeMap) -> bool, what: &str) { fn snapshot() -> BTreeMap { let conn = endow::service(NET.port).expect("the image gives test-runner netstack"); conn.signal(MSG_INSPECT).expect("the inspect request"); - let header = conn.recv_header().expect("netstack's answer"); - assert_eq!(header.msg_type, MSG_SNAPSHOT, "netstack answered something that is not a snapshot"); - let mut buf = vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut buf).expect("the snapshot's bytes"); - toyos_inspect::decode(&buf[..n], NET).expect("a snapshot for `net`") + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("netstack's answer"); + assert_eq!(frame.msg_type, MSG_SNAPSHOT, "netstack answered something that is not a snapshot"); + toyos_inspect::decode(rx.payload(frame.payload_len), NET).expect("a snapshot for `net`") } /// The places a snapshot says netstack holds. diff --git a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs index 5616f971b1e..8d8698215e6 100644 --- a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs +++ b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs @@ -38,10 +38,11 @@ //! `NetError::Io`, which is `ErrorKind::Other`, which is sshserver's `panic!`. //! //! **Nothing here waits.** The child's ordering edge is a *second* connection -//! to the same port, read with a blocking read: `Acceptor::on_zero_handles` -//! closes every queued connection's inbox and only then drops the connections, -//! so a read that answers `0` on one of them proves the other one's outbox is -//! already closed. The hook is deferred to a drain site +//! to the same port, taken with a blocking receive: `Acceptor::on_zero_handles` +//! drops the queued connections in the order they were made, each one's +//! receiving end before its sending end, so a receive that answers `Gone` on +//! the second proves the first can no longer be sent to. The hook is deferred +//! to a drain site //! (`kernel/src/object/mod.rs`'s `ZERO_QUEUE`), so a sleep would be a guess and //! this is not one. @@ -53,7 +54,7 @@ use std::process::{Child, ChildStdout, Command, Stdio}; use toyos::endow::EndowError; use toyos::net::{BoundResponse, MsgType, NetError, NetstackConn, TcpConnectRequest, TcpListenRequest, TcpOptions}; use toyos::{namespace, port, AsHandle}; -use toyos_abi::syscall::{self, SVC_LABEL}; +use toyos_abi::syscall::{self, SyscallError, SVC_LABEL}; const SELF_PATH: &str = "/system/bin/test_rs_netstack_gone_mid_bind"; @@ -190,8 +191,8 @@ fn mid_flight(request: Request) -> ! { say("connected"); let mut byte = [0u8; 1]; - let n = syscall::read(watch.as_handle().raw(), &mut byte).expect("read the watch connection"); - assert_eq!(n, 0, "the watch connection carried {n} byte(s); nobody ever wrote to it"); + let taken = syscall::message_recv(watch.as_handle().raw(), &mut byte, &mut [], false); + assert_eq!(taken, Err(SyscallError::Gone), "the watch connection carried a message; nobody ever sent one"); let err = match request { Request::Handle => { diff --git a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs index 98c4d6984e8..f4e3b14b762 100644 --- a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs +++ b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs @@ -105,8 +105,9 @@ fn test() { let (reader, writer) = pipe(""); client.signal_with(1, [OwnedHandle::from(writer)]).expect("the write end into the connection's queue"); held(&reader, "its handle sits unreceived in a connection's queue"); - let header = server.recv_header().expect("the frame that carried it"); - let [received] = server.recv_handles_exact::<1>(&header).expect("the handle the queue held"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + rx.recv(&server).expect("the frame that carried it"); + let [received] = rx.take_handles::<1>().expect("the handle the queue held"); held(&reader, "its handle was received"); drop(received); assert_eq!(at_last(&reader), Ok(OTHER_END_GONE), "the received handle closed"); diff --git a/tests/guest/rust-tests/src/bin/process_tree.rs b/tests/guest/rust-tests/src/bin/process_tree.rs index 28c3a7671d3..29c6ebc21d0 100644 --- a/tests/guest/rust-tests/src/bin/process_tree.rs +++ b/tests/guest/rust-tests/src/bin/process_tree.rs @@ -146,9 +146,10 @@ fn grow() -> Grown { .expect("spawn B"); let held = b.stdin.take().expect("B's input"); let conn = acceptor.accept().expect("B connected back"); - let header = conn.recv_header().expect("B's word that its subtree is grown"); - assert_eq!(header.msg_type, MSG_GROWN, "B said something else"); - let [c, d, b_self, e] = conn.recv_handles_exact::<4>(&header).expect("B sent four handles"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("B's word that its subtree is grown"); + assert_eq!(frame.msg_type, MSG_GROWN, "B said something else"); + let [c, d, b_self, e] = rx.take_handles::<4>().expect("B sent four handles"); let (c, d, e) = (Process::from_handle(c), Process::from_handle(d), Process::from_handle(e)); Grown { b, conn, c, d, b_self, e, _held: held } } @@ -408,8 +409,9 @@ fn b() -> ! { ]; conn.signal_with(MSG_GROWN, handles).expect("send A the subtree"); - let header = conn.recv_header().expect("A's word to fault"); - assert_eq!(header.msg_type, MSG_FAULT, "A said something else"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("A's word to fault"); + assert_eq!(frame.msg_type, MSG_FAULT, "A said something else"); // SAFETY: a volatile write to address 0, which no region of this process // covers: it faults, and the kernel ends this process before anything // after it runs. diff --git a/tests/guest/rust-tests/src/bin/shm_release_reclaims.rs b/tests/guest/rust-tests/src/bin/shm_release_reclaims.rs index 89a4c2a9fb7..85514f3bf66 100644 --- a/tests/guest/rust-tests/src/bin/shm_release_reclaims.rs +++ b/tests/guest/rust-tests/src/bin/shm_release_reclaims.rs @@ -83,8 +83,9 @@ fn main() { .expect("spawn donor"); let conn = acceptor.accept().expect("the donor connects"); - let header = conn.recv_header().expect("the donor announces its region"); - let [sent] = conn.recv_handles_exact::<1>(&header).expect("the donor sent the region with the frame"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + rx.recv(&conn).expect("the donor announces its region"); + let [sent] = rx.take_handles::<1>().expect("the donor sent the region with the frame"); // Only now, after the donor has let go — its line says so. let mut out = BufReader::new(donor.stdout.take().expect("donor stdout")); diff --git a/tests/guest/rust-tests/src/bin/spawn_lands_claimed.rs b/tests/guest/rust-tests/src/bin/spawn_lands_claimed.rs index 0d9522e5154..0312a5042c7 100644 --- a/tests/guest/rust-tests/src/bin/spawn_lands_claimed.rs +++ b/tests/guest/rust-tests/src/bin/spawn_lands_claimed.rs @@ -53,9 +53,10 @@ fn test() { .spawn() .expect("spawn the place"); let conn = acceptor.accept().expect("the place connected back"); - let header = conn.recv_header().expect("the place's word"); - assert_eq!(header.msg_type, MSG_SELF, "the place said something else"); - let [place_self] = conn.recv_handles_exact::<1>(&header).expect("the place sent its self"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("the place's word"); + assert_eq!(frame.msg_type, MSG_SELF, "the place said something else"); + let [place_self] = rx.take_handles::<1>().expect("the place sent its self"); assert!(place.try_wait().expect("ask after the place").is_none(), "the place ended before the marked spawn"); assert_eq!( diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs index 1821e52ef62..4840a6e09f0 100644 --- a/tests/guest/rust-tests/src/bin/stream_handles.rs +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -227,8 +227,8 @@ fn refuses_a_foreign_handle_writing_nothing() { let stderr = String::from_utf8_lossy(&said.stderr); assert!(!said.status.success(), "the foreign handle was taken: {}", String::from_utf8_lossy(&said.stdout)); assert!(stderr.contains("is no stream of this process's"), "refused, but not by name: {stderr}"); - let read = syscall::read(conn.as_handle().raw(), &mut bytes); - assert_eq!(read, Ok(0), "the refusal wrote on the stream's connection before its end"); + let taken = syscall::message_recv(conn.as_handle().raw(), &mut bytes, &mut [], false); + assert_eq!(taken, Err(SyscallError::Gone), "the refusal sent on the stream's connection before its end"); assert_eq!(rung.try_take(), Err(Hungup), "the refusal rang the stream before its end"); println!(" a foreign handle: refused by name, nothing written"); } @@ -504,10 +504,11 @@ fn run(role: &str, peer: &[u8]) -> (Child, Netstack) { .spawn() .unwrap_or_else(|e| panic!("spawn {role}: {e}")); let conn = acceptor.accept().expect("the child's connect"); - let header = conn.recv_header().expect("a request"); - assert_eq!(header.msg_type, MsgType::TcpConnect as u32, "a stream's connect"); - let _: TcpConnectRequest = conn.recv_payload(&header).expect("the connect's request"); - let [region] = conn.recv_handles_exact::<1>(&header).expect("the stream's region"); + let mut rx = toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new(); + let frame = rx.recv(&conn).expect("a request"); + assert_eq!(frame.msg_type, MsgType::TcpConnect as u32, "a stream's connect"); + let _: TcpConnectRequest = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the connect's request"); + let [region] = rx.take_handles::<1>().expect("the stream's region"); let region = SharedMemory::lent(region, REGION_BYTES).expect("a region netstack takes"); let mut server = Server::new(words_of(®ion), LAYOUT); region.bytes().write(RECEIVE_RING, peer); diff --git a/tests/guest/rust-tests/src/bin/window_refusal.rs b/tests/guest/rust-tests/src/bin/window_refusal.rs index d7dd81de1b8..0741c0eab3e 100644 --- a/tests/guest/rust-tests/src/bin/window_refusal.rs +++ b/tests/guest/rust-tests/src/bin/window_refusal.rs @@ -108,10 +108,11 @@ fn server() { fn serve_one(acceptor: &Acceptor, reply: Reply) { let accepted = acceptor.accept().expect("accept a client"); let handle = accepted.as_handle(); - let header = ipc::recv_header(handle).expect("request header"); - assert_eq!(header.msg_type, window::MSG_CREATE_WINDOW, "client sent the wrong request"); + let mut rx = ipc::FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(&handle).expect("the request"); + assert_eq!(frame.msg_type, window::MSG_CREATE_WINDOW, "client sent the wrong request"); let _req: window::CreateWindowRequest = - ipc::recv_payload(handle, &header).expect("request payload"); + ipc::decode_payload(rx.payload(frame.payload_len)).expect("request payload"); match reply { Reply::Refuse(reason) => { ipc::send(handle, window::MSG_WINDOW_REFUSED, &window::WindowRefused { reason }) diff --git a/tests/guest/rust-tests/src/master.rs b/tests/guest/rust-tests/src/master.rs index d0339d047fe..6a97012ae86 100644 --- a/tests/guest/rust-tests/src/master.rs +++ b/tests/guest/rust-tests/src/master.rs @@ -8,13 +8,14 @@ pub fn connect() -> toyos::Connection { } fn answer(volume: &toyos::Connection) -> Volume { - let header = volume.recv_header().expect("soundserver answers the request"); + let mut rx = toyos::ipc::FrameRx::<{ core::mem::size_of::() }>::new(); + let frame = rx.recv(volume).expect("soundserver answers the request"); assert_ne!( - header.msg_type, MSG_VOLUME_UNAVAILABLE, + frame.msg_type, MSG_VOLUME_UNAVAILABLE, "soundserver has no master to set: it plays to no audio device with one" ); - assert_eq!(header.msg_type, MSG_VOLUME, "soundserver answered with message {}", header.msg_type); - let read: Volume = volume.recv_payload(&header).expect("the answer is a volume"); + assert_eq!(frame.msg_type, MSG_VOLUME, "soundserver answered with message {}", frame.msg_type); + let read: Volume = toyos::ipc::decode_payload(rx.payload(frame.payload_len)).expect("the answer is a volume"); println!("master: position {} of {}, muted {}", read.position, read.bottom, read.muted); read } diff --git a/tests/guest/rust-tests/src/usbd_spare.rs b/tests/guest/rust-tests/src/usbd_spare.rs index a002022b07e..e3261dba3d0 100644 --- a/tests/guest/rust-tests/src/usbd_spare.rs +++ b/tests/guest/rust-tests/src/usbd_spare.rs @@ -143,11 +143,10 @@ impl Usbd { let ns = toyos::namespace::build().add(USB.port, &self.conn).finish().expect("usbd_spare: a namespace"); let conn: Connection = ns.open(USB.port).expect("usbd_spare: a connection to usbd"); conn.signal(MSG_INSPECT).expect("usbd_spare: the request"); - let header = conn.recv_header().expect("usbd_spare: usbd's answer"); - assert_eq!(header.msg_type, MSG_SNAPSHOT, "usbd_spare: usbd answered something that is not a snapshot"); - let mut buf = vec![0u8; header.len() as usize]; - let n = conn.recv_bytes(&header, &mut buf).expect("usbd_spare: the snapshot's bytes"); - let snapshot = toyos_inspect::decode(&buf[..n], USB).expect("usbd_spare: a snapshot for `usb`"); + let mut rx = Box::new(toyos::ipc::FrameRx::<{ toyos::ipc::MAX_FRAME_LEN as usize }>::new()); + let frame = rx.recv(&conn).expect("usbd_spare: usbd's answer"); + assert_eq!(frame.msg_type, MSG_SNAPSHOT, "usbd_spare: usbd answered something that is not a snapshot"); + let snapshot = toyos_inspect::decode(rx.payload(frame.payload_len), USB).expect("usbd_spare: a snapshot for `usb`"); for (path, value) in &snapshot { println!("usbd_spare: {when}: {}", toyos_inspect::line(path, value)); } From ab0209bca719012eb2bf8d2b9d29c49db3890afd Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 02:53:39 +0100 Subject: [PATCH 2/5] tests: session_lifecycle detaches its holder once it listens The job detached hangupholder through a shell and took it as ready once the roster listed it. A process that has not yet listened for its quits is ended by the first one, so a stop asked in that window killed the holder before its first syscall, the stop found nothing to wait for, and the machine rebooted under the job: the run stalled at "the job to reach the forced reboot". Whole-message connections made the job's own requests fast enough to land in that window: red at 343a655b1 in both runs, where the base, 18060b521, passed. A second red, the forced reboot's last lines missing from the console, is the base's too, and is recorded on its own (issues/a-forced-reboots-last-userland-lines-miss-the-console.md). The job now detaches the holder through the session's launcher itself, with its stdout a pipe, and waits for the line the holder says once it listens; sessioncase's test-runner row lists hangupholder in place of the shell, which the job no longer starts. Keeping the shell for the first detach was tried: that run reached the same window at the sign-out, the holder killed after five syscalls. The session still parents what is detached in it, which is the claim; a detached program outliving its shell is process_tree's. Co-Authored-By: Claude Opus 5.5 --- .../rust-tests/src/bin/session_lifecycle.rs | 29 +++++++++++-------- tests/images/sessioncase.toml | 11 +++---- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/session_lifecycle.rs b/tests/guest/rust-tests/src/bin/session_lifecycle.rs index f7900f90b62..001c5e5c8bd 100644 --- a/tests/guest/rust-tests/src/bin/session_lifecycle.rs +++ b/tests/guest/rust-tests/src/bin/session_lifecycle.rs @@ -7,6 +7,7 @@ //! ends the job. `hangupholder` is the app that ignores every quit. use std::os::toyos::process::CommandExt; +use std::io::{BufRead, BufReader}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; @@ -43,10 +44,9 @@ fn main() { }; let logout = || String::from_utf8_lossy(&launched(&admin, "/system/bin/logout", &[]).stdout).into_owned(); - // A detached app outlives the shell that started it. The shell's output - // is nobody's, since the holder it detaches keeps what it inherits open. + // An app detached in the session runs there, listening for its quits. let holder = detach(&launcher); - println!("session_lifecycle: a detached app (pid {holder}) outlives its shell"); + println!("session_lifecycle: a detached app (pid {holder}) runs in the session"); // What `logout` lists active now, among which this job's session is. let active: Vec = logout() @@ -113,22 +113,27 @@ fn main() { panic!("session_lifecycle: the forced reboot ended {:?}", pending.outcome()); } -/// `hangupholder` detached in `launcher`'s session by a shell that has ended: -/// its pid. +/// `hangupholder` detached in `launcher`'s session, once it listens for its +/// quits: its pid. Until it listens a quit ends it, so a stop asked before +/// that would find nothing to wait for. fn detach(launcher: &Namespace) -> u32 { let before = live(HOLDER); - let shell = Command::new("/system/bin/shell") + let mut holder = Command::new(format!("/system/bin/{HOLDER}")) .independent() + .detached() .launcher(launcher.as_handle().into()) - .args(["-c", &format!("detach /system/bin/{HOLDER}")]) .stdin(Stdio::null()) - .stdout(Stdio::null()) + .stdout(Stdio::piped()) .stderr(Stdio::null()) .current_dir("/") - .status() - .expect("launch a shell in the session"); - assert!(shell.success(), "the shell's detach ended {shell:?}"); - live(HOLDER).into_iter().find(|pid| !before.contains(pid)).expect("the detached holder runs once its shell has ended") + .spawn() + .expect("detach the holder in the session"); + let mut said = String::new(); + BufReader::new(holder.stdout.take().expect("the holder's stdout")) + .read_line(&mut said) + .expect("the holder's first line"); + assert_eq!(said.trim_end(), "hangupholder: holding", "the holder said something else first"); + live(HOLDER).into_iter().find(|pid| !before.contains(pid)).expect("the detached holder runs") } /// The stop's lists until one names the holder: the first may be read before diff --git a/tests/images/sessioncase.toml b/tests/images/sessioncase.toml index 90307fb1230..6a7b092f845 100644 --- a/tests/images/sessioncase.toml +++ b/tests/images/sessioncase.toml @@ -7,9 +7,10 @@ # out, which Escape cancels and Enter forces. # # test-runner's row is `login`, so its job opens sessions with its `login` -# connector, and lists what a session's launcher starts: a shell, which -# detaches `hangupholder`, the app that ignores a hang-up; and `logout`, the -# one row receiving `session-admin`. `session` owns each session opened. +# connector, and lists what a session's launcher starts: `hangupholder`, the +# app that ignores a hang-up, which the job detaches with its stdout a pipe, +# to hear that it listens; and `logout`, the one row receiving +# `session-admin`. `session` owns each session opened. # test-runner receives `power` for `session_lifecycle`'s closing reboot, which # waits for the sessions until the job forces it. @@ -22,12 +23,12 @@ start = ["logkeeper", "compositor", "ps2server", "test-runner"] service = true syscap = ["logread"] -# `roster`, which the job reads to see the holder outlive its shell and end +# `roster`, which the job reads to find the holder it detached and see it end # when forced; `dup`, because test-runner hands each job its capability. [programs.test-runner] syscap = ["dup", "roster"] receives = ["power"] -starts = ["shell", "logout"] +starts = ["hangupholder", "logout"] login = true [programs.compositor] From 8aa8ccf2b7513872efee551671c418ed81d14c65 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 02:53:39 +0100 Subject: [PATCH 3/5] issues: a forced reboot's last userland lines can miss the console session_lifecycle's remaining red: the job's last line and the supervisor's forced-stop line, written just before the forced reboot, sometimes never reach the serial console, while the kernel's records of the forced stop do. It is red at #1108 18060b521 too (1 of 4 runs on the remote QEMU runner), so it predates whole messages, which make it likelier (3 of 6 at e396bb5f1) by shortening the time from the terminate to the stop. Co-Authored-By: Claude Opus 5.5 --- ...ts-last-userland-lines-miss-the-console.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 issues/a-forced-reboots-last-userland-lines-miss-the-console.md diff --git a/issues/a-forced-reboots-last-userland-lines-miss-the-console.md b/issues/a-forced-reboots-last-userland-lines-miss-the-console.md new file mode 100644 index 00000000000..98f74d3fe7d --- /dev/null +++ b/issues/a-forced-reboots-last-userland-lines-miss-the-console.md @@ -0,0 +1,33 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A forced reboot's last userland lines can miss the serial console + +`session_lifecycle` (`tests/common/sessionlifecycle.rs`) forces a waiting +reboot and then waits on the console for two userland lines written just +before it: the job's `session_lifecycle: ok, and the reboot is forced`, said +the statement before its terminate, and the supervisor's `power: the stop ends +what stays`. In some runs one of them never reaches the wire. The kernel's own +records still do: the forced stop's kills of the holder and its session, and +`Rebooting.`. So the stop did happen as the job asked, and only the lines are +lost. The harness then reports `STALLED: waiting for the job to reach the +forced reboot`, or `the supervisor never said what the forced reboot ended`. + +**Measured** on the remote QEMU x86-64 runner, `cargo test --test +toyos-build -- session_lifecycle`, 2026-11-01: +- #1108 `18060b521` (byte-stream connections): 1 red of 4. +- #1121 `e396bb5f1` (whole-message connections): 3 red of 6 so far. + +Faster connections shorten the time from the terminate to the stop, so the +window opens more often, but it is there without them. + +Where the lines go between a program's log ring and the wire at a stop +(`logkeeper`'s flush answers the supervisor before the stop goes on; +`console: klogd let the wire go to the stop`) is not yet traced. + +**Exit**: every userland line written before a forced stop's request reaches +the console before the machine resets, and `session_lifecycle` is green 20 +runs in a row on the runner. From aadce7fab13413fb0455793b443d3a985feeb3a0 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 03:17:35 +0100 Subject: [PATCH 4/5] a receive takes a message straight into the caller's buffer Whole messages put a copy back into every receive whose caller keeps the payload itself. The kernel copied a frame into the receiver's FrameRx, and the caller then copied it out again, where the byte stream had read straight into the caller's buffer. osbench's ipc.reqrep rows pay it per frame. On the AMD they regressed: 7.6 -> 9.5 us at 4096 bytes and 31 -> 66 us at 65536, with one park per side per round trip and every pair inside one LLC (the diary-counting batch on ipc-messages-ab-base/-h1). The only reason for the intermediate buffer was the frame's 4-byte message type in front of its payload. SYS_MESSAGE_RECV now takes two places: the message's first bytes fill `head` and the rest `bytes`, and a message longer than both together is refused whole and stays queued, as before. The ABI gives the two places no meaning. The SDK receives the type into a word and the payload where the caller keeps it: FrameRx holds the two apart, Connection::recv_into takes a payload into the caller's own buffer with its handles, and osbench's peer and toyos::net's response_bytes use it, with no copy out. The send side is unchanged. It frames type and payload into one buffer, as it did before whole messages. Co-Authored-By: Claude Opus 5.5 --- kernel/src/message.rs | 17 ++-- kernel/src/syscall/dispatch.rs | 10 ++- kernel/src/syscall/ipc.rs | 4 +- sdk/abi/src/syscall.rs | 33 ++++--- sdk/toyos/src/ipc.rs | 90 +++++++++++-------- sdk/toyos/src/net.rs | 18 ++-- tests/guest/osbench/src/sys/toyos.rs | 35 +++----- .../src/bin/connect_before_serve.rs | 6 +- .../rust-tests/src/bin/messages_are_whole.rs | 22 +++-- .../src/bin/netstack_gone_mid_bind.rs | 2 +- .../rust-tests/src/bin/stream_handles.rs | 2 +- 11 files changed, 138 insertions(+), 101 deletions(-) diff --git a/kernel/src/message.rs b/kernel/src/message.rs index beaabfc78ed..206d0769e97 100644 --- a/kernel/src/message.rs +++ b/kernel/src/message.rs @@ -162,21 +162,24 @@ impl QueueRef { Ok(()) } - /// Take the front message into `buf` and at most `cap` handles, each - /// installed in `table` under this queue's lock. - pub fn recv(&self, buf: &mut UserBytesMut, cap: usize, table: &mut HandleTable) -> Taken { + /// Take the front message into `head` and then `body`, its first bytes + /// filling `head`, and at most `cap` handles, each installed in `table` + /// under this queue's lock. + pub fn recv(&self, head: &mut UserBytesMut, body: &mut UserBytesMut, cap: usize, table: &mut HandleTable) -> Taken { let (taken, boost) = { let mut guard = self.0.state.lock(); let State { bytes, queue, holders, rt_boost_pending } = &mut *guard; // Bound, not the block's tail: a tail match's temporaries outlive `guard`. - let taken = match queue.pop(buf.len(), cap, |width| table.make_room(width)) { - Some(Ok((head, handles))) => { + let taken = match queue.pop(head.len() + body.len(), cap, |width| table.make_room(width)) { + Some(Ok((message, handles))) => { let mut installed = [RawHandle(0); MAX_TRANSFER_HANDLES]; for (slot, entry) in installed.iter_mut().zip(handles) { *slot = table.install(entry).expect("room was made first"); } - bytes.read(buf, head.len); - (Taken::Message(head, installed), core::mem::take(rt_boost_pending)) + let first = message.len.min(head.len()); + bytes.read(head, first); + bytes.read(body, message.len - first); + (Taken::Message(message, installed), core::mem::take(rt_boost_pending)) } Some(Err(untaken)) => (Taken::Untaken(untaken), false), None if holders.held(End::Write) => (Taken::Empty, false), diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 29edbd62adc..b3230793673 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -692,17 +692,19 @@ fn outgoing<'a>(ctx: &SyscallContext<'a>, args: &MessageSend) -> Result(ctx: &SyscallContext<'a>, args: &MessageRecv) -> Result>, ()> { if args.flags & !MESSAGE_NONBLOCK != 0 { return Err(()); } - let len = args.len.min(MAX_MESSAGE_LEN as u64); + let head_len = args.head_len.min(MAX_MESSAGE_LEN as u64); + let len = args.len.min(MAX_MESSAGE_LEN as u64 - head_len); let cap = (args.cap as usize).min(MAX_TRANSFER_HANDLES); Ok((|| { Some(Incoming { + head: ctx.user_bytes_mut(UserAddr::new(args.head), head_len)?, bytes: ctx.user_bytes_mut(UserAddr::new(args.bytes), len)?, handles: ctx.user_bytes_mut(UserAddr::new(args.handles), (cap * HANDLE_LEN) as u64)?, cap, diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 9390a3d02ac..0d54af17cc7 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -411,6 +411,8 @@ pub(super) struct Outgoing<'a> { /// A receive's buffers as the dispatch bounded them. pub(super) struct Incoming<'a> { + /// Where the message's first bytes go, before [`Self::bytes`]. + pub head: crate::user_ptr::UserBytesMut<'a>, pub bytes: crate::user_ptr::UserBytesMut<'a>, pub handles: crate::user_ptr::UserBytesMut<'a>, pub cap: usize, @@ -487,7 +489,7 @@ fn recv(conn_h: RawHandle, into: &mut Incoming<'_>, nonblock: bool) -> u64 { let taken = process::with_process_data(|data| { let conn = data.handles.get::(conn_h, Rights::READ)?; let rx = conn.rx().clone(); - let taken = rx.recv(&mut into.bytes, into.cap, &mut data.handles); + let taken = rx.recv(&mut into.head, &mut into.bytes, into.cap, &mut data.handles); Ok::<_, crate::object::Refusal>((rx, taken)) }); let rx = match taken { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 0e7691b7641..b8c60cdba3e 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2170,9 +2170,12 @@ pub unsafe fn message_send(conn: RawHandle, bytes: &[u8], handles: &[RawHandle], crate::user_safe! { /// Arguments for `SYS_MESSAGE_RECV`, passed as a single pointer: where - /// the message's bytes and handles go. + /// the message's bytes go, its first `head_len` to `head` and the rest to + /// `bytes`, and where its handles go. #[derive(Clone, Copy)] pub struct MessageRecv { + pub head: u64, + pub head_len: u64, pub bytes: u64, pub len: u64, pub handles: u64, @@ -2183,8 +2186,10 @@ crate::user_safe! { } impl MessageRecv { - fn new(bytes: &mut [u8], handles: &mut [RawHandle], nonblock: bool) -> Self { + fn new(head: &mut [u8], bytes: &mut [u8], handles: &mut [RawHandle], nonblock: bool) -> Self { Self { + head: head.as_mut_ptr() as u64, + head_len: head.len() as u64, bytes: bytes.as_mut_ptr() as u64, len: bytes.len() as u64, handles: handles.as_mut_ptr() as u64, @@ -2215,17 +2220,25 @@ impl Received { const _: () = assert!(Received::encode(MAX_MESSAGE_LEN, MAX_TRANSFER_HANDLES) < u64::MAX - 255); -/// Take `conn`'s next message whole: its bytes into `bytes` and its handles, -/// each installed in this process's table, into `handles`. +/// Take `conn`'s next message whole: its first bytes into `head`, the rest +/// into `bytes`, and its handles, each installed in this process's table, +/// into `handles`. Two places, so a protocol's own header and the payload +/// behind it each land where the caller keeps them, with no copy between. /// /// Waits for one unless `nonblock`, which answers `WouldBlock`. Once the peer /// has gone and nothing is queued, `Gone`. **A message is never cut**: one -/// larger than `bytes` or carrying more handles than `handles` holds is -/// `InvalidArgument`, and a table without room for its handles -/// `ResourceExhausted`, both leaving it queued. A protocol bounds its own -/// messages, so either is the peer breaking it. -pub fn message_recv(conn: RawHandle, bytes: &mut [u8], handles: &mut [RawHandle], nonblock: bool) -> Result { - let args = MessageRecv::new(bytes, handles, nonblock); +/// larger than `head` and `bytes` together, or carrying more handles than +/// `handles` holds, is `InvalidArgument`, and a table without room for its +/// handles `ResourceExhausted`, both leaving it queued. A protocol bounds its +/// own messages, so either is the peer breaking it. +pub fn message_recv( + conn: RawHandle, + head: &mut [u8], + bytes: &mut [u8], + handles: &mut [RawHandle], + nonblock: bool, +) -> Result { + let args = MessageRecv::new(head, bytes, handles, nonblock); check(syscall(SYS_MESSAGE_RECV, conn.0 as u64, &args as *const MessageRecv as u64, 0, 0)).map(Received::decode) } diff --git a/sdk/toyos/src/ipc.rs b/sdk/toyos/src/ipc.rs index cb44491c914..8bb36c33b66 100644 --- a/sdk/toyos/src/ipc.rs +++ b/sdk/toyos/src/ipc.rs @@ -90,21 +90,14 @@ const TYPE_LEN: usize = 4; const _: () = assert!(TYPE_LEN + MAX_FRAME_LEN as usize <= syscall::MAX_MESSAGE_LEN); -/// A frame as the kernel carries it: the message type, then its payload. -/// Bytes only, so no padding. +/// A frame as a send lays it out for the kernel: the message type, then +/// its payload. Bytes only, so no padding. #[repr(C)] struct Wire

{ msg_type: [u8; TYPE_LEN], payload: P, } -impl Wire<[u8; MAX]> { - fn bytes_mut(&mut self) -> &mut [u8] { - // SAFETY: a `Wire` of initialised bytes, whole. - unsafe { core::slice::from_raw_parts_mut(self as *mut Self as *mut u8, core::mem::size_of::()) } - } -} - #[derive(Debug)] pub enum IpcError { Disconnected, @@ -241,6 +234,16 @@ impl Connection { message(self.as_handle(), msg_type, &[], handles, true).map_err(nonblocking) } + + /// Wait for the next frame and take its payload straight into + /// `payload`, with the handles it carried. For a caller that keeps the + /// payload in its own buffer: a [`FrameRx`] would be one more copy. A + /// payload longer than `payload` is [`IpcError::Malformed`] and stays + /// queued; otherwise it fails as [`FrameRx::recv`] does. + pub fn recv_into(&self, payload: &mut [u8]) -> Result<(Frame, Batch), IpcError> { + let mut msg_type = [0; TYPE_LEN]; + receive(self.as_handle(), &mut msg_type, payload, false).map_err(Refused::blocking) + } } // Free functions, for a connection a caller holds by something other than a @@ -427,7 +430,8 @@ pub struct Frame { /// [`pump`]: Self::pump /// [`recv`]: Self::recv pub struct FrameRx { - wire: Wire<[u8; MAX]>, + msg_type: [u8; TYPE_LEN], + payload: [u8; MAX], held: Batch, } @@ -459,12 +463,12 @@ impl Refused { impl FrameRx { pub const fn new() -> Self { - Self { wire: Wire { msg_type: [0; TYPE_LEN], payload: [0; MAX] }, held: Batch::EMPTY } + Self { msg_type: [0; TYPE_LEN], payload: [0; MAX], held: Batch::EMPTY } } /// The payload of the frame the last [`RxStep::Frame`] announced. pub fn payload(&self, len: usize) -> &[u8] { - &self.wire.payload[..len] + &self.payload[..len] } /// The handles the last frame carried, now the caller's to claim. @@ -494,25 +498,38 @@ impl FrameRx { fn take(&mut self, conn: BorrowedHandle<'_>, nonblock: bool) -> Result { self.held = Batch::EMPTY; - let mut handles = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; - let got = syscall::message_recv(conn.raw(), self.wire.bytes_mut(), &mut handles, nonblock); - self.taken(got, handles) + let (frame, batch) = receive(conn, &mut self.msg_type, &mut self.payload, nonblock)?; + self.held = batch; + Ok(frame) } +} - /// A receive's answer, the handles it installed now this receiver's. - fn taken( - &mut self, - got: Result, - handles: [RawHandle; syscall::MAX_TRANSFER_HANDLES], - ) -> Result { - let got = got.map_err(Refused::Kernel)?; - // The kernel installed each number in this process's table for this - // call, so this batch is the only thing that answers for them. - self.held = Batch { handles, next: 0, len: got.handles }; - let payload_len = got.len.checked_sub(TYPE_LEN).ok_or(Refused::Malformed)?; - let msg_type = u32::from_ne_bytes(self.wire.msg_type); - Ok(Frame { msg_type, payload_len, handles: got.handles }) - } +/// One frame off `conn`: its type into `msg_type` and its payload into +/// `payload`, straight from the kernel, and the handles it carried. +fn receive( + conn: BorrowedHandle<'_>, + msg_type: &mut [u8; TYPE_LEN], + payload: &mut [u8], + nonblock: bool, +) -> Result<(Frame, Batch), Refused> { + let mut handles = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; + let got = syscall::message_recv(conn.raw(), msg_type, payload, &mut handles, nonblock); + taken(got, *msg_type, handles) +} + +/// A receive's answer, the handles it installed now the caller's. +fn taken( + got: Result, + msg_type: [u8; TYPE_LEN], + handles: [RawHandle; syscall::MAX_TRANSFER_HANDLES], +) -> Result<(Frame, Batch), Refused> { + let got = got.map_err(Refused::Kernel)?; + // The kernel installed each number in this process's table for this + // call, so this batch is the only thing that answers for them: dropped + // with a malformed frame, it closes them. + let batch = Batch { handles, next: 0, len: got.handles }; + let payload_len = got.len.checked_sub(TYPE_LEN).ok_or(Refused::Malformed)?; + Ok((Frame { msg_type: u32::from_ne_bytes(msg_type), payload_len, handles: got.handles }, batch)) } /// What a non-blocking receive's answer is to a server. @@ -661,12 +678,11 @@ mod tests { #[test] fn a_message_shorter_than_a_type_is_malformed() { let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; - let mut rx = FrameRx::<8>::new(); - rx.wire.msg_type = 7u32.to_ne_bytes(); - assert!(matches!(rx.taken(Ok(Received { len: TYPE_LEN - 1, handles: 0 }), none), Err(Refused::Malformed))); - let frame = rx.taken(Ok(Received { len: TYPE_LEN, handles: 0 }), none).ok().unwrap(); + let seven = 7u32.to_ne_bytes(); + assert!(matches!(taken(Ok(Received { len: TYPE_LEN - 1, handles: 0 }), seven, none), Err(Refused::Malformed))); + let (frame, _) = taken(Ok(Received { len: TYPE_LEN, handles: 0 }), seven, none).ok().unwrap(); assert_eq!(frame, Frame { msg_type: 7, payload_len: 0, handles: 0 }); - let frame = rx.taken(Ok(Received { len: TYPE_LEN + 8, handles: 0 }), none).ok().unwrap(); + let (frame, _) = taken(Ok(Received { len: TYPE_LEN + 8, handles: 0 }), seven, none).ok().unwrap(); assert_eq!(frame.payload_len, 8); } @@ -674,14 +690,13 @@ mod tests { #[test] fn a_receive_refused_is_the_step_a_server_acts_on() { let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; - let mut rx = FrameRx::<0>::new(); for (e, idle, eof) in [ (SyscallError::WouldBlock, true, false), (SyscallError::Gone, false, true), (SyscallError::InvalidArgument, false, false), (SyscallError::ResourceExhausted, false, false), ] { - let step = step(rx.taken(Err(e), none)); + let step = step(taken(Err(e), [0; TYPE_LEN], none).map(|(frame, _)| frame)); assert_eq!(matches!(step, RxStep::Idle), idle, "{e:?} answered {step:?}"); assert_eq!(matches!(step, RxStep::Eof), eof, "{e:?} answered {step:?}"); } @@ -693,8 +708,7 @@ mod tests { #[test] fn a_receive_refused_is_the_error_a_client_acts_on() { let none = [toyos_abi::HANDLE_INVALID; syscall::MAX_TRANSFER_HANDLES]; - let mut rx = FrameRx::<0>::new(); - let mut blocking = |got| rx.taken(got, none).map_err(Refused::blocking); + let blocking = |got| taken(got, [0; TYPE_LEN], none).map(|(frame, _)| frame).map_err(Refused::blocking); assert!(matches!(blocking(Err(SyscallError::InvalidArgument)), Err(IpcError::Malformed))); assert!(matches!(blocking(Ok(Received { len: TYPE_LEN - 1, handles: 0 })), Err(IpcError::Malformed))); assert!(matches!(blocking(Err(SyscallError::Gone)), Err(IpcError::Disconnected))); diff --git a/sdk/toyos/src/net.rs b/sdk/toyos/src/net.rs index 0c8f7417739..6ba0b761b56 100644 --- a/sdk/toyos/src/net.rs +++ b/sdk/toyos/src/net.rs @@ -384,13 +384,19 @@ impl PendingResponse { crate::ipc::decode_payload(rx.payload(frame.payload_len)).map_err(hangup) } - /// The answer's bytes, as many as `buf` holds. + /// The answer's bytes, taken straight into `buf`: their length. An + /// answer longer than `buf` is [`NetError::Io`]. pub fn response_bytes(self, buf: &mut [u8]) -> Result { - let mut rx = FrameRx::<{ crate::ipc::MAX_FRAME_LEN as usize }>::new(); - let frame = answer(self.conn(), &mut rx)?; - let n = frame.payload_len.min(buf.len()); - buf[..n].copy_from_slice(&rx.payload(frame.payload_len)[..n]); - Ok(n) + let (frame, _) = self.conn().recv_into(buf).map_err(hangup)?; + let payload = &buf[..frame.payload_len]; + match frame.msg_type { + t if t == RespType::Error as u32 => { + let err: ErrorResponse = crate::ipc::decode_payload(payload).map_err(hangup)?; + Err(NetError::from_error_code(err.code)) + } + t if t == RespType::Result as u32 => Ok(frame.payload_len), + other => Err(NetError::Protocol(other)), + } } /// The answer, the `N` handles it hands over, and the connection, which diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 5c3891b66f4..1b38f178abc 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -84,10 +84,8 @@ pub mod ipc { use std::os::toyos::process::CommandExt; use std::process::{Child, Command, Stdio}; - use std::cell::RefCell; - use toyos::endow::{Endowments, FromHandle}; - use toyos::ipc::{Connection, FrameRx, MAX_FRAME_LEN}; + use toyos::ipc::{Connection, MAX_FRAME_LEN}; use toyos::port::{self, Acceptor}; use toyos::{OwnedHandle, Pipe}; @@ -98,7 +96,7 @@ pub mod ipc { /// One end of a connection. A message longer than one frame /// ([`MAX_FRAME_LEN`]) goes as consecutive frames: a ToyOS connection /// frames each message, and a frame carries at most that much. - pub struct Peer(Connection, RefCell>>); + pub struct Peer(Connection); /// What a message carries beside its bytes: one end of a pipe. pub type Token = Pipe; @@ -133,11 +131,11 @@ pub mod ipc { } } - /// One frame into the front of `buf`: the bytes it held. + /// One frame into the front of `buf`, straight from the kernel: the + /// bytes it held. fn take(&self, buf: &mut [u8]) -> io::Result { - let mut rx = self.1.borrow_mut(); - let frame = rx.recv(&self.0).map_err(|e| failed("receive", e))?; - Ok(copy(rx.payload(frame.payload_len), buf)) + let (frame, _) = self.0.recv_into(buf).map_err(|e| failed("receive", e))?; + Ok(frame.payload_len) } /// One frame of `data` (at most one frame's worth) and `token`, moved. @@ -146,26 +144,13 @@ pub mod ipc { } pub fn recv_with(&self, buf: &mut [u8]) -> io::Result { - let mut rx = self.1.borrow_mut(); - let frame = rx.recv(&self.0).map_err(|e| failed("receive", e))?; - copy(rx.payload(frame.payload_len), buf); - let [handle] = rx.take_handles::<1>().ok_or_else(|| io::Error::other("no handle came"))?; + let (_, handles) = self.0.recv_into(buf).map_err(|e| failed("receive", e))?; + let [handle] = handles.exact::<1>().ok_or_else(|| io::Error::other("no handle came"))?; // The peer sends only pipe ends. Ok(Pipe::from_handle(handle)) } } - /// As much of `from` as `to` holds, at its front. - fn copy(from: &[u8], to: &mut [u8]) -> usize { - let n = from.len().min(to.len()); - to[..n].copy_from_slice(&from[..n]); - n - } - - fn peer(conn: Connection) -> Peer { - Peer(conn, RefCell::new(Box::new(FrameRx::new()))) - } - pub fn token() -> io::Result { let (read, _write) = toyos::pipe_pair().map_err(|e| failed("pipe", e))?; Ok(read) @@ -182,14 +167,14 @@ pub mod ipc { .endow(LABEL, acceptor.into()) .stdin(Stdio::null()) .spawn()?; - Ok((child, peer(conn))) + Ok((child, Peer(conn))) } /// The child's end: the connection its parent made to the port it was /// endowed. pub fn served() -> io::Result { let acceptor: Acceptor = Endowments::get().take(LABEL).ok_or_else(|| io::Error::other("no port endowed"))?; - acceptor.accept().map(peer).map_err(|e| failed("accept", e)) + acceptor.accept().map(Peer).map_err(|e| failed("accept", e)) } /// A port this process both connects to and accepts on, for timing a diff --git a/tests/guest/rust-tests/src/bin/connect_before_serve.rs b/tests/guest/rust-tests/src/bin/connect_before_serve.rs index 61a45a8f2f9..cddec4b97f5 100644 --- a/tests/guest/rust-tests/src/bin/connect_before_serve.rs +++ b/tests/guest/rust-tests/src/bin/connect_before_serve.rs @@ -155,7 +155,7 @@ fn client() -> ! { say("sent"); let mut buf = [0u8; 64]; - let got = syscall::message_recv(conn.as_handle().raw(), &mut buf, &mut [], false); + let got = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut buf, &mut [], false); match expect.as_str() { "reply" => { let n = got.expect("the reply").len; @@ -181,13 +181,13 @@ fn server() -> ! { // The instruction after `accept`, and it must not wait: the client sent // before this process existed, so the message is queued already. let mut buf = [0u8; 64]; - let n = syscall::message_recv(conn.as_handle().raw(), &mut buf, &mut [], true) + let n = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut buf, &mut [], true) .expect("the client's message was not queued at the server's first look") .len; say(&format!("already here: {}", String::from_utf8_lossy(&buf[..n]))); send(&conn, ANSWER, false).expect("reply"); // Held until the client has taken the reply, which its own exit reports. - let _ = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut [], false); + let _ = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut [], &mut [], false); std::process::exit(0); } diff --git a/tests/guest/rust-tests/src/bin/messages_are_whole.rs b/tests/guest/rust-tests/src/bin/messages_are_whole.rs index 935c776d9ba..36c3d66a8a8 100644 --- a/tests/guest/rust-tests/src/bin/messages_are_whole.rs +++ b/tests/guest/rust-tests/src/bin/messages_are_whole.rs @@ -6,7 +6,8 @@ //! //! - Three messages arrive in order, each with its own bytes and handles. //! - A buffer a byte or a handle short is refused, and the message stays -//! queued to be taken whole. +//! queued to be taken whole; one taken into two places fills them in +//! order. //! - Each bound (256 messages, 64 handles, 2 MiB of bytes) refuses the send //! past it with nothing moved, and room comes back as messages are taken. //! - Plain reads and writes of a connection are refused. @@ -64,7 +65,7 @@ fn send(conn: &Connection, bytes: &[u8], handles: Vec, nonblock: bo fn recv(conn: &Connection) -> Result<(Vec, Vec), SyscallError> { let mut bytes = vec![0u8; MAX_MESSAGE_LEN]; let mut handles = [RawHandle(0); MAX_TRANSFER_HANDLES]; - let got = syscall::message_recv(conn.as_handle().raw(), &mut bytes, &mut handles, true)?; + let got = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut bytes, &mut handles, true)?; bytes.truncate(got.len); // SAFETY: the kernel installed these numbers in this process for this call. Ok((bytes, handles[..got.handles].iter().map(|&h| unsafe { OwnedHandle::from_raw(h) }).collect())) @@ -111,20 +112,31 @@ fn a_short_buffer_leaves_it_queued((client, server): (Connection, Connection)) { let raw = server.as_handle().raw(); let mut handles = [RawHandle(0); MAX_TRANSFER_HANDLES]; assert_eq!( - syscall::message_recv(raw, &mut [0; 7], &mut handles, true), + syscall::message_recv(raw, &mut [], &mut [0; 7], &mut handles, true), Err(SyscallError::InvalidArgument), "a buffer a byte short" ); assert_eq!( - syscall::message_recv(raw, &mut [0; 8], &mut handles[..1], true), + syscall::message_recv(raw, &mut [], &mut [0; 8], &mut handles[..1], true), Err(SyscallError::InvalidArgument), "a handle buffer one short" ); + assert_eq!( + syscall::message_recv(raw, &mut [0; 3], &mut [0; 4], &mut handles, true), + Err(SyscallError::InvalidArgument), + "two places a byte short together" + ); let mut bytes = [0u8; 8]; - let got = syscall::message_recv(raw, &mut bytes, &mut handles, true).expect("the message, whole"); + let got = syscall::message_recv(raw, &mut [], &mut bytes, &mut handles, true).expect("the message, whole"); assert_eq!((got, bytes), (Received { len: 8, handles: 2 }, [7; 8]), "the refused receives took part of it"); // SAFETY: installed for this call. handles[..2].iter().for_each(|&h| drop(unsafe { OwnedHandle::from_raw(h) })); + + // Its first bytes fill the head, and the rest the body, in order. + send(&client, &[1, 2, 3, 4, 5], vec![], false).expect("a message of five bytes"); + let (mut head, mut body) = ([0u8; 2], [0u8; 8]); + let got = syscall::message_recv(raw, &mut head, &mut body, &mut [], true).expect("the message, split"); + assert_eq!((got.len, head, &body[..3]), (5, [1, 2], &[3, 4, 5][..]), "a split receive cut or reordered the message"); } fn each_bound_refuses_the_send_past_it((client, server): (Connection, Connection)) { diff --git a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs index 8d8698215e6..a5b8137f726 100644 --- a/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs +++ b/tests/guest/rust-tests/src/bin/netstack_gone_mid_bind.rs @@ -191,7 +191,7 @@ fn mid_flight(request: Request) -> ! { say("connected"); let mut byte = [0u8; 1]; - let taken = syscall::message_recv(watch.as_handle().raw(), &mut byte, &mut [], false); + let taken = syscall::message_recv(watch.as_handle().raw(), &mut [], &mut byte, &mut [], false); assert_eq!(taken, Err(SyscallError::Gone), "the watch connection carried a message; nobody ever sent one"); let err = match request { diff --git a/tests/guest/rust-tests/src/bin/stream_handles.rs b/tests/guest/rust-tests/src/bin/stream_handles.rs index 4840a6e09f0..fb12c593014 100644 --- a/tests/guest/rust-tests/src/bin/stream_handles.rs +++ b/tests/guest/rust-tests/src/bin/stream_handles.rs @@ -227,7 +227,7 @@ fn refuses_a_foreign_handle_writing_nothing() { let stderr = String::from_utf8_lossy(&said.stderr); assert!(!said.status.success(), "the foreign handle was taken: {}", String::from_utf8_lossy(&said.stdout)); assert!(stderr.contains("is no stream of this process's"), "refused, but not by name: {stderr}"); - let taken = syscall::message_recv(conn.as_handle().raw(), &mut bytes, &mut [], false); + let taken = syscall::message_recv(conn.as_handle().raw(), &mut [], &mut bytes, &mut [], false); assert_eq!(taken, Err(SyscallError::Gone), "the refusal sent on the stream's connection before its end"); assert_eq!(rung.try_take(), Err(Hungup), "the refusal rang the stream before its end"); println!(" a foreign handle: refused by name, nothing written"); From b456763902724e77c2402994a823ca7cb958bc93 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 11:54:01 +0100 Subject: [PATCH 5/5] review r2: netstack makes both of a link's bells A NIC driver's hello carried the bell the driver rings, and netstack used it typed on every receive pass. A handle of another kind in that slot, even from a plain driver bug, ended the whole stack with a handle fault, and with it every stream and the healthy second link #1106's failover keeps. No check settles an event's kind, so netstack cannot refuse it. The serving side now makes every bell, as blockserve and netstack's own stream bells already do: Session::open makes both pairs and sends the driver its waker and its bell with MSG_OPENED (OPENED_HANDLES). The hello carries only the region, which Region::lent checks, and HELLO_HANDLES goes. netstack holds only events it made; only the driver trusts received bells, as diskserver does the block server's. The driver's departure is still its waker's last handle, so a hang-up keeps its meaning. Also the review's suggestions: the receive bounds' comment reads, the pipe-lock record is rewrapped with its page-population sentence on both copies, a double blank line goes, and response_bytes says its buffer must hold an error answer too. Co-Authored-By: Claude Opus 5.5 --- ...ns-a-user-copy-and-its-first-allocation.md | 11 ++++--- kernel/src/syscall/dispatch.rs | 6 ++-- lib/link/src/driver.rs | 31 +++++++++---------- lib/link/src/lib.rs | 5 +-- lib/link/src/wire.rs | 16 +++++----- sdk/toyos/src/ipc.rs | 1 - sdk/toyos/src/net.rs | 3 +- system/netstack/src/link.rs | 26 ++++++++-------- system/netstack/src/main.rs | 8 ++--- 9 files changed, 53 insertions(+), 54 deletions(-) diff --git a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md index 4ce80934d39..fbe4c32682c 100644 --- a/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md +++ b/issues/a-pipes-lock-spans-a-user-copy-and-its-first-allocation.md @@ -17,6 +17,9 @@ runs inside that lock, through the frame store pipes and connections share * `FrameBytes::write` makes one walk, `buf.read_pieces`, and copies each piece into its frames with `bytes.place`, in the same acquisition. +Each 16 pages of either copy take the caller's space lock and populate any +untouched page, a frame from the allocator and a 4 KiB zeroing. + A pipe's copy is bounded only by the pipe: `PIPE_CAPACITY` is 2 MiB, the largest single copy under the lock, and nothing above caps it (`SYS_READ`/`SYS_WRITE` pass the caller's length through). @@ -26,9 +29,7 @@ its queue's lock, so its copy is bounded at `MAX_MESSAGE_LEN`, 16 KiB, and its frames are taken the same way. Both its send and its receive run that copy inside `process::with_process_data`, nesting the queue's lock in the process lock: the receive installs the message's handles in the caller's table and -copies its bytes in one hold, so a refusal takes nothing. Each 16 pages of it take the caller's space -lock and populate any untouched page, a frame from the allocator and a 4 KiB -zeroing. +copies its bytes in one hold, so a refusal takes nothing. A write that grows the pipe also takes its frames under the same lock, nesting the frame allocator's lock: up to 513 frames, unzeroed, under one @@ -51,7 +52,7 @@ be taken before the lock and given back after it, as the read end's last holder already gives back what the pipe held. **Exit**: no pipe or queue lock is held across more than a bounded copy or any -allocation, the connection's nesting in the process lock included, the hold time is read before and after, and `audio_tone`'s worst -wake does not move. +allocation, the connection's nesting in the process lock included, the hold +time is read before and after, and `audio_tone`'s worst wake does not move. This is stage S0-a of `issues/anonymous-memory-swaps-to-an-encrypted-partition.md`. diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index b3230793673..97267e80b90 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -692,9 +692,9 @@ fn outgoing<'a>(ctx: &SyscallContext<'a>, args: &MessageSend) -> Result(ctx: &SyscallContext<'a>, args: &MessageRecv) -> Result>, ()> { if args.flags & !MESSAGE_NONBLOCK != 0 { return Err(()); diff --git a/lib/link/src/driver.rs b/lib/link/src/driver.rs index 371226d9fe0..979e4704d51 100644 --- a/lib/link/src/driver.rs +++ b/lib/link/src/driver.rs @@ -126,11 +126,9 @@ struct Session<'a> { region: Region, end: Driver, rx: FrameRx<4>, - /// Rung for the stack; the stack's bell answers `Hungup` once it goes. - ring: Waker, - /// The stack's doorbell, once the link is open; its waker going is the - /// stack's end. - rung: Option, + /// Once the link is open, the stack's two bells: the waker rung for the + /// stack, and the bell it rings, its waker going being the stack's end. + bells: Option<(Waker, Bell)>, said: Option, /// Received frames longer than a slot, dropped. oversize: u64, @@ -146,18 +144,16 @@ impl<'a> Session<'a> { .unwrap_or_else(|| panic!("{label}: the device's address is a group's, which no link carries")); let hello = Hello { mac, medium: N::MEDIUM, label: label.into() }; let shared = region.share().unwrap_or_else(|why| panic!("{label}: the link's region would not share: {why:?}")); - let (ring, stack_hears) = toyos::wake::pair().unwrap_or_else(|why| panic!("{label}: no doorbell for a link: {why:?}")); // A connection that will not take the hello is one whose stack has // gone: its hang-up is read in the first pass. - let _ = conn.try_send_bytes_with(wire::MSG_HELLO, &hello.encode(), [shared, stack_hears.into()]); + let _ = conn.try_send_bytes_with(wire::MSG_HELLO, &hello.encode(), [shared]); Self { label, conn: Some(conn), region, end, rx: FrameRx::new(), - ring, - rung: None, + bells: None, said: None, oversize: 0, oversize_said: 0, @@ -168,12 +164,15 @@ impl<'a> Session<'a> { let poller = Poller::new(2); loop { let ring = self.pass(nic).unwrap_or_else(|why| panic!("{}: the stack wrote {why}", self.label)); - if ring && self.rung.is_some() && self.ring.ring().is_err() { + if let Some((waker, _)) = &self.bells + && ring + && waker.ring().is_err() + { return Ended::HungUp("its doorbell is gone"); } poller.watch(nic.claim(), READABLE, TOKEN_CLAIM); - match (&self.rung, &self.conn) { - (Some(rung), _) => poller.watch(rung, READABLE, TOKEN_LINK), + match (&self.bells, &self.conn) { + (Some((_, rung)), _) => poller.watch(rung, READABLE, TOKEN_LINK), (None, Some(conn)) => poller.watch(conn, READABLE, TOKEN_LINK), (None, None) => unreachable!("{}: a link neither opening nor open", self.label), } @@ -255,16 +254,16 @@ impl<'a> Session<'a> { /// Read the stack's answer, then its doorbell; the end of the session /// if that is what came. fn hear(&mut self) -> Option { - if let Some(rung) = &self.rung { + if let Some((_, rung)) = &self.bells { return rung.try_take().err().map(|_| Ended::HungUp("its process ended")); } let conn = self.conn.as_ref().expect("a link not open is opening"); match self.rx.pump(conn) { RxStep::Idle => None, RxStep::Eof => Some(Ended::HungUp("its process ended before it answered")), - RxStep::Frame { msg_type: wire::MSG_OPENED, payload_len: 0, handles: 1 } => { - let [rung] = self.rx.take_handles::<1>().expect("counted above"); - self.rung = Some(Bell::from(rung)); + RxStep::Frame { msg_type: wire::MSG_OPENED, payload_len: 0, handles: wire::OPENED_HANDLES } => { + let [ring, rung] = self.rx.take_handles::<{ wire::OPENED_HANDLES }>().expect("counted above"); + self.bells = Some((Waker::from(ring), Bell::from(rung))); // Its work is done: the bells carry the link from here. self.conn = None; say!("{}: the link is open", self.label); diff --git a/lib/link/src/lib.rs b/lib/link/src/lib.rs index 59936db774a..8929196b90f 100644 --- a/lib/link/src/lib.rs +++ b/lib/link/src/lib.rs @@ -9,8 +9,9 @@ //! into and out of once on each side ([`ends`]). //! //! **A doorbell is an event**, rung after what it announces is in the -//! region: each side holds the waker of the other's bell, sent with the -//! hello and its answer ([`wire`]). A bell whose waker is gone is the link's +//! region: the stack makes both and sends the driver its ends with its +//! answer ([`wire`]), so it holds no event a driver sent. A bell whose waker +//! is gone is the link's //! end: the stack keeps every connection it carries across it, and the //! driver started again opens a new link. //! diff --git a/lib/link/src/wire.rs b/lib/link/src/wire.rs index 28d2367ab73..cb726fc0a81 100644 --- a/lib/link/src/wire.rs +++ b/lib/link/src/wire.rs @@ -1,8 +1,8 @@ //! The frames a link is opened with, before the region carries anything. //! //! **One way in, one answer.** A driver says who it is ([`MSG_HELLO`]) and -//! sends the region it made and the bell it rings with it; the stack answers -//! [`MSG_OPENED`] with the bell it rings, or [`MSG_REFUSED`] with a +//! sends the region it made with it; the stack answers [`MSG_OPENED`] with +//! the link's two bells, both of which it made, or [`MSG_REFUSED`] with a //! [`Refusal`]. After `MSG_OPENED` the connection has nothing left to carry, //! and either side's waker going ends the link. @@ -25,14 +25,14 @@ impl Address { } } -/// Driver to stack: a [`Hello`]. Handles: the region, then the bell the -/// driver rings. +/// Driver to stack: a [`Hello`]. Handles: the region. pub const MSG_HELLO: u32 = 1; -/// Driver to stack: the hello's handles. -pub const HELLO_HANDLES: usize = 2; -/// Stack to driver: the link is open. No payload. Handles: the bell the -/// stack rings. +/// Stack to driver: the link is open. No payload. Handles +/// ([`OPENED_HANDLES`]): the waker the driver rings the stack with, then the +/// bell the stack rings the driver on. pub const MSG_OPENED: u32 = 2; +/// Stack to driver: the opened answer's handles. +pub const OPENED_HANDLES: usize = 2; /// Stack to driver: refused, a [`Refusal`]'s word. pub const MSG_REFUSED: u32 = 3; diff --git a/sdk/toyos/src/ipc.rs b/sdk/toyos/src/ipc.rs index 8bb36c33b66..9269a67cca4 100644 --- a/sdk/toyos/src/ipc.rs +++ b/sdk/toyos/src/ipc.rs @@ -234,7 +234,6 @@ impl Connection { message(self.as_handle(), msg_type, &[], handles, true).map_err(nonblocking) } - /// Wait for the next frame and take its payload straight into /// `payload`, with the handles it carried. For a caller that keeps the /// payload in its own buffer: a [`FrameRx`] would be one more copy. A diff --git a/sdk/toyos/src/net.rs b/sdk/toyos/src/net.rs index 6ba0b761b56..c77a679c7a6 100644 --- a/sdk/toyos/src/net.rs +++ b/sdk/toyos/src/net.rs @@ -385,7 +385,8 @@ impl PendingResponse { } /// The answer's bytes, taken straight into `buf`: their length. An - /// answer longer than `buf` is [`NetError::Io`]. + /// answer longer than `buf` is [`NetError::Io`], an error answer + /// included, so `buf` must hold an [`ErrorResponse`] too. pub fn response_bytes(self, buf: &mut [u8]) -> Result { let (frame, _) = self.conn().recv_into(buf).map_err(hangup)?; let payload = &buf[..frame.payload_len]; diff --git a/system/netstack/src/link.rs b/system/netstack/src/link.rs index f8c05b409ea..8b8d877bbe6 100644 --- a/system/netstack/src/link.rs +++ b/system/netstack/src/link.rs @@ -1,5 +1,6 @@ //! The stack's side of a link: the region a driver sent with its hello, the -//! stack's end of the region's rings (`toyos-link`) and the two doorbells. +//! stack's end of the region's rings (`toyos-link`) and the two doorbells, +//! both of which the stack makes, so that it holds no event a driver sent. //! //! **What the driver writes is believed only as `toyos-link`'s ends bound //! it**: a frame's length by its slot, the link by its word, and the counters @@ -37,9 +38,8 @@ pub enum Heard { /// The driver went, or said what no driver says: it was refused, or let go. Gone, /// The driver said who it is, its address as the node takes one, with - /// its region and the bell it rings: its connection is - /// [`Pending::into_conn`]. - Hello(Hello, IndividualMac, Region, Bell), + /// its region: its connection is [`Pending::into_conn`]. + Hello(Hello, IndividualMac, Region), } impl Pending { @@ -59,12 +59,12 @@ impl Pending { match self.rx.pump(&self.conn) { RxStep::Idle => Heard::Nothing, RxStep::Eof => Heard::Gone, - RxStep::Frame { msg_type: wire::MSG_HELLO, payload_len, handles: wire::HELLO_HANDLES } => { + RxStep::Frame { msg_type: wire::MSG_HELLO, payload_len, handles: 1 } => { let hello = Hello::decode(self.rx.payload(payload_len)); let mac = hello.as_ref().and_then(|hello| IndividualMac::new(MacAddr(hello.mac.bytes()))); - let [region, rung] = self.rx.take_handles::<{ wire::HELLO_HANDLES }>().expect("counted above"); + let [region] = self.rx.take_handles::<1>().expect("counted above"); match (hello, mac, Region::lent(region).ok()) { - (Some(hello), Some(mac), Some(region)) => Heard::Hello(hello, mac, region, Bell::from(rung)), + (Some(hello), Some(mac), Some(region)) => Heard::Hello(hello, mac, region), _ => { say!("netstack: a NIC driver's hello is not one this protocol makes, or its region is none"); refuse(&self.conn, Refusal::Malformed); @@ -114,18 +114,18 @@ pub struct Session { impl Session { /// Open the link `hello` asked for over `region` as the node's `iface`, - /// and say so to the driver with the bell this end rings, its `conn` let - /// go then; `None` for a driver gone before it heard. - pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, rung: Bell, groups: &[Group]) -> Option { + /// and say so to the driver with its ends of the two bells, its `conn` + /// let go then; `None` for a driver gone before it heard. + pub fn open(hello: Hello, iface: IfIndex, conn: Connection, region: Region, groups: &[Group]) -> Option { let mut end = Stack::new(region.words()); for group in groups { assert!(end.join(region.words(), *group), "netstack: more groups than a link holds"); } - let Ok((ring_driver, driver_hears)) = toyos::wake::pair() else { - say!("netstack: no doorbell for {}'s link", hello.label); + let (Ok((ring_driver, driver_hears)), Ok((driver_rings, rung))) = (toyos::wake::pair(), toyos::wake::pair()) else { + say!("netstack: no doorbells for {}'s link", hello.label); return None; }; - conn.try_signal_with(wire::MSG_OPENED, [driver_hears.into()]).ok()?; + conn.try_signal_with(wire::MSG_OPENED, [driver_rings.into(), driver_hears.into()]).ok()?; Some(Self { ring_driver, rung, diff --git a/system/netstack/src/main.rs b/system/netstack/src/main.rs index 2be79753eb0..1c117e59559 100644 --- a/system/netstack/src/main.rs +++ b/system/netstack/src/main.rs @@ -348,7 +348,7 @@ fn answer_inspect(request: &Request, links: &[Held], counters: &[(IfIndex, Count /// the boot, and say so. fn open(asked: Asked, iface: IfIndex, n: usize, groups: &[Group]) -> Option { let label = asked.hello.label.clone(); - let opened = Session::open(asked.hello, iface, asked.conn, asked.region, asked.rung, groups); + let opened = Session::open(asked.hello, iface, asked.conn, asked.region, groups); match &opened { Some(_) => say!("netstack: {label}'s link is open, as link {n}"), None => say!("netstack: {label} went before its link {n} was opened"), @@ -375,9 +375,9 @@ fn hear_links(pending: &mut Vec) -> Vec { Heard::Gone => { pending.remove(i); } - Heard::Hello(hello, mac, region, rung) => { + Heard::Hello(hello, mac, region) => { let conn = pending.remove(i).into_conn(); - asked.push(Asked { hello, mac, conn, region, rung }); + asked.push(Asked { hello, mac, conn, region }); } } } @@ -463,8 +463,6 @@ struct Asked { mac: IndividualMac, conn: toyos::Connection, region: toyos_link::region::Region, - /// The bell the driver rings. - rung: toyos::wake::Bell, } /// An `inspect` waiting for the drivers' counters.