From d8f9a490b50f97a27704343e88346d66dcf1480a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 14:59:22 +0200 Subject: [PATCH 1/3] A descriptor call holds its object without a lock A `read` or `write` through a descriptor took the process's one descriptor-table lock to take its hold and again to give it up, so two threads doing I/O on different descriptors met on it twice a call. In #1150's metal A/B the pipe benchmark's two threads slept on it 128,000 to 153,000 times a run once the kernel stopped serializing them itself. The table's representation now carries the hold. A descriptor's word names an object's slot, and a slot counts its references, the descriptors naming it and the calls holding it: - A call counts itself onto the slot its descriptor names only while the count is not 0, then keeps the reference only if the descriptor still names that slot. That is one compare-exchange and two loads; giving the hold up is one subtraction. - A slot is free while its count is 0, and only the writer takes a count off 0, after writing the object. Whoever holds a reference reads the object, so nothing is written under a reader and a slot is reused only after its last hold. - Whoever takes a count to 0 is answered the object, read while it still held its reference, and lets it go: once, after the last call. - A descriptor changes in one store, so a call beside a `dup2` or a `replace` holds the old object or the new, and a closed number is free at once, as lowest-free numbering needs and as before. What changes a descriptor (`install`, `dup`, `dup2`, `close`, `replace`, the flag) still takes one lock, as a `Writer` the table hands out to one holder at a time; the mutators are its methods, so none is reachable without it. The lock is held across no blocking call: a released object is let go after it, as before. The first use still installs the endowed descriptors under it, and a call looks at one flag to know that happened. Each slot is a cache line of its own, so calls on two descriptors write no line they share. The table is all zeros now and costs an image no initialised data; the old one's free objects were not. A call that finds its descriptor naming a free slot reads the descriptor again: that state is a close in flight, and no longer told from a broken table on this path. The writer, under whose lock it cannot be in flight, still answers it as broken. `fd::table::model` runs a call beside a close, a close and a reuse of the slot, a reopen of the number, a second call and a `dup2` under loom, the object in loom's cell. Three controls show it can red: a call that counts itself onto a free slot, one that does not look at the descriptor again, and a writer that publishes a count unordered after its object. Closes issues/every-descriptor-call-takes-one-process-wide-lock-twice.md: a hold takes no lock at all. Co-Authored-By: Claude Fable 5.2 --- build/ci.rs | 13 + ...-call-takes-one-process-wide-lock-twice.md | 27 -- sdk/toyos/Cargo.toml | 12 + sdk/toyos/src/fd.rs | 83 ++-- sdk/toyos/src/fd/table.rs | 458 +++++++++++++----- sdk/toyos/src/fd/table/model.rs | 228 +++++++++ 6 files changed, 632 insertions(+), 189 deletions(-) delete mode 100644 issues/every-descriptor-call-takes-one-process-wide-lock-twice.md create mode 100644 sdk/toyos/src/fd/table/model.rs diff --git a/build/ci.rs b/build/ci.rs index 32a8297bd99..bfee018ecbc 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -840,6 +840,19 @@ pub(crate) const CONTROLS: &[Control] = &[ red(SDK, "transport-end-keeps-inflight", None, &[Fails( "transport::inflight::tests::an_end_answers_every_tag_once_and_a_late_completion_nothing", )]), + // The descriptor table's hold beside its writer. + red(SDK, "fd-hold-revives", None, &[ + Fails("fd::table::model::a_call_beside_a_close_holds_a_live_object_or_none"), + Fails("fd::table::model::two_calls_beside_a_close_let_the_object_go_once"), + Fails("fd::table::model::a_call_beside_a_dup2_holds_the_old_object_or_the_new"), + ]), + red(SDK, "fd-hold-unchecked", None, &[Fails( + "fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot", + )]), + red(SDK, "fd-publish-relaxed", None, &[ + Fails("fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot"), + Fails("fd::table::model::a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new"), + ]), ]; /// Whether a control's run showed its teeth. diff --git a/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md b/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md deleted file mode 100644 index ab71522a6c3..00000000000 --- a/issues/every-descriptor-call-takes-one-process-wide-lock-twice.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-04 ---- - -# Every descriptor call takes one process-wide lock twice - -A `read` or `write` through a descriptor (`sdk/std/sys/pal/mod.rs`'s `on_fd`) -holds its object for the call through `toyos::fd::hold`. That takes the -process's one descriptor-table lock (`sdk/toyos/src/fd.rs`, `TABLE`) to acquire -the slot and again to let it go. Two threads of one process doing I/O on -different descriptors meet on that lock twice per call. - -The lock was a plain futex lock, so any collision slept in `FUTEX_WAIT` and -woke through `FUTEX_WAKE`. In the S0-a metal A/B (#1150, round 2), the pipe -benchmark's writer and reader threads (`osbench` `pipe.throughput.*`) made -about 125-150 futex waits a run on main. With the kernel no longer -serializing their pipe calls under its process lock, they made 128,000-153,000 -on the T14. That was much of the small-pipe-write regression measured there. - -`toyos::sync::lock_word` now spins briefly before sleeping, which turns a -collision into a short wait. Spinning bounds the cost of a collision; it does -not stop two threads meeting on a lock neither needs. - -**Exit**: a descriptor's hold for one call takes no lock that a call on another -descriptor takes. diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index 4dbfda621c4..c4785a34e83 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -95,6 +95,18 @@ transport-no-resume = [] # A look without the gate is believed when it names a violation, so a cursor # another holder moved under it ends the call. transport-ungated-violation = [] +# The descriptor table's controls (`src/fd/table.rs`), declared and run as the +# transport's are; `src/fd/table/model.rs` holds a call beside the writer. +# +# A call counts itself onto a slot whose count is 0, so it holds an object +# the close already let go. +fd-hold-revives = [] +# A call keeps its reference without looking at the descriptor again, so it +# holds whatever took the slot its descriptor named. +fd-hold-unchecked = [] +# The writer stores a new object's count `Relaxed`, so a call that counts +# itself on need not see the object written. +fd-publish-relaxed = [] [lints.rust] # The guest's target: its stream sinks are compiled only for it. diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index 3a82b0c54ab..334ceff5902 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -389,48 +389,70 @@ fn file_group(label: &str) -> Option { /// The table itself, in the copy that owns it. #[cfg(all(feature = "fd-table", not(test)))] mod owner { - use super::table::{Fault, Table}; + use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + + use super::table::{Cell, Fault, Table, Writer}; use super::*; use crate::endow::Endowments; use crate::sync::Lock; use crate::OwnedHandle; - /// The table, and whether its endowed descriptors were installed yet. - static TABLE: Lock<(bool, Table)> = Lock::new((false, Table::new())); + type Descriptors = Table, MAX_FDS>; + type Names<'a> = Writer<'a, AtomicU32, Cell, MAX_FDS>; + + static TABLE: Descriptors = Table::new(); + /// Held by whoever changes what a descriptor names. + static WRITING: Lock<()> = Lock::new(()); + /// Whether the endowed descriptors were installed yet. + static ENDOWED: AtomicBool = AtomicBool::new(false); + + fn refusal(fault: Fault) -> Error { + match fault { + Fault::Refused(e) => e, + Fault::Broken => panic!("toyos::fd: the descriptor table is inconsistent"), + } + } - /// `f` on the table, whose first use installs every descriptor the - /// process was endowed at its number. Nothing panics under the lock: a - /// panic writes to stderr, a descriptor, which would wait on it forever. - fn table(f: impl FnOnce(&mut Table) -> Result) -> Result { + /// `f` on the table's writer, whose first use installs every descriptor + /// the process was endowed at its number. Nothing panics under the lock. + fn change(f: impl FnOnce(&mut Names<'_>) -> Result) -> Result { let endowed = Endowments::get(); let mut refused = 0; let answer = { - let mut held = TABLE.lock(); - let (ready, table) = &mut *held; - let installed = match *ready { + let _held = WRITING.lock(); + // SAFETY: the one writer while `WRITING` is held. + let mut names = unsafe { TABLE.writer() }; + let installed = match ENDOWED.load(Ordering::Relaxed) { true => Ok(()), false => { - *ready = true; - endow(table, endowed).map(|n| refused = n) + let installed = endow(&mut names, endowed).map(|n| refused = n); + ENDOWED.store(true, Ordering::Release); + installed } }; - installed.and_then(|()| f(table)) + installed.and_then(|()| f(&mut names)) }; - // Said once the lock is let go, since stderr is a descriptor. + // Said once the lock is let go: the write may block. for _ in 0..refused { let _ = crate::files::write(2, b"toyos::fd: an fd: endowment whose number was taken was refused\n"); } - answer.map_err(|fault| match fault { - Fault::Refused(e) => e, - Fault::Broken => panic!("toyos::fd: the descriptor table is inconsistent"), - }) + answer.map_err(refusal) + } + + /// The table for a call, which takes no lock once the endowed + /// descriptors are installed. + fn table() -> Result<&'static Descriptors, Error> { + if !ENDOWED.load(Ordering::Acquire) { + change(|_| Ok(()))?; + } + Ok(&TABLE) } /// Install each `fd:` endowment at its number, a socket's taken up as /// this process's holder first. One whose number another already took, /// or whose socket this process holds no room for, is refused, its handle /// closed: how many were. - fn endow(table: &mut Table, endowed: &Endowments) -> Result { + fn endow(table: &mut Names<'_>, endowed: &Endowments) -> Result { let mut refused = 0; let mut files = alloc::collections::BTreeMap::new(); for label in endowed.labels() { @@ -485,7 +507,7 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_install(record: Record, inheritable: bool, floor: Fd) -> i32 { - table(|t| t.install(record, inheritable, floor)).unwrap_or_else(|e| { + change(|t| t.install(record, inheritable, floor)).unwrap_or_else(|e| { // SAFETY: refused, so the record is still the caller's, given here. unsafe { record.release() }; encode(e) @@ -494,7 +516,8 @@ mod owner { #[no_mangle] unsafe extern "C" fn toyos_fd_acquire(fd: Fd, out: *mut Record) -> i32 { - match table(|t| t.acquire(fd)) { + let held = table().and_then(|t| t.acquire(fd, |record| released(Some(record))).map_err(refusal)); + match held { Ok((token, object)) => { // SAFETY: the caller's live place. unsafe { out.write(object) }; @@ -505,14 +528,16 @@ mod owner { } #[no_mangle] - extern "C" fn toyos_fd_unuse(token: i32) { + unsafe extern "C" fn toyos_fd_unuse(token: i32) { let token = usize::try_from(token).expect("toyos::fd: a token no acquire answered"); - released(table(|t| t.unuse(token)).expect("toyos::fd: a token no acquire answered")); + // SAFETY: the caller's: a token `toyos_fd_acquire` answered, given + // back once. + released(unsafe { TABLE.unuse(token) }.expect("toyos::fd: a token no acquire answered")); } #[no_mangle] extern "C" fn toyos_fd_close(fd: Fd) -> i32 { - match table(|t| t.close(fd)) { + match change(|t| t.close(fd)) { Ok(object) => { released(object); 0 @@ -523,12 +548,12 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_dup(fd: Fd, floor: Fd, inheritable: bool) -> i32 { - table(|t| t.dup(fd, floor, inheritable)).unwrap_or_else(encode) + change(|t| t.dup(fd, floor, inheritable)).unwrap_or_else(encode) } #[no_mangle] extern "C" fn toyos_fd_dup2(old: Fd, new: Fd, inheritable: bool) -> i32 { - match table(|t| t.dup2(old, new, inheritable)) { + match change(|t| t.dup2(old, new, inheritable)) { Ok(displaced) => { released(displaced); new @@ -539,7 +564,7 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_replace(fd: Fd, record: Record) -> i32 { - match table(|t| t.replace(fd, record)) { + match change(|t| t.replace(fd, record)) { Ok(old) => { released(old); 0 @@ -554,12 +579,12 @@ mod owner { #[no_mangle] extern "C" fn toyos_fd_inheritable(fd: Fd) -> i32 { - table(|t| t.inheritable(fd)).map_or_else(encode, i32::from) + table().and_then(|t| t.inheritable(fd).map_err(refusal)).map_or_else(encode, i32::from) } #[no_mangle] extern "C" fn toyos_fd_set_inheritable(fd: Fd, inheritable: bool) -> i32 { - table(|t| t.set_inheritable(fd, inheritable)).map_or_else(encode, |()| 0) + change(|t| t.set_inheritable(fd, inheritable)).map_or_else(encode, |()| 0) } } diff --git a/sdk/toyos/src/fd/table.rs b/sdk/toyos/src/fd/table.rs index 941278113ff..4fd1a42e969 100644 --- a/sdk/toyos/src/fd/table.rs +++ b/sdk/toyos/src/fd/table.rs @@ -1,27 +1,93 @@ -//! The table's state, without its lock: every decision the table makes, -//! which a host tests. Nothing in it panics. - -use super::{Error, Fd, Record, MAX_FDS}; - -/// A descriptor's word: 0 for a free number, else its object's index plus -/// one, with [`INHERITABLE`] set for an inheritable one. The empty table is -/// all zeros, so it costs a program no initialised data. -type Word = u16; -const INHERITABLE: Word = 0x8000; -const _: () = assert!(MAX_FDS < INHERITABLE as usize); - -#[derive(Clone, Copy)] -struct Held { - object: Record, - /// The descriptors naming it and the calls holding it; 0 for a free one. - refs: u32, +//! The table's state: every decision the table makes, which a host tests and +//! `model` checks under loom. Nothing in it panics. +//! +//! A descriptor's word names an object's slot, and a slot counts its +//! references: the descriptors naming it and the calls holding it. +//! +//! - **A slot is free while its count is 0**, and only the [`Writer`] takes a +//! count off 0: it writes the object, then stores the count. Whoever holds a +//! reference reads the object, so nothing is written under a reader. +//! - **A call counts itself onto a slot only while the count is not 0**, and +//! keeps the reference only if the descriptor still names that slot. So the +//! object a call holds is one its descriptor named during the call, and a +//! slot is reused only after its last hold. +//! - **Whoever takes a count to 0 is answered the object**, read while it +//! still held its reference, and lets it go: once, after the last call. +//! - **A descriptor changes in one store**, so a call beside a `dup2` holds +//! the old object or the new, and a closed number is free at once. + +use core::cell::UnsafeCell; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicU32, Ordering}; + +use super::{Error, Fd}; + +#[cfg(test)] +mod model; + +/// A descriptor's word: 0 for a free number, else its object's slot plus one, +/// with [`INHERITABLE`] set for an inheritable one. The empty table is all +/// zeros, so it costs a program no initialised data. +const INHERITABLE: u32 = 1 << 31; + +/// A shared word of the table: an atomic, or a model's. +pub(super) trait Word { + fn load(&self, order: Ordering) -> u32; + fn store(&self, value: u32, order: Ordering); + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result; + fn fetch_sub(&self, value: u32, order: Ordering) -> u32; +} + +impl Word for AtomicU32 { + fn load(&self, order: Ordering) -> u32 { + AtomicU32::load(self, order) + } + fn store(&self, value: u32, order: Ordering) { + AtomicU32::store(self, value, order) + } + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result { + AtomicU32::compare_exchange(self, current, new, success, failure) + } + fn fetch_sub(&self, value: u32, order: Ordering) -> u32 { + AtomicU32::fetch_sub(self, value, order) + } } -const FREE: Held = Held { object: Record::FREE, refs: 0 }; +/// Where a slot keeps its object: a cell, or a model's, which checks each +/// access against the others. +pub(super) trait Place { + type Object: Copy; + + /// # Safety + /// The place was written, and no write is concurrent. + unsafe fn read(&self) -> Self::Object; + + /// # Safety + /// No other access is concurrent. + unsafe fn write(&self, object: Self::Object); +} + +pub(super) struct Cell(UnsafeCell>); + +// SAFETY: reached only through `Place`, whose callers keep a write alone. +unsafe impl Sync for Cell {} + +impl Place for Cell { + type Object = O; + + unsafe fn read(&self) -> O { + // SAFETY: the caller's: written, and not being written. + unsafe { (*self.0.get()).assume_init() } + } + + unsafe fn write(&self, object: O) { + // SAFETY: the caller's: nothing else reaches the cell. + unsafe { (*self.0.get()).write(object) }; + } +} /// Why a table operation failed: refused, or the table found inconsistent, -/// which the owner panics on only once its lock is released, since the panic -/// writes to a descriptor. +/// which the owner panics on only once its lock is released. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Fault { Refused(Error), @@ -34,68 +100,183 @@ impl From for Fault { } } -/// A removal answers the object it released, for the caller to let go -/// outside the lock. -pub(super) struct Table { - fds: [Word; N], - objects: [Held; N], +/// An object's slot, on a cache line of its own: calls on two descriptors +/// write no line they share. +#[repr(align(64))] +struct Slot { + /// The slot's count of references. + refs: W, + object: P, } -impl Table { +pub(super) struct Table { + fds: [W; N], + slots: [Slot; N], +} + +impl Table, N> { pub(super) const fn new() -> Self { - Self { fds: [0; N], objects: [FREE; N] } + assert!(N < INHERITABLE as usize); + Self { + fds: [const { AtomicU32::new(0) }; N], + slots: [const { Slot { refs: AtomicU32::new(0), object: Cell(UnsafeCell::new(MaybeUninit::uninit())) } }; N], + } + } +} + +impl Table { + fn word(&self, fd: Fd) -> Result<&W, Error> { + usize::try_from(fd).ok().and_then(|i| self.fds.get(i)).ok_or(Error::BadFd) + } + + /// The slot a descriptor's word names. + fn named(word: u32) -> Result { + if word == 0 { + return Err(Error::BadFd.into()); + } + ((word & !INHERITABLE) as usize).checked_sub(1).filter(|&slot| slot < N).ok_or(Fault::Broken) + } + + /// One reference more on `slot`, unless it is free. + fn reference(&self, slot: usize) -> Result { + let refs = &self.slots.get(slot).ok_or(Fault::Broken)?.refs; + let mut count = refs.load(Ordering::Acquire); + loop { + if count == 0 && !cfg!(feature = "fd-hold-revives") { + return Ok(false); + } + let more = count.checked_add(1).ok_or(Fault::Broken)?; + match refs.compare_exchange(count, more, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return Ok(true), + Err(now) => count = now, + } + } + } + + /// One reference fewer on `slot`: its object, once none is left. + /// + /// # Safety + /// The caller gives up a reference it holds on `slot`. + unsafe fn unref(&self, slot: usize) -> Result, Fault> { + let slot = self.slots.get(slot).ok_or(Fault::Broken)?; + // SAFETY: the caller's reference keeps the slot from the writer, and + // it is given up only below. + let object = unsafe { slot.object.read() }; + match slot.refs.fetch_sub(1, Ordering::AcqRel) { + 0 => Err(Fault::Broken), + 1 => Ok(Some(object)), + _ => Ok(None), + } + } + + /// `fd`'s object, held until [`Self::unuse`] of the token answered. An + /// object whose last reference a lost race left with this call goes to + /// `let_go`. + pub(super) fn acquire(&self, fd: Fd, mut let_go: impl FnMut(P::Object)) -> Result<(usize, P::Object), Fault> { + let word = self.word(fd)?; + loop { + let slot = Self::named(word.load(Ordering::Acquire))?; + // A free slot was closed since the word was read, which the next + // read of the word shows. + if !self.reference(slot)? { + continue; + } + // SAFETY: the reference just taken. + let object = unsafe { self.slots[slot].object.read() }; + if cfg!(feature = "fd-hold-unchecked") || Self::named(word.load(Ordering::Acquire)) == Ok(slot) { + return Ok((slot, object)); + } + // SAFETY: the reference just taken. + if let Some(last) = unsafe { self.unref(slot) }? { + let_go(last); + } + } } - /// `fd`'s index, object and flag. + /// A call done with the object `token` names: the object, if nothing + /// names it any more. + /// + /// # Safety + /// `token` is one this table's [`Self::acquire`] answered, given back + /// once. + pub(super) unsafe fn unuse(&self, token: usize) -> Result, Fault> { + // SAFETY: the call's reference. + unsafe { self.unref(token) } + } + + pub(super) fn inheritable(&self, fd: Fd) -> Result { + let word = self.word(fd)?.load(Ordering::Acquire); + Self::named(word).map(|_| word & INHERITABLE != 0) + } + + /// The table's one writer. + /// + /// # Safety + /// No other [`Writer`] of this table exists while this one does. + pub(super) unsafe fn writer(&self) -> Writer<'_, W, P, N> { + Writer(self) + } +} + +/// The right to change what the table's descriptors name. A removal answers +/// the object it released, for the caller to let go outside its lock. +pub(super) struct Writer<'a, W, P, const N: usize>(&'a Table); + +impl Writer<'_, W, P, N> { + /// `fd`'s index, slot and flag. fn entry(&self, fd: Fd) -> Result<(usize, usize, bool), Fault> { let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - let word = *self.fds.get(i).ok_or(Error::BadFd)?; - let object = usize::from(word & !INHERITABLE).checked_sub(1).ok_or(match word { - 0 => Fault::Refused(Error::BadFd), - _ => Fault::Broken, - })?; - Ok((i, object, word & INHERITABLE != 0)) + let word = self.0.fds.get(i).ok_or(Error::BadFd)?.load(Ordering::Relaxed); + let slot = Table::::named(word)?; + // Only this writer takes a name's reference away. + if self.0.slots[slot].refs.load(Ordering::Relaxed) == 0 { + return Err(Fault::Broken); + } + Ok((i, slot, word & INHERITABLE != 0)) } - fn set(&mut self, i: usize, object: usize, inheritable: bool) -> Result<(), Fault> { - let index = Word::try_from(object).ok().filter(|&o| o < INHERITABLE - 1).ok_or(Fault::Broken)?; + fn set(&mut self, i: usize, slot: usize, inheritable: bool) -> Result<(), Fault> { + let index = u32::try_from(slot).ok().filter(|&s| s < INHERITABLE - 1).ok_or(Fault::Broken)?; let flag = if inheritable { INHERITABLE } else { 0 }; - *self.fds.get_mut(i).ok_or(Fault::Broken)? = (index + 1) | flag; + self.0.fds.get(i).ok_or(Fault::Broken)?.store((index + 1) | flag, Ordering::Release); Ok(()) } - fn lowest_free(&self, floor: Fd) -> Result { - let floor = usize::try_from(floor).map_err(|_| Error::Full)?; - let above = self.fds.get(floor..).ok_or(Error::Full)?; - above.iter().position(|&w| w == 0).map(|i| floor + i).ok_or(Error::Full) + fn is_free(&self, i: usize) -> bool { + self.0.fds.get(i).is_some_and(|word| word.load(Ordering::Relaxed) == 0) } - /// A free object holding `object`, referenced once. - fn hold(&mut self, object: Record) -> Result { - let slot = self.objects.iter().position(|h| h.refs == 0).ok_or(Error::Full)?; - self.objects[slot] = Held { object, refs: 1 }; - Ok(slot) + fn lowest_free(&self, floor: Fd) -> Result { + let floor = usize::try_from(floor).map_err(|_| Error::Full)?; + (floor..N).find(|&i| self.is_free(i)).ok_or(Error::Full) } - fn held(&mut self, object: usize) -> Result<&mut Held, Fault> { - self.objects.get_mut(object).filter(|h| h.refs != 0).ok_or(Fault::Broken) + /// A free slot holding `object`, referenced once. + fn hold(&mut self, object: P::Object) -> Result { + // The load that finds a slot free is ordered after its last reader. + let index = self.0.slots.iter().position(|slot| slot.refs.load(Ordering::Acquire) == 0).ok_or(Error::Full)?; + let slot = &self.0.slots[index]; + // SAFETY: a free slot is this writer's alone until its count is stored. + unsafe { slot.object.write(object) }; + let publish = if cfg!(feature = "fd-publish-relaxed") { Ordering::Relaxed } else { Ordering::Release }; + slot.refs.store(1, publish); + Ok(index) } - fn reference(&mut self, object: usize) -> Result { - let held = self.held(object)?; - held.refs = held.refs.checked_add(1).ok_or(Fault::Broken)?; - Ok(held.object) + /// One reference more on a slot a descriptor names. + fn reference(&mut self, slot: usize) -> Result<(), Fault> { + self.0.reference(slot)?.then_some(()).ok_or(Fault::Broken) } - /// A reference fewer on `object`: the object, once none is left. - fn unref(&mut self, object: usize) -> Result, Fault> { - let held = self.held(object)?; - held.refs -= 1; - Ok((held.refs == 0).then(|| core::mem::replace(held, FREE).object)) + /// One reference fewer on a slot [`Self::entry`] answered, whose name + /// was just taken away. + fn unname(&mut self, slot: usize) -> Result, Fault> { + // SAFETY: the name's reference, which `entry` found counted. + unsafe { self.0.unref(slot) } } /// Name `object` by the lowest free descriptor at or above `floor`. - pub(super) fn install(&mut self, object: Record, inheritable: bool, floor: Fd) -> Result { + pub(super) fn install(&mut self, object: P::Object, inheritable: bool, floor: Fd) -> Result { let fd = self.lowest_free(floor)?; let slot = self.hold(object)?; self.set(fd, slot, inheritable)?; @@ -104,11 +285,8 @@ impl Table { /// Name `object` by `fd` exactly: `BadFd` for a number past the table /// or already open. - pub(super) fn install_at(&mut self, fd: Fd, object: Record, inheritable: bool) -> Result<(), Fault> { - let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - if self.fds.get(i) != Some(&0) { - return Err(Error::BadFd.into()); - } + pub(super) fn install_at(&mut self, fd: Fd, object: P::Object, inheritable: bool) -> Result<(), Fault> { + let i = usize::try_from(fd).ok().filter(|&i| self.is_free(i)).ok_or(Error::BadFd)?; let slot = self.hold(object)?; self.set(i, slot, inheritable) } @@ -116,80 +294,64 @@ impl Table { /// An endowed alias at its exact number, without displacing an earlier /// endowment. Both descriptors share one object and one final release. pub(super) fn alias_at(&mut self, source: Fd, fd: Fd) -> Result<(), Fault> { - let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; - if self.fds.get(i) != Some(&0) { return Err(Error::BadFd.into()); } - let (_, object, _) = self.entry(source)?; - self.reference(object)?; - self.set(i, object, true) + let i = usize::try_from(fd).ok().filter(|&i| self.is_free(i)).ok_or(Error::BadFd)?; + let (_, slot, _) = self.entry(source)?; + self.reference(slot)?; + self.set(i, slot, true) } /// A second descriptor on `fd`'s object, the lowest free at or above /// `floor`. pub(super) fn dup(&mut self, fd: Fd, floor: Fd, inheritable: bool) -> Result { - let (_, object, _) = self.entry(fd)?; + let (_, slot, _) = self.entry(fd)?; let new = self.lowest_free(floor)?; - self.reference(object)?; - self.set(new, object, inheritable)?; + self.reference(slot)?; + self.set(new, slot, inheritable)?; Ok(new as Fd) } /// `new` on `old`'s object, whatever `new` named before: that object, /// if this was its last reference. `new == old` changes nothing. - pub(super) fn dup2(&mut self, old: Fd, new: Fd, inheritable: bool) -> Result, Fault> { - let (_, object, _) = self.entry(old)?; - let slot = usize::try_from(new).ok().filter(|&i| i < N).ok_or(Error::BadFd)?; + pub(super) fn dup2(&mut self, old: Fd, new: Fd, inheritable: bool) -> Result, Fault> { + let (_, slot, _) = self.entry(old)?; + let i = usize::try_from(new).ok().filter(|&i| i < N).ok_or(Error::BadFd)?; if new == old { return Ok(None); } let displaced = self.entry(new).ok(); - self.reference(object)?; - self.set(slot, object, inheritable)?; + self.reference(slot)?; + self.set(i, slot, inheritable)?; match displaced { - Some((_, displaced, _)) => self.unref(displaced), + Some((_, displaced, _)) => self.unname(displaced), None => Ok(None), } } /// Forget `fd`: its object, if this was its last reference. - pub(super) fn close(&mut self, fd: Fd) -> Result, Fault> { - let (i, object, _) = self.entry(fd)?; - *self.fds.get_mut(i).ok_or(Fault::Broken)? = 0; - self.unref(object) - } - - /// `fd`'s object, held until [`Self::unuse`] of the token answered. - pub(super) fn acquire(&mut self, fd: Fd) -> Result<(usize, Record), Fault> { - let (_, object, _) = self.entry(fd)?; - Ok((object, self.reference(object)?)) - } - - /// A call done with the object `token` names: the object, if nothing - /// names it any more. - pub(super) fn unuse(&mut self, token: usize) -> Result, Fault> { - self.unref(token) + pub(super) fn close(&mut self, fd: Fd) -> Result, Fault> { + let (i, slot, _) = self.entry(fd)?; + self.0.fds.get(i).ok_or(Fault::Broken)?.store(0, Ordering::Release); + self.unname(slot) } /// Make `fd` name `object` from here on, in place: the object it named, /// if this was its last reference. Its flag stays. - pub(super) fn replace(&mut self, fd: Fd, object: Record) -> Result, Fault> { + pub(super) fn replace(&mut self, fd: Fd, object: P::Object) -> Result, Fault> { let (i, old, inheritable) = self.entry(fd)?; let slot = self.hold(object)?; self.set(i, slot, inheritable)?; - self.unref(old) - } - - pub(super) fn inheritable(&self, fd: Fd) -> Result { - self.entry(fd).map(|(_, _, inheritable)| inheritable) + self.unname(old) } pub(super) fn set_inheritable(&mut self, fd: Fd, inheritable: bool) -> Result<(), Fault> { - let (i, object, _) = self.entry(fd)?; - self.set(i, object, inheritable) + let (i, slot, _) = self.entry(fd)?; + self.set(i, slot, inheritable) } } #[cfg(test)] mod tests { + use super::super::Record; use super::*; extern "C" fn nothing(_: u64) {} @@ -198,21 +360,33 @@ mod tests { Record { handle: toyos_abi::HANDLE_INVALID, tag: super::super::UNTAGGED, word, release: Some(nothing) } } - type T = Table<8>; + type T = Table, 8>; + type W<'a> = Writer<'a, AtomicU32, Cell, 8>; + + /// These tests' one writer of `t`. + fn writer(t: &T) -> W<'_> { + // SAFETY: each test makes one. + unsafe { t.writer() } + } /// The word of what a removal released, which names it in these tests. fn w(released: Result, Fault>) -> Result, Fault> { released.map(|o| o.map(|o| o.word)) } + fn acquire(t: &T, fd: Fd) -> Result<(usize, Record), Fault> { + t.acquire(fd, |_| panic!("a call with no race was left an object")) + } + /// The objects something still references. fn held(t: &T) -> usize { - t.objects.iter().filter(|h| h.refs != 0).count() + t.slots.iter().filter(|slot| slot.refs.load(Ordering::Relaxed) != 0).count() } #[test] fn descriptors_are_numbered_lowest_free_from_the_floor() { - let mut t = T::new(); + let t = T::new(); + let mut t = writer(&t); assert_eq!(t.install(o(1), false, 0), Ok(0)); assert_eq!(t.install(o(2), false, 0), Ok(1)); assert_eq!(t.install(o(3), false, 5), Ok(5)); @@ -226,43 +400,55 @@ mod tests { /// keeps a flag of its own. #[test] fn an_object_goes_with_its_last_descriptor_and_flags_are_per_descriptor() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let a = t.install(o(7), true, 0).unwrap(); let b = t.dup(a, 0, false).unwrap(); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(true), Ok(false))); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(true), Ok(false))); assert_eq!(t.set_inheritable(b, true), Ok(())); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(true), Ok(true))); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(true), Ok(true))); assert_eq!(t.set_inheritable(a, false), Ok(())); - assert_eq!((t.inheritable(a), t.inheritable(b)), (Ok(false), Ok(true)), "a flag is the descriptor's, not its object's"); + assert_eq!((table.inheritable(a), table.inheritable(b)), (Ok(false), Ok(true)), "a flag is the descriptor's, not its object's"); assert_eq!(w(t.close(a)), Ok(None), "released while a dup named it"); assert_eq!(w(t.close(b)), Ok(Some(7))); assert_eq!(w(t.close(b)), Err(Error::BadFd.into())); - assert_eq!(held(&t), 0, "an object outlived its descriptors"); + assert_eq!(held(&table), 0, "an object outlived its descriptors"); } /// A call holds its object: a close meanwhile makes the descriptor gone - /// at once, and the object goes when the call is done. + /// at once, its number free, and the object goes when the call is done. #[test] fn a_close_during_a_call_releases_the_object_when_the_call_is_done() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let fd = t.install(o(9), false, 0).unwrap(); - let (token, object) = t.acquire(fd).unwrap(); + let (token, object) = acquire(&table, fd).unwrap(); assert_eq!(object.word, 9); assert_eq!(w(t.close(fd)), Ok(None), "the object went under a call"); - assert!(matches!(t.acquire(fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); - assert_eq!(w(t.unuse(token)), Ok(Some(9))); - assert_eq!(held(&t), 0, "an object outlived its call"); + assert!(matches!(acquire(&table, fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); + assert_eq!(t.install(o(10), false, 0), Ok(fd), "a closed number waited for a call"); + let (other, object) = acquire(&table, fd).unwrap(); + assert_eq!(object.word, 10); + assert_ne!(other, token, "an object took the slot of one a call still holds"); + // SAFETY: each a token `acquire` answered, given back once. + unsafe { + assert_eq!(w(table.unuse(token)), Ok(Some(9))); + assert_eq!(w(table.unuse(other)), Ok(None)); + } + assert_eq!(w(t.close(fd)), Ok(Some(10))); + assert_eq!(held(&table), 0, "an object outlived its call"); } #[test] fn dup2_displaces_what_the_target_named_and_onto_itself_changes_nothing() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let a = t.install(o(1), false, 0).unwrap(); let b = t.install(o(2), false, 0).unwrap(); assert_eq!(w(t.dup2(a, b, true)), Ok(Some(2))); - assert_eq!(t.inheritable(b), Ok(true)); + assert_eq!(table.inheritable(b), Ok(true)); assert_eq!(w(t.dup2(a, a, true)), Ok(None)); - assert_eq!(t.inheritable(a), Ok(false), "dup2 onto itself set a flag"); + assert_eq!(table.inheritable(a), Ok(false), "dup2 onto itself set a flag"); assert_eq!(w(t.dup2(a, 6, false)), Ok(None), "an unused target"); assert_eq!(w(t.dup2(a, 8, false)), Err(Error::BadFd.into()), "a target past the table"); assert_eq!(w(t.dup2(7, 1, false)), Err(Error::BadFd.into())); @@ -270,15 +456,16 @@ mod tests { let last = t.close(fd).unwrap(); assert_eq!(last.is_some(), fd == 6, "descriptor {fd}"); } - assert_eq!(held(&t), 0, "an object outlived its descriptors"); + assert_eq!(held(&table), 0, "an object outlived its descriptors"); } #[test] fn replace_keeps_the_number_and_the_flag_and_releases_the_old_object() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); let fd = t.install(o(1), true, 0).unwrap(); assert_eq!(w(t.replace(fd, o(2))), Ok(Some(1))); - assert_eq!(t.inheritable(fd), Ok(true)); + assert_eq!(table.inheritable(fd), Ok(true)); assert_eq!(w(t.close(fd)), Ok(Some(2))); assert_eq!(w(t.replace(fd, o(3))), Err(Error::BadFd.into())); } @@ -287,9 +474,10 @@ mod tests { /// open already is refused, and the table is unchanged. #[test] fn an_endowment_is_installed_at_its_number_or_refused() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); assert_eq!(t.install_at(3, o(1), true), Ok(())); - assert_eq!(t.inheritable(3), Ok(true)); + assert_eq!(table.inheritable(3), Ok(true)); assert_eq!(t.install_at(3, o(2), true), Err(Error::BadFd.into())); assert_eq!(t.install_at(8, o(2), true), Err(Error::BadFd.into())); assert_eq!(t.install_at(-1, o(2), true), Err(Error::BadFd.into())); @@ -299,25 +487,29 @@ mod tests { #[test] fn endowed_aliases_share_one_object_and_cannot_displace_an_endowment() { - let mut t = T::new(); + let table = T::new(); + let mut t = writer(&table); t.install_at(3, o(1), true).unwrap(); t.alias_at(3, 0).unwrap(); - assert_eq!(held(&t), 1); + assert_eq!(held(&table), 1); assert_eq!(t.alias_at(3, 0), Err(Error::BadFd.into())); assert_eq!(w(t.close(3)), Ok(None)); assert_eq!(w(t.close(0)), Ok(Some(1))); } - /// A descriptor naming no object is a broken table, which the table - /// answers rather than panics on: the owner panics once its lock is let - /// go. + /// A descriptor naming no slot, or a free one under the writer, is a + /// broken table, which the table answers rather than panics on. #[test] fn an_inconsistent_table_is_answered_not_panicked_on() { - let mut t = T::new(); - t.fds[2] = 1; - assert_eq!(t.acquire(2).map(|_| ()), Err(Fault::Broken)); + let table = T::new(); + let mut t = writer(&table); + table.fds[2].store(9, Ordering::Relaxed); + assert_eq!(acquire(&table, 2).map(|_| ()), Err(Fault::Broken)); + assert_eq!(w(t.close(2)), Err(Fault::Broken)); + table.fds[2].store(INHERITABLE, Ordering::Relaxed); + assert_eq!(table.inheritable(2), Err(Fault::Broken)); + table.fds[2].store(1, Ordering::Relaxed); assert_eq!(w(t.close(2)), Err(Fault::Broken)); - t.fds[2] = INHERITABLE; - assert_eq!(t.inheritable(2), Err(Fault::Broken)); + assert_eq!(t.dup(2, 0, false), Err(Fault::Broken)); } } diff --git a/sdk/toyos/src/fd/table/model.rs b/sdk/toyos/src/fd/table/model.rs new file mode 100644 index 00000000000..b16171721f2 --- /dev/null +++ b/sdk/toyos/src/fd/table/model.rs @@ -0,0 +1,228 @@ +//! Calls beside the writer on two CPUs, under loom: every interleaving and +//! every memory order of a hold against a close, a reuse and a `dup2`. Each +//! case names the controls that red it. +//! +//! cargo test -p toyos --lib --features fd::table::model + +use core::sync::atomic::Ordering; + +use loom::sync::atomic::AtomicU32; +use loom::sync::{Arc, Mutex}; +use loom::thread; + +use super::{Error, Fault, Fd, Place, Slot, Table, Word, Writer}; + +struct Shared(AtomicU32); + +impl Word for Shared { + fn load(&self, order: Ordering) -> u32 { + self.0.load(order) + } + fn store(&self, value: u32, order: Ordering) { + self.0.store(value, order) + } + fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result { + self.0.compare_exchange(current, new, success, failure) + } + fn fetch_sub(&self, value: u32, order: Ordering) -> u32 { + self.0.fetch_sub(value, order) + } +} + +/// A slot's object, which loom fails on a read beside a write or one not +/// ordered after it. +struct Tracked(loom::cell::UnsafeCell); + +impl Place for Tracked { + type Object = usize; + + unsafe fn read(&self) -> usize { + // SAFETY: the caller's. + self.0.with(|object| unsafe { *object }) + } + + unsafe fn write(&self, object: usize) { + // SAFETY: the caller's. + self.0.with_mut(|place| unsafe { *place = object }); + } +} + +const N: usize = 2; +/// The objects a case installs, each named by its index. +const OBJECTS: usize = 3; + +struct World { + table: Table, + writing: Mutex<()>, + /// How often each object was let go. + gone: [AtomicU32; OBJECTS], +} + +impl World { + fn new() -> Arc { + Arc::new(Self { + table: Table { + fds: core::array::from_fn(|_| Shared(AtomicU32::new(0))), + slots: core::array::from_fn(|_| Slot { + refs: Shared(AtomicU32::new(0)), + object: Tracked(loom::cell::UnsafeCell::new(usize::MAX)), + }), + }, + writing: Mutex::new(()), + gone: core::array::from_fn(|_| AtomicU32::new(0)), + }) + } + + fn write(&self, f: impl FnOnce(&mut Writer<'_, Shared, Tracked, N>) -> Result) -> R { + let _held = self.writing.lock().unwrap(); + // SAFETY: the one writer while `writing` is held. + f(&mut unsafe { self.table.writer() }).expect("the writer was refused") + } + + fn let_go(&self, object: usize) { + assert_eq!(self.gone[object].fetch_add(1, Ordering::SeqCst), 0, "object {object} was let go twice"); + } + + fn released(&self, object: Option) { + if let Some(object) = object { + self.let_go(object); + } + } + + fn install(&self, object: usize, floor: Fd) -> Fd { + self.write(|w| w.install(object, false, floor)) + } + + fn close(&self, fd: Fd) { + self.released(self.write(|w| w.close(fd))); + } + + /// A call's hold of `fd`: its token and object. + fn hold(&self, fd: Fd) -> Option<(usize, usize)> { + match self.table.acquire(fd, |object| self.let_go(object)) { + Ok(held) => Some(held), + Err(Fault::Refused(Error::BadFd)) => None, + Err(fault) => panic!("a call answered {fault:?}"), + } + } + + /// The end of the call [`Self::hold`] answered, whose object was not let + /// go under it. + fn done(&self, (token, object): (usize, usize)) { + assert_eq!(self.gone[object].load(Ordering::SeqCst), 0, "a call held object {object} after it was let go"); + // SAFETY: a token `hold` answered, given back once. + self.released(unsafe { self.table.unuse(token) }.expect("a call's own token")); + } + + /// One call on `fd`: the object it held. + fn call(&self, fd: Fd) -> Option { + let held = self.hold(fd)?; + self.done(held); + Some(held.1) + } + + /// Every object in `objects` was let go once. + fn all_gone(&self, objects: impl IntoIterator) { + for object in objects { + assert_eq!(self.gone[object].load(Ordering::SeqCst), 1, "object {object} was not let go"); + } + } +} + +/// `fd-hold-revives` counts a call onto the slot the close freed, so the +/// call holds an object already let go, here and in the two cases below +/// that free a slot under a call. +#[test] +fn a_call_beside_a_close_holds_a_live_object_or_none() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + caller.join().unwrap(); + world.all_gone([0]); + }); +} + +/// Two calls give their references up beside each other and the close. +#[test] +fn two_calls_beside_a_close_let_the_object_go_once() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let held = world.hold(fd).expect("an open descriptor"); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + world.done(held); + caller.join().unwrap(); + world.all_gone([0]); + }); +} + +/// The close frees the descriptor's slot and another descriptor's object +/// takes it. `fd-hold-unchecked` keeps a reference without looking at the +/// descriptor again, so the call holds an object its descriptor never named; +/// `fd-publish-relaxed` stores the new count unordered after the object, so +/// the call reads an object it need not see written, here and in the next +/// case. +#[test] +fn a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + let other = world.install(1, fd + 1); + let held = caller.join().unwrap(); + assert!(matches!(held, None | Some(0)), "a call on a closed descriptor held {held:?}"); + world.close(other); + world.all_gone([0, 1]); + }); +} + +#[test] +fn a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new() { + loom::model(|| { + let world = World::new(); + let fd = world.install(0, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(fd) + }); + world.close(fd); + assert_eq!(world.install(1, 0), fd); + caller.join().unwrap(); + world.close(fd); + world.all_gone([0, 1]); + }); +} + +/// A descriptor is open throughout a `dup2` onto it. +#[test] +fn a_call_beside_a_dup2_holds_the_old_object_or_the_new() { + loom::model(|| { + let world = World::new(); + let old = world.install(0, 0); + let new = world.install(1, 0); + let caller = thread::spawn({ + let world = world.clone(); + move || world.call(new) + }); + world.released(world.write(|w| w.dup2(old, new, false))); + let held = caller.join().unwrap(); + assert!(matches!(held, Some(0 | 1)), "a call beside a dup2 held {held:?}"); + world.all_gone([1]); + world.close(old); + world.close(new); + world.all_gone([0]); + }); +} From 35d452b2a4310fcb8b9666983318582e73f927c7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 18:22:55 +0200 Subject: [PATCH 2/3] Review r1 of #1234: a fourth control, the reuse case widened, and comments that say what is true `fd-reuse-relaxed` makes the writer's load that finds a slot free `Relaxed`. Both reuse cases red with loom's causality violation, so the models have teeth on the edge "no use after close" rests on: the last call's read of an object before the writer's next write to its slot. The reuse case closes the second descriptor before the call ends, so it also covers a call left the object its own descriptor never named. The model implements `Word` on loom's atomic directly. `Cell` is `ObjectCell` and the owner's writer type `TableWriter`. The slot comment says two objects, not two descriptors, since a dup shares a slot. The model's header no longer claims every memory order (loom does not explore load buffering) or a map of controls it does not keep: build/ci.rs's `CONTROLS` is that map. Calls on two served files still meet on `files::owner`'s `OPEN` lock: recorded in issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md. Nothing the descriptor calls execute changes: a rename and a constant ordering compile to the same code. Co-Authored-By: Claude Opus 5.5 --- build/ci.rs | 4 ++ ...ved-files-meet-on-one-process-wide-lock.md | 25 +++++++++++ sdk/toyos/Cargo.toml | 3 ++ sdk/toyos/src/fd.rs | 16 ++++---- sdk/toyos/src/fd/table.rs | 21 +++++----- sdk/toyos/src/fd/table/model.rs | 41 ++++++++----------- 6 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md diff --git a/build/ci.rs b/build/ci.rs index bfee018ecbc..7bad31e168c 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -853,6 +853,10 @@ pub(crate) const CONTROLS: &[Control] = &[ Fails("fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot"), Fails("fd::table::model::a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new"), ]), + red(SDK, "fd-reuse-relaxed", None, &[ + Fails("fd::table::model::a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot"), + Fails("fd::table::model::a_call_on_a_reopened_descriptor_holds_the_old_object_or_the_new"), + ]), ]; /// Whether a control's run showed its teeth. diff --git a/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md b/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md new file mode 100644 index 00000000000..c03c55963ae --- /dev/null +++ b/issues/calls-on-two-served-files-meet-on-one-process-wide-lock.md @@ -0,0 +1,25 @@ +--- +status: open +kind: finding +opened: 2026-11-11 +--- + +# Calls on two served files meet on one process-wide lock + +A call through a descriptor holds its object without a lock (#1234). A served +file's object is an index into `files::owner`'s `OPEN` +(`sdk/toyos/src/files/owner.rs`), one `Lock` over every served file the +process holds. `held()` takes it in every call on a served file +(`on_fd_held`) to clone that file's `Arc`, and `release_held` takes it when +the descriptor table lets one go. So two threads reading two different served +files still meet on one lock word each call, which is the collision the +descriptor table's hold no longer has. The critical section is short, so a +collision costs a brief wait. Found by inspection in the review of #1234; +nothing has measured it. + +Owner: the SDK's served-file client (`toyos::files`). + +**Exit**: a call on one served file takes no lock that a call on another +served file takes, or a metal measurement of two threads on two served files +shows the lock costs nothing material, with that measurement cited here before +the file is deleted. diff --git a/sdk/toyos/Cargo.toml b/sdk/toyos/Cargo.toml index c4785a34e83..8e3f39d26e1 100644 --- a/sdk/toyos/Cargo.toml +++ b/sdk/toyos/Cargo.toml @@ -107,6 +107,9 @@ fd-hold-unchecked = [] # The writer stores a new object's count `Relaxed`, so a call that counts # itself on need not see the object written. fd-publish-relaxed = [] +# The writer finds a slot free by a `Relaxed` load, so its write of a new +# object need not follow the last call's read of the old one. +fd-reuse-relaxed = [] [lints.rust] # The guest's target: its stream sinks are compiled only for it. diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index 334ceff5902..c1051cf36de 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -391,14 +391,14 @@ fn file_group(label: &str) -> Option { mod owner { use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; - use super::table::{Cell, Fault, Table, Writer}; + use super::table::{Fault, ObjectCell, Table, Writer}; use super::*; use crate::endow::Endowments; use crate::sync::Lock; use crate::OwnedHandle; - type Descriptors = Table, MAX_FDS>; - type Names<'a> = Writer<'a, AtomicU32, Cell, MAX_FDS>; + type Descriptors = Table, MAX_FDS>; + type TableWriter<'a> = Writer<'a, AtomicU32, ObjectCell, MAX_FDS>; static TABLE: Descriptors = Table::new(); /// Held by whoever changes what a descriptor names. @@ -415,22 +415,22 @@ mod owner { /// `f` on the table's writer, whose first use installs every descriptor /// the process was endowed at its number. Nothing panics under the lock. - fn change(f: impl FnOnce(&mut Names<'_>) -> Result) -> Result { + fn change(f: impl FnOnce(&mut TableWriter<'_>) -> Result) -> Result { let endowed = Endowments::get(); let mut refused = 0; let answer = { let _held = WRITING.lock(); // SAFETY: the one writer while `WRITING` is held. - let mut names = unsafe { TABLE.writer() }; + let mut writer = unsafe { TABLE.writer() }; let installed = match ENDOWED.load(Ordering::Relaxed) { true => Ok(()), false => { - let installed = endow(&mut names, endowed).map(|n| refused = n); + let installed = endow(&mut writer, endowed).map(|n| refused = n); ENDOWED.store(true, Ordering::Release); installed } }; - installed.and_then(|()| f(&mut names)) + installed.and_then(|()| f(&mut writer)) }; // Said once the lock is let go: the write may block. for _ in 0..refused { @@ -452,7 +452,7 @@ mod owner { /// this process's holder first. One whose number another already took, /// or whose socket this process holds no room for, is refused, its handle /// closed: how many were. - fn endow(table: &mut Names<'_>, endowed: &Endowments) -> Result { + fn endow(table: &mut TableWriter<'_>, endowed: &Endowments) -> Result { let mut refused = 0; let mut files = alloc::collections::BTreeMap::new(); for label in endowed.labels() { diff --git a/sdk/toyos/src/fd/table.rs b/sdk/toyos/src/fd/table.rs index 4fd1a42e969..c7b6312e545 100644 --- a/sdk/toyos/src/fd/table.rs +++ b/sdk/toyos/src/fd/table.rs @@ -67,12 +67,12 @@ pub(super) trait Place { unsafe fn write(&self, object: Self::Object); } -pub(super) struct Cell(UnsafeCell>); +pub(super) struct ObjectCell(UnsafeCell>); // SAFETY: reached only through `Place`, whose callers keep a write alone. -unsafe impl Sync for Cell {} +unsafe impl Sync for ObjectCell {} -impl Place for Cell { +impl Place for ObjectCell { type Object = O; unsafe fn read(&self) -> O { @@ -100,8 +100,8 @@ impl From for Fault { } } -/// An object's slot, on a cache line of its own: calls on two descriptors -/// write no line they share. +/// An object's slot, on a cache line of its own: calls on two objects write +/// no line they share. #[repr(align(64))] struct Slot { /// The slot's count of references. @@ -114,12 +114,12 @@ pub(super) struct Table { slots: [Slot; N], } -impl Table, N> { +impl Table, N> { pub(super) const fn new() -> Self { assert!(N < INHERITABLE as usize); Self { fds: [const { AtomicU32::new(0) }; N], - slots: [const { Slot { refs: AtomicU32::new(0), object: Cell(UnsafeCell::new(MaybeUninit::uninit())) } }; N], + slots: [const { Slot { refs: AtomicU32::new(0), object: ObjectCell(UnsafeCell::new(MaybeUninit::uninit())) } }; N], } } } @@ -254,7 +254,8 @@ impl Writer<'_, W, P, N> { /// A free slot holding `object`, referenced once. fn hold(&mut self, object: P::Object) -> Result { // The load that finds a slot free is ordered after its last reader. - let index = self.0.slots.iter().position(|slot| slot.refs.load(Ordering::Acquire) == 0).ok_or(Error::Full)?; + let free = if cfg!(feature = "fd-reuse-relaxed") { Ordering::Relaxed } else { Ordering::Acquire }; + let index = self.0.slots.iter().position(|slot| slot.refs.load(free) == 0).ok_or(Error::Full)?; let slot = &self.0.slots[index]; // SAFETY: a free slot is this writer's alone until its count is stored. unsafe { slot.object.write(object) }; @@ -360,8 +361,8 @@ mod tests { Record { handle: toyos_abi::HANDLE_INVALID, tag: super::super::UNTAGGED, word, release: Some(nothing) } } - type T = Table, 8>; - type W<'a> = Writer<'a, AtomicU32, Cell, 8>; + type T = Table, 8>; + type W<'a> = Writer<'a, AtomicU32, ObjectCell, 8>; /// These tests' one writer of `t`. fn writer(t: &T) -> W<'_> { diff --git a/sdk/toyos/src/fd/table/model.rs b/sdk/toyos/src/fd/table/model.rs index b16171721f2..bc72a74323c 100644 --- a/sdk/toyos/src/fd/table/model.rs +++ b/sdk/toyos/src/fd/table/model.rs @@ -1,6 +1,7 @@ -//! Calls beside the writer on two CPUs, under loom: every interleaving and -//! every memory order of a hold against a close, a reuse and a `dup2`. Each -//! case names the controls that red it. +//! Calls beside the writer on two CPUs, under loom: every interleaving it +//! explores of a hold against a close, a reuse and a `dup2`. Loom does not +//! explore load buffering. `build/ci.rs`'s `CONTROLS` names the cases each +//! control reds. //! //! cargo test -p toyos --lib --features fd::table::model @@ -12,20 +13,18 @@ use loom::thread; use super::{Error, Fault, Fd, Place, Slot, Table, Word, Writer}; -struct Shared(AtomicU32); - -impl Word for Shared { +impl Word for AtomicU32 { fn load(&self, order: Ordering) -> u32 { - self.0.load(order) + AtomicU32::load(self, order) } fn store(&self, value: u32, order: Ordering) { - self.0.store(value, order) + AtomicU32::store(self, value, order) } fn compare_exchange(&self, current: u32, new: u32, success: Ordering, failure: Ordering) -> Result { - self.0.compare_exchange(current, new, success, failure) + AtomicU32::compare_exchange(self, current, new, success, failure) } fn fetch_sub(&self, value: u32, order: Ordering) -> u32 { - self.0.fetch_sub(value, order) + AtomicU32::fetch_sub(self, value, order) } } @@ -52,7 +51,7 @@ const N: usize = 2; const OBJECTS: usize = 3; struct World { - table: Table, + table: Table, writing: Mutex<()>, /// How often each object was let go. gone: [AtomicU32; OBJECTS], @@ -62,9 +61,9 @@ impl World { fn new() -> Arc { Arc::new(Self { table: Table { - fds: core::array::from_fn(|_| Shared(AtomicU32::new(0))), + fds: core::array::from_fn(|_| AtomicU32::new(0)), slots: core::array::from_fn(|_| Slot { - refs: Shared(AtomicU32::new(0)), + refs: AtomicU32::new(0), object: Tracked(loom::cell::UnsafeCell::new(usize::MAX)), }), }, @@ -73,7 +72,7 @@ impl World { }) } - fn write(&self, f: impl FnOnce(&mut Writer<'_, Shared, Tracked, N>) -> Result) -> R { + fn write(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> R { let _held = self.writing.lock().unwrap(); // SAFETY: the one writer while `writing` is held. f(&mut unsafe { self.table.writer() }).expect("the writer was refused") @@ -129,9 +128,6 @@ impl World { } } -/// `fd-hold-revives` counts a call onto the slot the close freed, so the -/// call holds an object already let go, here and in the two cases below -/// that free a slot under a call. #[test] fn a_call_beside_a_close_holds_a_live_object_or_none() { loom::model(|| { @@ -165,12 +161,9 @@ fn two_calls_beside_a_close_let_the_object_go_once() { }); } -/// The close frees the descriptor's slot and another descriptor's object -/// takes it. `fd-hold-unchecked` keeps a reference without looking at the -/// descriptor again, so the call holds an object its descriptor never named; -/// `fd-publish-relaxed` stores the new count unordered after the object, so -/// the call reads an object it need not see written, here and in the next -/// case. +/// The close frees the descriptor's slot, another descriptor's object takes +/// it and is closed in turn, so a call that counted itself onto the slot can +/// be left the object its own descriptor never named. #[test] fn a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot() { loom::model(|| { @@ -182,9 +175,9 @@ fn a_call_on_a_closed_descriptor_never_holds_the_object_that_took_its_slot() { }); world.close(fd); let other = world.install(1, fd + 1); + world.close(other); let held = caller.join().unwrap(); assert!(matches!(held, None | Some(0)), "a call on a closed descriptor held {held:?}"); - world.close(other); world.all_gone([0, 1]); }); } From dafb2e86071f0a487604e1350b6ce647e9e65df6 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 18:47:28 +0200 Subject: [PATCH 3/3] A descriptor naming a free slot is settled under the writer, not spun on A call whose descriptor names a free slot reread the word until it changed. That state is a close in flight or a broken table, and only the writer can tell the two apart, so a broken table spun the call forever where main panicked. The table's `acquire` now answers `None` there, and the owner holds the descriptor under the writer's lock once: a closed descriptor is refused, and a broken table panics once the lock is let go. The settling call is cold, out of line and tail-called, so the lock-free loop keeps its shape: one more callee-saved register saved and a frame of 0x20 bytes instead of 0x18, against d8f9a490b, with the instructions otherwise the same. The A/B on metal measured d8f9a490b. Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/src/fd.rs | 14 ++++++++- sdk/toyos/src/fd/table.rs | 52 +++++++++++++++++++++++++-------- sdk/toyos/src/fd/table/model.rs | 15 ++++++++-- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index c1051cf36de..711897b4afb 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -514,9 +514,21 @@ mod owner { }) } + /// A call whose descriptor named a free slot, held under the writer's + /// lock, where that is a finished close or a broken table. + #[cold] + #[inline(never)] + fn settled(fd: Fd) -> Result<(usize, Record), Error> { + change(|t| t.acquire(fd)) + } + #[no_mangle] unsafe extern "C" fn toyos_fd_acquire(fd: Fd, out: *mut Record) -> i32 { - let held = table().and_then(|t| t.acquire(fd, |record| released(Some(record))).map_err(refusal)); + let held = table().and_then(|t| match t.acquire(fd, |record| released(Some(record))) { + Ok(Some(held)) => Ok(held), + Ok(None) => settled(fd), + Err(fault) => Err(refusal(fault)), + }); match held { Ok((token, object)) => { // SAFETY: the caller's live place. diff --git a/sdk/toyos/src/fd/table.rs b/sdk/toyos/src/fd/table.rs index c7b6312e545..ca5d98d995c 100644 --- a/sdk/toyos/src/fd/table.rs +++ b/sdk/toyos/src/fd/table.rs @@ -169,22 +169,21 @@ impl Table { } } - /// `fd`'s object, held until [`Self::unuse`] of the token answered. An - /// object whose last reference a lost race left with this call goes to - /// `let_go`. - pub(super) fn acquire(&self, fd: Fd, mut let_go: impl FnMut(P::Object)) -> Result<(usize, P::Object), Fault> { + /// `fd`'s object, held until [`Self::unuse`] of the token answered, or + /// `None` for a descriptor naming a free slot: a close in flight or a + /// broken table, which only [`Writer::acquire`] tells apart. An object + /// whose last reference a lost race left with this call goes to `let_go`. + pub(super) fn acquire(&self, fd: Fd, mut let_go: impl FnMut(P::Object)) -> Result, Fault> { let word = self.word(fd)?; loop { let slot = Self::named(word.load(Ordering::Acquire))?; - // A free slot was closed since the word was read, which the next - // read of the word shows. if !self.reference(slot)? { - continue; + return Ok(None); } // SAFETY: the reference just taken. let object = unsafe { self.slots[slot].object.read() }; if cfg!(feature = "fd-hold-unchecked") || Self::named(word.load(Ordering::Acquire)) == Ok(slot) { - return Ok((slot, object)); + return Ok(Some((slot, object))); } // SAFETY: the reference just taken. if let Some(last) = unsafe { self.unref(slot) }? { @@ -197,8 +196,8 @@ impl Table { /// names it any more. /// /// # Safety - /// `token` is one this table's [`Self::acquire`] answered, given back - /// once. + /// `token` is one this table's [`Self::acquire`] or [`Writer::acquire`] + /// answered, given back once. pub(super) unsafe fn unuse(&self, token: usize) -> Result, Fault> { // SAFETY: the call's reference. unsafe { self.unref(token) } @@ -276,6 +275,16 @@ impl Writer<'_, W, P, N> { unsafe { self.0.unref(slot) } } + /// `fd`'s object for a call, held as [`Table::acquire`] holds it. Under + /// the writer no slot a descriptor names is freed, so one named free is + /// a broken table. + pub(super) fn acquire(&mut self, fd: Fd) -> Result<(usize, P::Object), Fault> { + let (_, slot, _) = self.entry(fd)?; + self.reference(slot)?; + // SAFETY: the reference just taken. + Ok((slot, unsafe { self.0.slots[slot].object.read() })) + } + /// Name `object` by the lowest free descriptor at or above `floor`. pub(super) fn install(&mut self, object: P::Object, inheritable: bool, floor: Fd) -> Result { let fd = self.lowest_free(floor)?; @@ -376,7 +385,8 @@ mod tests { } fn acquire(t: &T, fd: Fd) -> Result<(usize, Record), Fault> { - t.acquire(fd, |_| panic!("a call with no race was left an object")) + let held = t.acquire(fd, |_| panic!("a call with no race was left an object"))?; + Ok(held.expect("a call with no race found a free slot")) } /// The objects something still references. @@ -498,8 +508,24 @@ mod tests { assert_eq!(w(t.close(0)), Ok(Some(1))); } + /// A call the writer answers holds its object as any call does. + #[test] + fn a_call_the_writer_answers_holds_its_object_until_it_is_done() { + let table = T::new(); + let mut t = writer(&table); + let fd = t.install(o(5), false, 0).unwrap(); + let (token, object) = t.acquire(fd).unwrap(); + assert_eq!(object.word, 5); + assert_eq!(w(t.close(fd)), Ok(None), "the object went under a call"); + assert!(matches!(t.acquire(fd), Err(Fault::Refused(Error::BadFd))), "a closed descriptor still answered"); + // SAFETY: a token the writer's `acquire` answered, given back once. + assert_eq!(w(unsafe { table.unuse(token) }), Ok(Some(5))); + assert_eq!(held(&table), 0, "an object outlived its call"); + } + /// A descriptor naming no slot, or a free one under the writer, is a - /// broken table, which the table answers rather than panics on. + /// broken table, which the table answers rather than panics on. A call + /// whose descriptor names a free slot is left to the writer. #[test] fn an_inconsistent_table_is_answered_not_panicked_on() { let table = T::new(); @@ -510,6 +536,8 @@ mod tests { table.fds[2].store(INHERITABLE, Ordering::Relaxed); assert_eq!(table.inheritable(2), Err(Fault::Broken)); table.fds[2].store(1, Ordering::Relaxed); + assert!(matches!(table.acquire(2, |_| panic!("a call with no race was left an object")), Ok(None))); + assert_eq!(t.acquire(2).map(|_| ()), Err(Fault::Broken)); assert_eq!(w(t.close(2)), Err(Fault::Broken)); assert_eq!(t.dup(2, 0, false), Err(Fault::Broken)); } diff --git a/sdk/toyos/src/fd/table/model.rs b/sdk/toyos/src/fd/table/model.rs index bc72a74323c..497bb0a7541 100644 --- a/sdk/toyos/src/fd/table/model.rs +++ b/sdk/toyos/src/fd/table/model.rs @@ -72,10 +72,14 @@ impl World { }) } - fn write(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> R { + fn writer(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> Result { let _held = self.writing.lock().unwrap(); // SAFETY: the one writer while `writing` is held. - f(&mut unsafe { self.table.writer() }).expect("the writer was refused") + f(&mut unsafe { self.table.writer() }) + } + + fn write(&self, f: impl FnOnce(&mut Writer<'_, AtomicU32, Tracked, N>) -> Result) -> R { + self.writer(f).expect("the writer was refused") } fn let_go(&self, object: usize) { @@ -98,7 +102,12 @@ impl World { /// A call's hold of `fd`: its token and object. fn hold(&self, fd: Fd) -> Option<(usize, usize)> { - match self.table.acquire(fd, |object| self.let_go(object)) { + let held = match self.table.acquire(fd, |object| self.let_go(object)) { + Ok(Some(held)) => Ok(held), + Ok(None) => self.writer(|w| w.acquire(fd)), + Err(fault) => Err(fault), + }; + match held { Ok(held) => Some(held), Err(Fault::Refused(Error::BadFd)) => None, Err(fault) => panic!("a call answered {fault:?}"),