From 2583d7de81951194dddd818963afbe8667b7401d Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 17:59:45 +0200 Subject: [PATCH 1/3] A futex wait translates its word once and checks it with one load `SYS_FUTEX_WAIT` translated the word under its space's lock, and then `wait_until`'s predicate translated it again under the same lock on every check: before arming, after arming, after each wake and at the deadline, each with an `Arc` clone of the current space. wakeperf's segments put the translation at 60 to 63 ns, two to four times a round trip. The wait now translates once (`Space::translate`), and the translation carries the space's count of leaf changes read in that same hold. Every change to a present leaf is shot down, and `AddressSpace::shoot_down` now counts it first, under the lock, before the shootdown, the futex revoke and the free; `build`'s unwinding and `discard` go through it too, so their shootdowns are also counted in the test kernel's `SPACE_REACHED`. The predicate reads the word through the direct map and then asks `Translation::holds`: an acquire fence and a load of the count. While the count is unchanged the address still maps the frame and the frame is not freed. When it has moved, one hold of the lock asks whether the address still maps the frame, as every check did before, and takes the new count if it does, so a change elsewhere in the space neither ends the wait nor turns a timeout into a wake. What keeps a waiter from sleeping on a frame an unmap freed is unchanged: the unmap ends the waits on its frames under the bucket locks after it counts the change, and a waiter arms under its bucket's lock before it checks, so either the revoke finds it armed or its check sees the count moved and the address no longer mapping the frame. `futex::watch_of` and `scheduler::futex_wake` take the physical address rather than a direct-map pointer. Co-Authored-By: Claude Opus 5.5 --- kernel/src/mm/space.rs | 61 ++++++++++++++++++++++++++++++++++++--- kernel/src/process.rs | 14 ++++----- kernel/src/sched/futex.rs | 7 ++--- kernel/src/scheduler.rs | 50 ++++++++++---------------------- kernel/src/user_ptr.rs | 10 +++---- 5 files changed, 87 insertions(+), 55 deletions(-) diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 65e662cfea7..e860ed8c6d2 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -21,7 +21,8 @@ //! waits on the lock, whose spin answers the shootdown, and finds it gone. use alloc::sync::Arc; -use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use core::cell::Cell; +use core::sync::atomic::{fence, AtomicU64, Ordering::{Acquire, Relaxed}}; use kernel::vm::{self, Access, Entry, Kind, Prot}; use toyos_abi::syscall::SyscallError; @@ -82,7 +83,8 @@ impl At { } /// What a space holds, read without its lock: by `SYS_SYSINFO`, by a -/// process's statistics, and by its end's record. +/// process's statistics, and by its end's record; and how often its leaves +/// changed, by a futex wait ([`Translation`]). #[derive(Default)] pub struct Usage { /// Private frames populated now. @@ -94,6 +96,10 @@ pub struct Usage { /// Faults served, and the time they took. faults: AtomicU64, fault_ns: AtomicU64, + /// Changes to a leaf a CPU may have cached: every one is shot down, and + /// counted under the lock before its shootdown, so before a frame it + /// gave up is freed. + shot_down: AtomicU64, /// Changes to the space's leaves that reached another CPU by interrupt, /// for the test kernel's `SYS_DEBUG`. #[cfg(feature = "test-actuators")] @@ -190,6 +196,52 @@ impl Space { pub fn release_object(&self, at: UserAddr, len: u64, object: &SharedMemObject) { self.lock().release_object(at, len, object); } + + /// The frame behind `va`, populated where its range allows `access` + /// ([`AddressSpace::reach`]), as a translation a holder can ask about + /// later without the lock. + pub fn translate(space: PageTables, va: u64, access: Access) -> Option { + let (phys, at) = { + let mut held = space.lock(); + let phys = held.reach(va, access)?; + // Counted under this hold: a change made before it is in `at`, + // and one after it moves the count. + (phys, space.usage.shot_down.load(Relaxed)) + }; + Some(Translation { space, va, phys, at: Cell::new(at) }) + } +} + +/// A user address's frame as a hold of its space's lock found it. +pub struct Translation { + space: PageTables, + va: u64, + phys: u64, + /// The space's count of changes when `va` was last found mapping `phys`. + at: Cell, +} + +impl Translation { + pub fn phys(&self) -> u64 { + self.phys + } + + /// Whether the address still maps the frame, which is then not freed: no + /// leaf of the space has changed since it was last found there, or a hold + /// of the lock finds it there again. A load before this one is ordered + /// before it, so a value read from the frame was read while this held. + pub fn holds(&self) -> bool { + fence(Acquire); + if self.space.usage.shot_down.load(Relaxed) == self.at.get() { + return true; + } + let held = self.space.lock(); + let found = held.get(self.va).is_some_and(|(phys, _)| phys == self.phys); + if found { + self.at.set(self.space.usage.shot_down.load(Relaxed)); + } + found + } } pub struct AddressSpace { @@ -379,7 +431,7 @@ impl AddressSpace { let inventory = &self.inventory; let unlinked = self.tables.unlink(start, end, |s, e| inventory.backed(s, e)); if !unlinked.is_empty() { - self.tables.shoot_down(start, end, &unlinked); + self.shoot_down(start, end, &unlinked); } self.tables.free(unlinked); Err(SyscallError::ResourceExhausted) @@ -526,7 +578,7 @@ impl AddressSpace { Leaf::Held(_) | Leaf::Empty => None, }); if present { - self.tables.shoot_down(start, end, &Unlinked::none()); + self.shoot_down(start, end, &Unlinked::none()); } let mut freed = 0; let inventory = &self.inventory; @@ -684,6 +736,7 @@ impl AddressSpace { /// Every CPU has dropped what it cached of `[start, end)` and of the /// tables `unlinked` took out. fn shoot_down(&self, start: u64, end: u64, unlinked: &Unlinked) { + self.usage.shot_down.fetch_add(1, Relaxed); let _reached = self.tables.shoot_down(start, end, unlinked); #[cfg(feature = "test-actuators")] if _reached { diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 69cd1b45df9..eaa15eb2231 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -942,9 +942,9 @@ pub fn thread_sched(pid: Pid, tid: Tid) -> Option { table.get(pid)?.threads.get(&tid)?.sched().cloned() } -/// The physical address of a futex word, its page populated like any other user access. Alignment matters: the scheduler dereferences this directly on every wake check, so an unaligned word could straddle into the next physical page. -/// Not a lease: nothing pins the frame, and a sibling's `munmap` can free it under a parked wait. Made safe by an unmap ending the waits on the frames it frees (`sched::futex::revoke_frames`), and `scheduler::futex_wait` re-deriving the translation on every check. -fn futex_word(addr: UserAddr) -> Option { +/// A futex word's frame, its page populated like any other user access. Alignment matters: the scheduler reads the word through the frame, so an unaligned word could straddle into the next physical page. +/// Not a lease: nothing pins the frame, and a sibling's `munmap` can free it under a parked wait. Made safe by an unmap ending the waits on the frames it frees (`sched::futex::revoke_frames`) after it counts the change the translation's `holds` reads. +fn futex_word(addr: UserAddr) -> Option { if !addr.raw().is_multiple_of(4) { return None; } @@ -962,12 +962,12 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { }; // Physical, so a futex in shared memory works across processes. - let Some(phys_addr) = futex_word(addr) else { + let Some(word) = futex_word(addr) else { return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); }; // 0 covers both woken and never-blocked (the caller re-checks the word either way); 1 means only the caller's own deadline ended it. - match scheduler::futex_wait(addr, phys_addr, expected, deadline) { + match scheduler::futex_wait(&word, expected, deadline) { scheduler::FutexEnd::Changed => 0, scheduler::FutexEnd::Timeout => 1, } @@ -975,10 +975,10 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { /// Wake up to `count` threads blocked on the same physical address as `addr`. pub fn futex_wake(addr: UserAddr, count: u64) -> u64 { - let Some(phys_addr) = futex_word(addr) else { + let Some(word) = futex_word(addr) else { return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); }; - scheduler::futex_wake(phys_addr, count as usize) + scheduler::futex_wake(word.phys(), count as usize) } /// Wake processes blocked on reading from a pipe that now has data. diff --git a/kernel/src/sched/futex.rs b/kernel/src/sched/futex.rs index 2dbdd006e3d..40dd1ae0176 100644 --- a/kernel/src/sched/futex.rs +++ b/kernel/src/sched/futex.rs @@ -2,15 +2,14 @@ //! address so every process mapping a word shares it. use crate::watch::Watch; -use crate::DirectMap; // A collision only lengthens the walk: a futex wake is bounded by the waiter's token, not by which bucket it landed in. const FUTEX_BUCKETS: usize = 64; static FUTEX_WATCH: [Watch; FUTEX_BUCKETS] = [const { Watch::new() }; FUTEX_BUCKETS]; -/// The watch for a futex word. -pub fn watch_of(addr: DirectMap) -> &'static Watch { - &FUTEX_WATCH[(addr.phys() >> 2) as usize % FUTEX_BUCKETS] +/// The watch for the futex word at `phys`. +pub fn watch_of(phys: u64) -> &'static Watch { + &FUTEX_WATCH[(phys >> 2) as usize % FUTEX_BUCKETS] } /// End every futex wait on a word in one of `frames`, ascending 4 KiB frames diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index 42437e6d6f2..7cecf63d7ed 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -254,43 +254,25 @@ pub fn set_current_rt(enable: bool) { /// Block on a futex word unless it already changed, and answer which of the /// two things the ABI names ended the wait. #[track_caller] -pub fn futex_wait( - addr: crate::UserAddr, - phys_addr: DirectMap, - expected: u32, - deadline: Deadline, -) -> FutexEnd { +pub fn futex_wait(word: &crate::mm::space::Translation, expected: u32, deadline: Deadline) -> FutexEnd { let parkable = Parkable::at_entry(); - // Re-translated rather than trusted: `munmap` clears the entry before - // walking futex buckets, so a changed translation means this arm is stale. - let read = || { - let Some(pt) = current_address_space() else { - return true; - }; - let same_frame = pt.lock().get(addr.raw()).is_some_and(|(now, _)| now == phys_addr.phys()); - if !same_frame { - return true; - } - // SAFETY: `same_frame` re-translated `addr` and found it still names - // `phys_addr`'s frame, so this is a live, mapped, syscall-checked - // 4-byte-aligned word; volatile because this predicate may run more - // than once. - let word = unsafe { phys_addr.as_ptr::().read_volatile() }; - word != expected + let phys = word.phys(); + let changed = || { + // SAFETY: a frame the user's checked, 4-byte-aligned word translated + // to, read through the direct map, which maps every frame; volatile + // because this predicate may run more than once. Whether the frame + // was still the word's when read is `holds`, after the read. + let now = unsafe { DirectMap::from_phys(phys).as_ptr::().read_volatile() }; + // An address that no longer maps the frame is a wait the unmap's + // revoke may have missed: the word moved, and the caller re-checks it. + now != expected || !word.holds() }; - let _ = watch::wait_until( - &parkable, - futex::watch_of(phys_addr), - phys_addr.phys(), - WaitClass::Futex, - deadline, - read, - ); + let _ = watch::wait_until(&parkable, futex::watch_of(phys), phys, WaitClass::Futex, deadline, changed); // `wait_until` answers `Ok(())` for a satisfied predicate and an expired // deadline alike, so the word itself is what tells them apart. With no // deadline the word or its frame changed, or a kill ended it, whose answer // never reaches Ring 3. - if deadline.is_never() || read() { + if deadline.is_never() || changed() { FutexEnd::Changed } else { FutexEnd::Timeout @@ -306,9 +288,9 @@ pub enum FutexEnd { Timeout, } -/// Wake up to `count` waiters on this futex word, and answer how many. -pub fn futex_wake(phys_addr: DirectMap, count: usize) -> u64 { - futex::watch_of(phys_addr).post_n(phys_addr.phys(), count) as u64 +/// Wake up to `count` waiters on the futex word at `phys`, and answer how many. +pub fn futex_wake(phys: u64, count: usize) -> u64 { + futex::watch_of(phys).post_n(phys, count) as u64 } /// Set a thread's kill bit and ask its CPU for a safe point; returns at once. diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index e129405bf9c..6a0faf362cd 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -45,12 +45,10 @@ fn asked(access: Access) -> Asked { } } -/// Translate a user virtual address to its direct-map address, populating its -/// page if needed; `pub(crate)` because the futex word outlives its syscall. -pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { - let space = unlocked_space(); - let phys = space.lock().reach(addr.raw(), asked(access))?; - Some(crate::mm::DirectMap::from_phys(phys)) +/// Translate a user virtual address to its frame, populating its page if +/// needed; `pub(crate)` because the futex word outlives its syscall. +pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { + crate::mm::space::Space::translate(unlocked_space(), addr.raw(), asked(access)) } /// The current space, for a reach made holding no lock. From bb5ae53971755f7b93145c62ecbcc066d27d9302 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 19:08:38 +0200 Subject: [PATCH 2/3] Review r1 of #1253: the futex read's SAFETY names the frames the direct map holds Since #1211 the direct map leaves out the bring-up guard pages, so it no longer maps every frame; it maps every frame a user page can hold, which is what the read relies on. Co-Authored-By: Claude Opus 5.5 --- kernel/src/scheduler.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/src/scheduler.rs b/kernel/src/scheduler.rs index 7cecf63d7ed..b755e49d73f 100644 --- a/kernel/src/scheduler.rs +++ b/kernel/src/scheduler.rs @@ -259,9 +259,10 @@ pub fn futex_wait(word: &crate::mm::space::Translation, expected: u32, deadline: let phys = word.phys(); let changed = || { // SAFETY: a frame the user's checked, 4-byte-aligned word translated - // to, read through the direct map, which maps every frame; volatile - // because this predicate may run more than once. Whether the frame - // was still the word's when read is `holds`, after the read. + // to, read through the direct map, which maps every frame a user page + // can hold; volatile because this predicate may run more than once. + // Whether the frame was still the word's when read is `holds`, after + // the read. let now = unsafe { DirectMap::from_phys(phys).as_ptr::().read_volatile() }; // An address that no longer maps the frame is a wait the unmap's // revoke may have missed: the word moved, and the caller re-checks it. From 1ec47d7553df4e6436e16f8d1d0344ac5f502734 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:59:34 +0200 Subject: [PATCH 3/3] issues: the futex key path's whole-boot rows and A/B are owed on metal Both benches are down. #1253 lands on its review, host checks and guest rows, and `issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md`, the record of the futex round trip it targets, records what only metal can establish: the T14's whole `testcases`, `testcases-watchdog` and `shared-debug` boots of main after it, and the A/B of main before and after it on both benches with the Linux comparison. Its speed-up is not yet measured. Co-Authored-By: Claude Opus 5.5 --- ...sts-toyos-up-to-twice-linuxs-on-the-t14.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md b/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md index 33350fbedfd..23f273e90fd 100644 --- a/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md +++ b/issues/a-two-thread-ping-pong-costs-toyos-up-to-twice-linuxs-on-the-t14.md @@ -68,6 +68,25 @@ where the row measured what it means, it is behind as before. That neither thread slept in the short rounds is an inference from the timings; the row counts no parks, so a run's p50 is not this row's value. +#1253 translates a futex wait's word once per call and checks it with one +load of its space's change count, where every check took the space's lock +and walked its tables again (wakeperf: 60 to 63 ns a translation, two to four +times a round trip). Its speed-up is not yet measured on metal. + +- **Owed on metal:** the T14's whole `testcases`, `testcases-watchdog` and + `shared-debug` boots of main after #1253. Main must pass every member, + among them `futex_wake_counts` (whose `orphaned_by_unmap` arm is a wait on + a frame an unmap frees), `kill_ends_every_wait`, `exit_wait_storm`, + `std_threading`, `std_threads`, `shm_release_reclaims` and + `tlb_shootdown_waits`. +- **Owed on metal:** an A/B of main before and after #1253, on the T14 and + the AMD bench: `osbench sched`, `time`, `files` and `ipc`, four boots a + side in rotated order, each boot's p50, compared by an exact two-sided + Mann-Whitney test; and Linux's `osbench sched` on each bench. Main after + must read `pingpong.futex.same_cpu` and `pingpong.futex.cross_core` lower + or not separated, the latter only from boots whose round trips parked, and + no other row 5% or more worse unless the code #1253 changed explains it. + **Exit**: `pingpong.*` within Linux's on both benches in a labelled current-state comparison, or each remaining difference measured to its parts and justified; and `pingpong.futex.*` reports the parks of each round, so a