From 8731b292bc605a56ef36b914fe271a221d845933 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 19:34:46 +0200 Subject: [PATCH 01/34] A removed page keeps its frame until every mapping of its object let it go 7b step 2, first part: the paged index's revocation, before any space maps a paged object. A withdraw, failure, shrink or the object's end now marks a page revoking instead of removing it. A revoking page reads as absent, takes no supply (answered present) and no write, keeps its slot and its frame, and its dirty count goes at once. One walk per object visits each mapper (`kernel::paged::Mappers`) by epoch; a removal that finds a walk running is taken by another pass of that walk, known to its caller by the pass's epoch; a mapper linked during a pass maps nothing revoking and is passed by. The walk's end gives each frame out with what its page became: absent inside the object, failed, or removed past the size or with the object. The re-report of #1233 follows the pages: the walk's end names again every waiter's want a page that ends absent meets, and a withdraw names them for the failed pages it removes at once, which have no frame to revoke; both take the waiters (`Waiters`). The kernel's removals go through it. No space maps a paged object yet, so each walk ends in the hold of the lock that began it. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 547 ++++++++++++++++++++++++++++++------- kernel/src/object/paged.rs | 80 ++++-- 2 files changed, 499 insertions(+), 128 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index cf2f1e7b70..5394773439 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -2,11 +2,20 @@ //! serves, in one table of `capacity` slots made when the pager is, so no //! supply, read, write or write-back allocates. //! -//! **A page is present, failed or absent.** A present page holds a frame and -//! its dirty state; a failed one holds no frame and answers `EIO` until it is -//! supplied again; an absent one has no slot. Unwritten anonymous memory -//! reads zeros without a slot, and its present pages cannot be withdrawn. Nothing here waits: a caller -//! that finds a page absent records its request in its own wait record. +//! **A page is present, failed, revoking or absent.** A present page holds a +//! frame and its dirty state; a failed one holds no frame and answers `EIO` +//! until it is supplied again; an absent one has no slot. Unwritten anonymous +//! memory reads zeros without a slot, and its present pages cannot be +//! withdrawn. Nothing here waits: a caller that finds a page absent records +//! its request in its own wait record. +//! +//! **A removed page keeps its frame until no mapping reaches it.** A +//! withdraw, failure, shrink or the object's end marks the page revoking: it +//! reads as absent, takes no supply or write, and keeps its slot and frame, +//! so no page is ever two frames at once. One walk per object visits each of +//! its mappers ([`Mappers`]) after the marking, and a removal that finds a +//! walk running is taken by that walk's next pass; the last pass's end gives +//! every revoking page's frame out ([`Pages::walk`]). //! //! **A dirty page holds a slot until a write-back cleans it.** Slots are //! bounded by `dirty_ceiling`. A write-back begin clears the dirty bit and @@ -25,7 +34,7 @@ //! call until [`Object::end_write`]: the pager's whole write. //! //! What a removed frame owes (the last pin's free, a zeroed tail) is the -//! caller's: every removal answers its frames. +//! caller's: the walk's end answers every frame. #![forbid(unsafe_code)] @@ -246,6 +255,52 @@ pub enum Supplied { Present, } +/// What a revoking page becomes once no mapping reaches its frame, by the +/// object as the walk's end finds it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Revoked { + /// Absent inside the object: a reader waiting on it is named again. + Absent, + Failed, + /// Past the size, or the object is gone. + Removed, +} + +/// The mappers of one object, one node per space that maps it, in the order +/// a walk visits them. A node links at the tail, seen at +/// [`Object::epoch`] then, and leaves only once its space maps none of the +/// object's frames. +pub trait Mappers { + type Node: Copy; + fn head(&self) -> Option; + /// The epoch of the walk that last visited `node`, or the one current + /// when it linked. + fn seen(&self, node: Self::Node) -> u64; + /// `node` is visited at `epoch`: it moves to the tail. + fn visited(&mut self, node: Self::Node, epoch: u64); +} + +/// Where an object's walk stands. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Walk { + Idle, + /// A walker visits its mappers; `again` where pages were marked since + /// its pass began, which another pass covers. + Running { again: bool }, +} + +/// What the caller that marked an object's pages does about their mappers. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Revoke { + /// Nothing is revoking. + Done, + /// No walk ran: the caller walks it ([`Pages::walk`]) to its end. + Walk, + /// The running walk takes the pages in a later pass: they are gone once + /// [`Object::finished`] answers true of this epoch. + Queued(u64), +} + #[derive(Clone, Copy)] struct State { dirty: bool, @@ -268,6 +323,8 @@ struct Slot { /// `None` for a failed page. frame: Option, state: State, + /// Removed, its frame kept until no mapping reaches it. + revoking: Option, prev: u32, next: u32, } @@ -287,6 +344,13 @@ pub struct Object { writing: u32, /// A call that did not take all it was given left its write under way. held: bool, + /// Its slots that are revoking. + revoking: usize, + /// The pass of the walk over its mappers, moved at each pass's start. + epoch: u64, + walk: Walk, + /// The epoch whose pass last ended the walk. + finished: u64, } /// What a holder's read began at ([`Object::read_begin`]). @@ -347,6 +411,41 @@ impl Object { self.write_end(true); Ok(()) } + + /// The epoch a mapper linking now is seen at: the walk running passes it + /// by, since nothing it maps from now on is revoking. + pub fn epoch(&self) -> u64 { + self.epoch + } + + /// Pages were marked revoking in this hold of the lock: what the caller + /// does about their mappers. + pub fn revoke(&mut self) -> Revoke { + if self.revoking == 0 { + return Revoke::Done; + } + match self.walk { + Walk::Idle => { + self.epoch += 1; + self.walk = Walk::Running { again: false }; + Revoke::Walk + } + Walk::Running { .. } => { + self.walk = Walk::Running { again: true }; + Revoke::Queued(self.epoch + 1) + } + } + } + + /// Whether the walk's pass at `epoch`, or a later one, ended. + pub fn finished(&self, epoch: u64) -> bool { + self.finished >= epoch + } + + /// Whether every page it held is gone, and no walk still owes a frame. + pub fn empty(&self) -> bool { + self.pages == 0 && self.walk == Walk::Idle + } } /// A pager's table: `capacity` slots and a hash of them, both made here. @@ -424,7 +523,21 @@ impl Pages { /// [`release`](Self::release) takes every page it held. pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); - Object { id, anonymous: false, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false } + Object { + id, + anonymous: false, + size, + gone: false, + head: NONE, + pages: 0, + version: 0, + writing: 0, + held: false, + revoking: 0, + epoch: 0, + walk: Walk::Idle, + finished: 0, + } } /// An anonymous object owns its pages: no backing can supply them again, @@ -473,7 +586,7 @@ impl Pages { if next != NONE { self.slot_mut(next).prev = i; } - self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, prev: NONE, next }); + self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, revoking: None, prev: NONE, next }); o.head = i; o.pages += 1; let mut at = self.hash(o.id, page); @@ -531,6 +644,9 @@ impl Pages { if s.state.counts_dirty() { self.dirty_held -= 1; } + if s.revoking.is_some() { + o.revoking -= 1; + } self.free[self.free_len] = i; self.free_len += 1; s.frame @@ -543,6 +659,81 @@ impl Pages { !o.unchanged(since) || !matches!(self.lookup(o, page), Lookup::Absent) } + /// Remove slot `i` as `becomes` once no mapping reaches it: at once for a + /// failed page, which no mapping reaches, and as the walk's end for a + /// frame. A page already revoking becomes failed if either removal fails it. + fn revoke(&mut self, o: &mut Object, i: u32, becomes: Revoked) { + let s = self.slot_mut(i); + if s.frame.is_none() { + self.remove(o, i); + return; + } + if let Some(was) = s.revoking { + s.revoking = Some(if was == Revoked::Failed { was } else { becomes }); + return; + } + // Its bytes are no longer the file's: no write-back takes them. + let counted = s.state.counts_dirty(); + s.state = State::CLEAN; + s.revoking = Some(becomes); + o.revoking += 1; + if counted { + self.dirty_held -= 1; + } + } + + /// Visit `o`'s next mapper after pages were marked revoking + /// ([`Object::revoke`]), or, once every mapper has been visited since the + /// last marking, give out each revoking page's frame, its page and what it + /// became, and answer `None`. A page that ends absent was withdrawn, + /// perhaps after a supply for a waiter's notice, without moving the + /// version: every want of `waiters` it meets is named again. + pub fn walk( + &mut self, + o: &mut Object, + mappers: &mut M, + waiters: &mut impl Waiters, + out: &mut impl FnMut(F, u64, Revoked), + ) -> Option { + loop { + if let Some(node) = mappers.head().filter(|&n| mappers.seen(n) != o.epoch) { + mappers.visited(node, o.epoch); + return Some(node); + } + match o.walk { + Walk::Running { again: true } => { + o.epoch += 1; + o.walk = Walk::Running { again: false }; + } + Walk::Running { again: false } => break, + Walk::Idle => panic!("paged: a walk of object {} that never began", o.id), + } + } + let held = !o.gone; + let mut at = o.head; + while at != NONE { + let s = self.slot(at); + let (next, page, revoking) = (s.next, s.page, s.revoking); + if let Some(becomes) = revoking { + let frame = self.remove(o, at).expect("paged: a revoking page holds its frame"); + let becomes = if held && page < o.page_end() { becomes } else { Revoked::Removed }; + match becomes { + Revoked::Failed => self.insert(o, page, None, State::CLEAN).expect("paged: the slot just freed"), + Revoked::Absent => waiters.each(&mut |want| { + want.withdrawn(page, 1); + }), + Revoked::Removed => {} + } + out(frame, page, becomes); + } + at = next; + } + debug_assert_eq!(o.revoking, 0); + o.walk = Walk::Idle; + o.finished = o.epoch; + None + } + pub fn lookup(&self, o: &Object, page: u64) -> Lookup { if o.gone { return Lookup::Gone; @@ -550,16 +741,18 @@ impl Pages { if page >= o.page_end() { return Lookup::PastEnd; } - match self.find(o.id, page) { + match self.find(o.id, page).map(|i| self.slot(i)) { None if o.anonymous => Lookup::Zero, None => Lookup::Absent, - Some(i) => self.slot(i).frame.map_or(Lookup::Failed, Lookup::Present), + Some(s) if s.revoking.is_some() => Lookup::Absent, + Some(s) => s.frame.map_or(Lookup::Failed, Lookup::Present), } } /// Install `frame` at `page`, clean, or dirty with a slot. A dirty page /// extends the size to `end`, where its bytes end past it; a clean one is - /// what the file holds and moves nothing. A failed page is replaced. + /// what the file holds and moves nothing. A failed page is replaced; a + /// revoking one is present until its walk ends. pub fn supply(&mut self, o: &mut Object, page: u64, frame: F, dirty: bool, end: u64) -> Result { if o.gone { return Err(Refused::Gone); @@ -572,7 +765,7 @@ impl Pages { return Err(Refused::PastEnd); } match self.find(o.id, page) { - Some(i) if self.slot(i).frame.is_some() => return Ok(Supplied::Present), + Some(i) if self.slot(i).frame.is_some() || self.slot(i).revoking.is_some() => return Ok(Supplied::Present), Some(i) => { self.remove(o, i); } @@ -601,7 +794,7 @@ impl Pages { } let i = self.find(o.id, page).ok_or(Refused::InvalidArgument)?; let s = self.slot(i); - let frame = s.frame.ok_or(Refused::InvalidArgument)?; + let frame = s.frame.filter(|_| s.revoking.is_none()).ok_or(Refused::InvalidArgument)?; let needs_slot = !o.anonymous && !s.state.counts_dirty(); if needs_slot && self.dirty_held >= self.dirty_ceiling { return Err(Refused::DirtyFull); @@ -620,9 +813,9 @@ impl Pages { /// The end of a [`write_pin`](Self::write_pin) of `frame`, once its /// bytes are in: the size grows to `end`. A page a shrink or a failure - /// removed meanwhile, even one supplied again since, holds another frame - /// or none; there is nothing left to unpin, and the bytes are not the - /// file's. The caller keeps `frame` pinned until this, so it is not + /// removed meanwhile is revoking, or, supplied again since, holds another + /// frame or none; there is nothing left to unpin, and the bytes are not + /// the file's. The caller keeps `frame` pinned until this, so it is not /// another page's meanwhile. pub fn write_unpin(&mut self, o: &mut Object, page: u64, frame: F, end: u64) where @@ -631,7 +824,7 @@ impl Pages { assert!(end <= (page + 1) * PAGE && end > page * PAGE, "a write pin's end {end} outside page {page}"); let Some(i) = self.find(o.id, page) else { return }; let s = self.slot_mut(i); - if s.frame != Some(frame) { + if s.frame != Some(frame) || s.revoking.is_some() { return; } s.state.write_pins = s.state.write_pins.checked_sub(1).expect("a write pin released twice"); @@ -647,7 +840,7 @@ impl Pages { { let Some(i) = self.find(o.id, page) else { return }; let s = self.slot_mut(i); - if s.frame != Some(frame) { + if s.frame != Some(frame) || s.revoking.is_some() { return; } s.state.write_pins = s.state.write_pins.checked_sub(1).expect("a write pin released twice"); @@ -712,22 +905,28 @@ impl Pages { } /// Remove the clean pages of the range that no write-back or write pin - /// holds: their frames, for the caller to free at their last pin. How - /// many, and whether a waiter's range they meet is news its pager must - /// hear again: they end absent without moving the version, so they may - /// have been supplied for a notice the waiter has not yet looked at. + /// holds, and the failed ones: how many, and whether a waiter's want is + /// news its pager must hear again. Each ends absent without moving the + /// version, perhaps after a supply for a waiter's notice, so every want of + /// `waiters` it meets is named again: a failed page's now, as it goes at + /// once, a present page's at its walk's end, as it is revoking until then + /// ([`Object::revoke`]). /// A range longer than the object holds pages walks its pages, not the range. - pub fn withdraw(&mut self, o: &mut Object, first: u64, count: u64, waiters: &mut impl Waiters, out: &mut impl FnMut(F)) -> (usize, bool) { + pub fn withdraw(&mut self, o: &mut Object, first: u64, count: u64, waiters: &mut impl Waiters) -> (usize, bool) { if o.anonymous { return (0, false); } let end = first.saturating_add(count); - let mut removed = 0; + let (mut removed, mut news) = (0, false); let mut take = |pages: &mut Self, o: &mut Object, i: u32| { let s = pages.slot(i); - if s.state.counts_dirty() || s.state.write_pins > 0 { + if s.revoking.is_some() || s.state.counts_dirty() || s.state.write_pins > 0 { return; } - if let Some(frame) = pages.remove(o, i) { - out(frame); + if s.frame.is_none() { + let page = s.page; + pages.remove(o, i); + waiters.each(&mut |want| news |= want.withdrawn(page, 1)); + } else { + pages.revoke(o, i, Revoked::Absent); } removed += 1; }; @@ -748,25 +947,20 @@ impl Pages { } } } - let mut news = false; - if removed > 0 { - waiters.each(&mut |want| news |= want.withdrawn(first, count)); - } (removed, news) } - /// Mark the range failed: present pages give up their frames, and a read - /// of any page of it answers `Failed` until it is supplied again. - pub fn fail(&mut self, o: &mut Object, first: u64, count: u64, out: &mut impl FnMut(F)) -> Result<(), Refused> { + /// Mark the range failed: a read of any page of it answers `Failed` until + /// it is supplied again, a present page's from its walk's end. + pub fn fail(&mut self, o: &mut Object, first: u64, count: u64) -> Result<(), Refused> { if o.anonymous { return Err(Refused::InvalidArgument); } let end = first.saturating_add(count).min(o.page_end()); for page in first..end { - if let Some(i) = self.find(o.id, page) { - if let Some(frame) = self.remove(o, i) { - out(frame); - } + match self.find(o.id, page) { + Some(i) if self.slot(i).frame.is_some() => self.revoke(o, i, Revoked::Failed), + Some(_) => {} + None => self.insert(o, page, None, State::CLEAN)?, } - self.insert(o, page, None, State::CLEAN)?; } Ok(()) } @@ -774,7 +968,7 @@ impl Pages { /// Set the size. A shrink removes every page past it, dirty or not, and /// answers the frame of a partial last page, whose tail past the size the /// caller zeroes. - pub fn resize(&mut self, o: &mut Object, size: u64, out: &mut impl FnMut(F)) -> Option { + pub fn resize(&mut self, o: &mut Object, size: u64) -> Option { o.version += 1; let keep = size.div_ceil(PAGE); if size < o.size { @@ -785,9 +979,7 @@ impl Pages { (s.next, s.page) }; if page >= keep { - if let Some(frame) = self.remove(o, at) { - out(frame); - } + self.revoke(o, at, Revoked::Absent); } at = next; } @@ -802,12 +994,14 @@ impl Pages { } } - /// The pager ended: every page goes, and every later call answers `Gone`. - pub fn invalidate(&mut self, o: &mut Object, out: &mut impl FnMut(F)) { - while o.head != NONE { - if let Some(frame) = self.remove(o, o.head) { - out(frame); - } + /// The pager ended or let the object go: every page goes, and every + /// later call answers `Gone`. + pub fn invalidate(&mut self, o: &mut Object) { + let mut at = o.head; + while at != NONE { + let next = self.slot(at).next; + self.revoke(o, at, Revoked::Absent); + at = next; } o.gone = true; for range in &mut self.open { @@ -816,11 +1010,6 @@ impl Pages { } } } - - /// The object is let go: every page goes, as at [`invalidate`](Self::invalidate). - pub fn release(&mut self, mut o: Object, out: &mut impl FnMut(F)) { - self.invalidate(&mut o, out); - } } #[cfg(test)] @@ -830,10 +1019,147 @@ mod tests { use super::*; - fn none(_: u64) {} + /// No space maps anything: a walk ends at its first step. + struct Unmapped; + + impl Mappers for Unmapped { + type Node = (); + fn head(&self) -> Option<()> { + None + } + fn seen(&self, (): ()) -> u64 { + 0 + } + fn visited(&mut self, (): (), _: u64) {} + } + + /// The frames of `o`'s revoking pages, once a walk over no mappers ends. + fn settle(p: &mut Pages, o: &mut Object) -> Vec { + let mut freed = Vec::new(); + match o.revoke() { + Revoke::Done => {} + Revoke::Walk => assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), None), + Revoke::Queued(_) => panic!("a walk ran with no walker"), + } + freed + } + + /// One object's mappers: each node's space and the epoch it was last + /// seen at, in visiting order. + #[derive(Default)] + struct Ring(std::collections::VecDeque<(u32, u64)>); + + impl Ring { + fn link(&mut self, space: u32, o: &Object) { + self.0.push_back((space, o.epoch())); + } + } + + impl Mappers for Ring { + type Node = u32; + fn head(&self) -> Option { + self.0.front().map(|&(n, _)| n) + } + fn seen(&self, node: u32) -> u64 { + self.0.iter().find(|&&(n, _)| n == node).expect("linked").1 + } + fn visited(&mut self, node: u32, epoch: u64) { + let at = self.0.iter().position(|&(n, _)| n == node).expect("linked"); + self.0.remove(at); + self.0.push_back((node, epoch)); + } + } + + /// A removed page reads as absent, takes no second frame and no write, + /// and keeps its frame until every mapper of its object was visited; + /// then its frame goes out once and the page is supplied afresh. + #[test] + fn a_revoking_page_keeps_its_one_frame_until_every_mapper_is_visited() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + let mut ring = Ring::default(); + ring.link(10, &o); + ring.link(11, &o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(p.lookup(&o, 0), Lookup::Absent); + assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Present), "a second frame for the page"); + assert_eq!(p.write_pin(&mut o, 0), Err(Refused::InvalidArgument)); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 0, "withdrawn once"); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + let mut out = |f, page, becomes| freed.push((f, page, becomes)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(11)); + assert!(!o.empty()); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), None); + assert_eq!(freed, [(7, 0, Revoked::Absent)]); + assert!(o.finished(o.epoch())); + assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Installed)); + assert_eq!(o.revoke(), Revoke::Done, "nothing revoking"); + } + + /// A removal while a walk runs is taken by a second pass over every + /// mapper, those the first visited and one linked during it, which may + /// map the newly removed page; its caller knows the pass by its epoch. + #[test] + fn a_removal_during_a_walk_is_taken_by_a_second_pass() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + let mut ring = Ring::default(); + ring.link(10, &o); + ring.link(11, &o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.supply(&mut o, 1, 8, false, 2 * PAGE).unwrap(); + p.withdraw(&mut o, 0, 1, &mut Nobody); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + let mut out = |f, page, becomes| freed.push((f, page, becomes)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); + ring.link(12, &o); + p.fail(&mut o, 1, 1).unwrap(); + let Revoke::Queued(second) = o.revoke() else { panic!("a walk runs") }; + assert!(!o.finished(second)); + let visits: Vec<_> = core::iter::from_fn(|| p.walk(&mut o, &mut ring, &mut Nobody, &mut out)).collect(); + assert_eq!(visits, [11, 10, 12, 11], "the first pass ends at 11, the second visits all three"); + assert!(o.finished(second)); + assert_eq!(freed, [(8, 1, Revoked::Failed), (7, 0, Revoked::Absent)]); + assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed)); + } + + /// A mapper linked during a walk that nothing else joins maps nothing + /// revoking, so the walk passes it by. + #[test] + fn a_mapper_linked_during_a_walk_is_passed_by() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, PAGE); + let mut ring = Ring::default(); + ring.link(10, &o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.withdraw(&mut o, 0, 1, &mut Nobody); + assert_eq!(o.revoke(), Revoke::Walk); + let mut out = |_, _, _| {}; + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); + ring.link(11, &o); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), None); + } - fn sink(freed: &mut Vec) -> impl FnMut(u64) + '_ { - |f| freed.push(f) + /// A page failed and then shrunk away while revoking ends with no slot, + /// and one failed and then withdrawn still ends failed. + #[test] + fn a_revoking_page_ends_failed_only_where_the_object_still_holds_it() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.supply(&mut o, 1, 8, false, 2 * PAGE).unwrap(); + p.fail(&mut o, 0, 2).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 2, &mut Nobody).0, 0, "nothing left to withdraw"); + p.resize(&mut o, PAGE); + assert_eq!(o.revoke(), Revoke::Walk); + let mut ended = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), None); + assert_eq!(ended, [(8, 1, Revoked::Removed), (7, 0, Revoked::Failed)]); + assert_eq!((p.lookup(&o, 0), p.used()), (Lookup::Failed, 1)); } /// No reader waits. @@ -877,7 +1203,7 @@ mod tests { assert!(!o.unchanged(before), "and still does once it ended"); let after = o.read_begin().expect("the write ended"); assert!(o.unchanged(after)); - p.resize(&mut o, 0, &mut none); + p.resize(&mut o, 0); assert!(!o.unchanged(after), "a shrink is a change a read meets"); } @@ -912,20 +1238,20 @@ mod tests { assert_eq!(pages.dirty_held(), 0); assert_eq!(pages.write_pin(&mut object, 0), Ok(42)); pages.write_unpin(&mut object, 0, 42, PAGE); - assert_eq!(pages.withdraw(&mut object, 0, 1, &mut Nobody, &mut none).0, 0); + assert_eq!(pages.withdraw(&mut object, 0, 1, &mut Nobody).0, 0); assert_eq!(pages.writeback_begin(&mut object, 0, 1), Err(Refused::InvalidArgument)); - assert_eq!(pages.fail(&mut object, 0, 1, &mut none), Err(Refused::InvalidArgument)); + assert_eq!(pages.fail(&mut object, 0, 1), Err(Refused::InvalidArgument)); assert_eq!(pages.lookup(&object, 0), Lookup::Present(42)); assert_eq!(pages.supply(&mut object, 1, 43, true, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(pages.supply(&mut object, 2, 44, true, 3 * PAGE), Err(Refused::NoRoom)); - pages.resize(&mut object, 4 * PAGE, &mut none); + pages.resize(&mut object, 4 * PAGE); assert_eq!(pages.lookup(&object, 3), Lookup::Zero, "a hole reads without any free slot"); - let mut freed = Vec::new(); - pages.resize(&mut object, PAGE, &mut sink(&mut freed)); - assert_eq!(freed, [43]); - pages.release(object, &mut sink(&mut freed)); - assert_eq!(freed, [43, 42]); + pages.resize(&mut object, PAGE); + assert_eq!(settle(&mut pages, &mut object), [43]); + pages.invalidate(&mut object); + assert_eq!(settle(&mut pages, &mut object), [42]); assert_eq!(pages.used(), 0); + assert!(object.empty()); } #[test] @@ -986,11 +1312,12 @@ mod tests { let mut o = p.object(5, PAGE); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.write_pin(&mut o, 0).unwrap(); - p.resize(&mut o, 0, &mut none); + p.resize(&mut o, 0); + settle(&mut p, &mut o); p.supply(&mut o, 0, 8, true, 10).unwrap(); p.write_unpin(&mut o, 0, 7, PAGE); assert_eq!(o.size(), 10, "the shrink's size, and the new page's, stand over a copy they overtook"); - assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody, &mut none).0, 0, "the new page is dirty"); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 0, "the new page is dirty"); p.writeback_begin(&mut o, 0, 1).unwrap(); assert_eq!(p.writeback_end(&mut o, 0, 1, true).unwrap()[0], 0, "and holds no pin of the old one"); } @@ -1022,9 +1349,10 @@ mod tests { p.write_unpin(&mut o, 0, 0, PAGE); p.write_pin(&mut o, 1).unwrap(); // dirtied inside p.write_unpin(&mut o, 1, 1, 2 * PAGE); - let mut freed = Vec::new(); - p.resize(&mut o, 5 * PAGE, &mut sink(&mut freed)); // page 5 removed inside - p.fail(&mut o, 4, 1, &mut sink(&mut freed)).unwrap(); // page 4 failed inside + p.resize(&mut o, 5 * PAGE); // page 5 removed inside + p.fail(&mut o, 4, 1).unwrap(); // page 4 failed inside + assert_eq!(p.dirty_held(), 4, "a revoking page's bytes are no longer written back"); + assert_eq!(settle(&mut p, &mut o), [5, 4]); let kept = p.writeback_end(&mut o, 0, 6, true).unwrap()[0]; assert_eq!(kept, 0b11, "pinned since the begin and dirtied since it; the removed are clear"); assert_eq!(p.dirty_held(), 2); @@ -1067,9 +1395,9 @@ mod tests { p.supply(&mut o, 3, 13, false, 4 * PAGE).unwrap(); p.writeback_begin(&mut o, 2, 1).unwrap(); // clean but under write-back p.write_pin(&mut o, 3).unwrap(); // write-pinned - let mut freed = Vec::new(); - assert_eq!(p.withdraw(&mut o, 0, 4, &mut Nobody, &mut sink(&mut freed)).0, 1); - assert_eq!(freed, [10]); + assert_eq!(p.withdraw(&mut o, 0, 4, &mut Nobody).0, 1); + assert_eq!(p.lookup(&o, 0), Lookup::Absent, "revoking reads as absent"); + assert_eq!(settle(&mut p, &mut o), [10]); assert_eq!(p.lookup(&o, 0), Lookup::Absent); } @@ -1121,13 +1449,13 @@ mod tests { let mut o = p.object(1, LAST_END); p.supply(&mut o, 7, 1, false, 8 * PAGE).unwrap(); p.supply(&mut o, 1 << 40, 2, false, ((1 << 40) + 1) * PAGE).unwrap(); - let mut freed = Vec::new(); p.probes.set(0); - assert_eq!(p.withdraw(&mut o, 8, 1 << 24, &mut Nobody, &mut sink(&mut freed)).0, 0, "neither page lies in 8..8+2^24"); + assert_eq!(p.withdraw(&mut o, 8, 1 << 24, &mut Nobody).0, 0, "neither page lies in 8..8+2^24"); assert!(p.probes.get() < 8, "{} probes for a range of 2^24 pages", p.probes.get()); - assert_eq!(p.withdraw(&mut o, 8, u64::MAX, &mut Nobody, &mut sink(&mut freed)).0, 1); - assert_eq!(freed, [2], "only the page inside the range"); - assert_eq!(p.withdraw(&mut o, 0, u64::MAX, &mut Nobody, &mut sink(&mut freed)).0, 1); + assert_eq!(p.withdraw(&mut o, 8, u64::MAX, &mut Nobody).0, 1); + assert_eq!(settle(&mut p, &mut o), [2], "only the page inside the range"); + assert_eq!(p.withdraw(&mut o, 0, u64::MAX, &mut Nobody).0, 1); + settle(&mut p, &mut o); assert_eq!(o.pages(), 0); } @@ -1153,12 +1481,11 @@ mod tests { for page in 0..3 { p.supply(&mut o, page, page + 100, true, (page + 1) * PAGE).unwrap(); } - let mut freed = Vec::new(); - assert_eq!(p.resize(&mut o, PAGE + 5, &mut sink(&mut freed)), Some(101)); - assert_eq!(freed, [102]); + assert_eq!(p.resize(&mut o, PAGE + 5), Some(101)); assert_eq!(p.dirty_held(), 2); assert_eq!(p.lookup(&o, 2), Lookup::PastEnd); - assert_eq!(p.resize(&mut o, 4 * PAGE, &mut none), None, "growth removes nothing"); + assert_eq!(settle(&mut p, &mut o), [102]); + assert_eq!(p.resize(&mut o, 4 * PAGE), None, "growth removes nothing"); assert_eq!(p.lookup(&o, 2), Lookup::Absent, "and leaves the new range absent"); } @@ -1167,9 +1494,9 @@ mod tests { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(12, 2 * PAGE); p.supply(&mut o, 0, 5, false, PAGE).unwrap(); - let mut freed = Vec::new(); - p.fail(&mut o, 0, 2, &mut sink(&mut freed)).unwrap(); - assert_eq!(freed, [5]); + p.fail(&mut o, 0, 2).unwrap(); + assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed), "page 0 revoking"); + assert_eq!(settle(&mut p, &mut o), [5]); assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Failed, Lookup::Failed)); assert_eq!(p.supply(&mut o, 1, 6, false, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(p.lookup(&o, 1), Lookup::Present(6)); @@ -1181,8 +1508,9 @@ mod tests { let mut o = p.object(13, 4 * PAGE); p.supply(&mut o, 0, 1, true, PAGE).unwrap(); p.supply(&mut o, 2, 3, false, 3 * PAGE).unwrap(); - let mut freed = Vec::new(); - p.invalidate(&mut o, &mut sink(&mut freed)); + p.invalidate(&mut o); + assert_eq!(p.lookup(&o, 0), Lookup::Gone); + let mut freed = settle(&mut p, &mut o); freed.sort(); assert_eq!(freed, [1, 3]); assert_eq!((p.used(), p.dirty_held()), (0, 0)); @@ -1202,9 +1530,8 @@ mod tests { let page = (x >> 40) % 97; let o = &mut objects[which]; if (x >> 20).is_multiple_of(3) { - let mut freed = Vec::new(); - p.withdraw(o, page, 1, &mut Nobody, &mut sink(&mut freed)); - if !freed.is_empty() { + p.withdraw(o, page, 1, &mut Nobody); + if !settle(&mut p, o).is_empty() { assert!(model.remove(&(which, page))); } } else { @@ -1238,14 +1565,15 @@ mod tests { assert!(!p.wait_over(&o, 0, since), "absent"); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); assert!(p.wait_over(&o, 0, since), "supplied"); - p.resize(&mut o, 0, &mut none); - p.resize(&mut o, PAGE, &mut none); + p.resize(&mut o, 0); + settle(&mut p, &mut o); + p.resize(&mut o, PAGE); assert_eq!(p.lookup(&o, 0), Lookup::Absent); assert!(p.wait_over(&o, 0, since), "shrunk and grown back before the reader looked"); let mut failed = p.object(2, PAGE); let since = failed.read_begin().unwrap(); - p.fail(&mut failed, 0, 1, &mut none).unwrap(); + p.fail(&mut failed, 0, 1).unwrap(); failed.write_begin(); assert_eq!(p.supply(&mut failed, 0, 8, true, PAGE), Err(Refused::DirtyFull)); failed.write_end(true); @@ -1258,10 +1586,12 @@ mod tests { let mut want = Want::NONE; want.ask(0, 1); assert_eq!(want.report(), Some((0, 1))); - let mut waiting = One(&mut want); - assert_eq!(p.withdraw(&mut taken, 0, 1, &mut waiting, &mut none), (1, true)); + assert_eq!(p.withdraw(&mut taken, 0, 1, &mut Nobody).0, 1); + assert_eq!(taken.revoke(), Revoke::Walk); + assert!(!want.unreported(), "named only once no mapping reaches the page"); + assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), None); assert!(!p.wait_over(&taken, 0, since), "a withdraw leaves the read waiting"); - assert!(want.unreported(), "and names its range again"); + assert!(want.unreported(), "and its walk's end names its range again"); } /// One reader's want. @@ -1325,17 +1655,20 @@ mod tests { let next = self.next + 1; match step { Step::Withdraw => { - let mut waiting = One(&mut self.want); - self.p.withdraw(&mut self.o, 0, 1, &mut waiting, &mut |_| ()); + self.p.withdraw(&mut self.o, 0, 1, &mut One(&mut self.want)); + self.settle(); } Step::Fail => { - self.p.fail(&mut self.o, 0, 1, &mut |_| ()).unwrap(); + self.p.fail(&mut self.o, 0, 1).unwrap(); + self.settle(); } Step::Shrink if self.o.size() == PAGE => { - self.p.resize(&mut self.o, 0, &mut |_| ()); + self.p.resize(&mut self.o, 0); + self.settle(); } Step::Grow if self.o.size() == 0 => { - self.p.resize(&mut self.o, PAGE, &mut |_| ()); + self.p.resize(&mut self.o, PAGE); + self.settle(); } Step::Refused => { self.o.write_begin(); @@ -1351,7 +1684,7 @@ mod tests { // The file server fails a requested range past the // end, which inserts nothing there. if self.p.supply(&mut self.o, 0, next, false, PAGE) == Err(Refused::PastEnd) { - self.p.fail(&mut self.o, 0, 1, &mut |_| ()).unwrap(); + self.p.fail(&mut self.o, 0, 1).unwrap(); } } Step::Look if self.waits && self.p.wait_over(&self.o, 0, self.since) => { @@ -1369,6 +1702,14 @@ mod tests { true } + /// A removal's walk, over no mapper, to its end. + fn settle(&mut self) { + if self.o.revoke() == Revoke::Walk { + let walked = self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()); + assert_eq!(walked, None); + } + } + /// The pager answers what it hears and the reader looks whenever /// its wait is over: the read ends. pub(super) fn drain(&mut self, steps: &[Step]) { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 0d7f5c5ca1..071a635a59 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -50,7 +50,7 @@ use core::ptr; use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::Stack; -use kernel::paged::{Core, Draw, Leave, Lookup, Object, Pages, Refused, Supplied, Version, Waiters, Want, PAGE}; +use kernel::paged::{Core, Draw, Leave, Lookup, Mappers, Object, Pages, Refused, Revoke, Revoked, Supplied, Version, Waiters, Want, PAGE}; use kernel::sched::task::WaitClass; use kernel::slots::Slots; use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; @@ -311,25 +311,45 @@ impl PagerObject { u64::from(made.expect("paged: a slot was made free")) } - /// `f` over the index, the live object `id` and its waiters, each frame - /// it removes given back as it goes; waiters read again. - fn removing( - &self, - id: u64, - f: impl FnOnce(&mut Pages, &mut Object, &mut Linked, &mut dyn FnMut(u64)) -> R, - ) -> Result { + /// `f` over the index, the live object `id` and its waiters, marking + /// pages revoking, then their walk ([`Self::revoked`]); waiters read again. + fn removing(&self, id: u64, f: impl FnOnce(&mut Pages, &mut Object, &mut Linked) -> R) -> Result { let mut state = self.state.lock(); let slot = state.live(id)?; - let job = self.core.job(); - let State { pages, entries, held, gone, requests, .. } = &mut *state; - let object = &mut entries.get_mut(slot).expect("live").object; - let mut waiters = Linked { head: *requests, slot }; - let answer = f(pages, object, &mut waiters, &mut |phys| held.give_back(job, *gone, phys, true)); + let mut waiters = Linked { head: state.requests, slot }; + let (pages, object) = state.split(slot); + let answer = f(pages, object, &mut waiters); + self.revoked(&mut state, slot); drop(state); self.changed.post(); Ok(answer) } + /// Under the lock, `state`: the pages of the object at `slot` just marked + /// revoking leave once no mapping reaches them, each frame given back, + /// and every waiter's want a page that ends absent meets is named again + /// in a notice (`Pages::walk`). + fn revoked(&self, state: &mut State, slot: u32) { + let job = self.core.job(); + let mut waiters = Linked { head: state.requests, slot }; + let State { pages, entries, held, gone, .. } = &mut *state; + let object = &mut entries.get_mut(slot).expect("paged: a slot an object names is held").object; + match object.revoke() { + Revoke::Done => return, + Revoke::Walk => {} + Revoke::Queued(_) => unreachable!("paged: a walk outlived the hold of the lock that began it"), + } + let mut absent = false; + let next = pages.walk(object, &mut Unmapped, &mut waiters, &mut |phys, _, becomes| { + held.give_back(job, *gone, phys, true); + absent |= becomes == Revoked::Absent; + }); + debug_assert!(next.is_none()); + if absent { + self.watch.post(); + } + } + /// The pager lets object `id` go, in one hold of the lock: it answers /// `Gone` from now on, its pages go, and its slot with its last holder. A /// second release, or one racing the pager's end, finds it released. @@ -347,12 +367,12 @@ impl PagerObject { /// Under the lock, `state`: the unreleased object at `slot` released. fn let_go(&self, state: &mut State, slot: u32) { - let job = self.core.job(); - let State { pages, entries, held, gone, .. } = state; + let State { pages, entries, .. } = &mut *state; let entry = entries.get_mut(slot).expect("paged: a slot an object names is held"); - pages.invalidate(&mut entry.object, &mut |phys| held.give_back(job, *gone, phys, true)); + pages.invalidate(&mut entry.object); assert!(core::mem::take(&mut entry.owned), "paged: an object released twice"); entries.unmark(slot); + self.revoked(state, slot); self.unref_locked(state, slot); } @@ -391,9 +411,7 @@ impl PagerObject { return; } let entry = state.entries.remove(slot).expect("paged: a slot an object names is held"); - let job = self.core.job(); - let State { pages, held, gone, .. } = state; - pages.release(entry.object, &mut |phys| held.give_back(job, *gone, phys, true)); + assert!(entry.object.empty(), "paged: an object's slot freed while it holds pages"); } /// Fill `out` with notices, each answered once. @@ -885,10 +903,8 @@ impl PagerObject { done as u64 } - /// Its pages end absent, so every reader waiting on one is named in a - /// notice again (`Want::withdrawn`). pub fn withdraw(&self, id: u64, first: u64, count: u64) -> u64 { - match self.removing(id, |pages, object, waiters, out| pages.withdraw(object, first, count, waiters, &mut |f| out(f))) { + match self.removing(id, |pages, object, waiters| pages.withdraw(object, first, count, waiters)) { Ok((n, news)) => { if news { self.watch.post(); @@ -900,7 +916,7 @@ impl PagerObject { } pub fn fail(&self, id: u64, first: u64, count: u64) -> u64 { - match self.removing(id, |pages, object, _, out| pages.fail(object, first, count, &mut |f| out(f))) { + match self.removing(id, |pages, object, _| pages.fail(object, first, count)) { Ok(Ok(())) => 0, Ok(Err(why)) => word(why).to_u64(), Err(e) => e.to_u64(), @@ -911,8 +927,8 @@ impl PagerObject { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } - let resized = self.removing(id, |pages, object, _, out| { - if let Some(phys) = pages.resize(object, size, &mut |f| out(f)) { + let resized = self.removing(id, |pages, object, _| { + if let Some(phys) = pages.resize(object, size) { let within = (size % PAGE) as usize; // SAFETY: the last page's frame, the object's while the lock is // held: zeroed past the size through the direct map, never @@ -1156,6 +1172,20 @@ impl Drop for PagerObject { } } +/// No space maps a paged object yet: every walk ends at its first step. +struct Unmapped; + +impl Mappers for Unmapped { + type Node = (); + fn head(&self) -> Option<()> { + None + } + fn seen(&self, (): ()) -> u64 { + 0 + } + fn visited(&mut self, (): (), _: u64) {} +} + /// One call's reference to its pager's object. struct Ref<'a> { pager: &'a PagerObject, From a189b6756e49ecbe7904086a933c337dad41bf92 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 17:55:53 +0200 Subject: [PATCH 02/34] The index's revocation, modelled with two spaces mapping it A host model of 7b's mapped paged objects over the real index and walk: two spaces mapping one object of two pages, a fault that writes a leaf only for a present page and fills its neighbour likewise, a visit that clears every leaf the index no longer holds present as that frame, an unmap that clears its leaves before its node leaves the ring, and supply, withdraw, fail, shrink, grow and the pager's end, each queued behind a running walk. At every step each leaf maps the frame the index holds for its page, present or revoking, and no frame given back; no page is revoking without a walk. Every sequence of six steps from both spaces mapping both pages, and 2,000 seeded runs of 200 steps. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 213 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 5394773439..98f67f972b 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -1748,4 +1748,217 @@ mod tests { } assert!(played > 10_000, "only {played} sequences played"); } + + /// Two spaces mapping one object of two pages, as the kernel drives the + /// index: a fault writes a leaf only for a present page, a visit clears + /// every leaf the index no longer holds present as that frame, and an + /// unmap clears its leaves before its node leaves the ring. + mod mapped { + use super::*; + + #[derive(Clone, Copy, Debug)] + pub(super) enum Step { + Map(u32), + Unmap(u32), + Fault(u32, u64), + Supply(u64), + Withdraw(u64), + Fail(u64), + Shrink, + Grow, + End, + /// The walker's step under the pager's lock. + Walk, + /// The walker's visit of the node it took, under that space's lock. + Visit, + } + + pub(super) const STEPS: [Step; 18] = [ + Step::Map(0), + Step::Map(1), + Step::Unmap(0), + Step::Unmap(1), + Step::Fault(0, 0), + Step::Fault(0, 1), + Step::Fault(1, 0), + Step::Fault(1, 1), + Step::Supply(0), + Step::Supply(1), + Step::Withdraw(0), + Step::Withdraw(1), + Step::Fail(0), + Step::Shrink, + Step::Grow, + Step::End, + Step::Walk, + Step::Visit, + ]; + + pub(super) struct World { + p: Pages, + o: Object, + ring: Ring, + mapped: [bool; 2], + leaves: [[Option; 2]; 2], + next: u64, + freed: BTreeSet, + walking: bool, + visiting: Option, + } + + impl World { + pub(super) fn new() -> Self { + let mut p = Pages::new(4, 4).unwrap(); + let o = p.object(1, 2 * PAGE); + World { + p, + o, + ring: Ring::default(), + mapped: [false; 2], + leaves: [[None; 2]; 2], + next: 100, + freed: BTreeSet::new(), + walking: false, + visiting: None, + } + } + + /// Whether `step` can be taken now; takes it if so. + pub(super) fn step(&mut self, step: Step) -> bool { + match step { + Step::Map(s) if !self.mapped[s as usize] => { + self.ring.link(s, &self.o); + self.mapped[s as usize] = true; + } + Step::Unmap(s) if self.mapped[s as usize] => { + self.leaves[s as usize] = [None; 2]; + let at = self.ring.0.iter().position(|&(n, _)| n == s).expect("linked while mapped"); + self.ring.0.remove(at); + self.mapped[s as usize] = false; + } + Step::Fault(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_none() => { + let Lookup::Present(_) = self.p.lookup(&self.o, page) else { return false }; + for at in 0..2 { + if let (None, Lookup::Present(f)) = (self.leaves[s as usize][at], self.p.lookup(&self.o, at as u64)) { + self.leaves[s as usize][at] = Some(f); + } + } + } + Step::Supply(page) if !self.o.gone() => { + self.next += 1; + let _ = self.p.supply(&mut self.o, page, self.next, false, (page + 1) * PAGE); + } + Step::Withdraw(page) if !self.o.gone() => { + self.p.withdraw(&mut self.o, page, 1, &mut Nobody); + self.revoke(); + } + Step::Fail(page) if !self.o.gone() => { + let _ = self.p.fail(&mut self.o, page, 1); + self.revoke(); + } + Step::Shrink if !self.o.gone() && self.o.size() == 2 * PAGE => { + self.p.resize(&mut self.o, PAGE); + self.revoke(); + } + Step::Grow if !self.o.gone() && self.o.size() == PAGE => { + self.p.resize(&mut self.o, 2 * PAGE); + self.revoke(); + } + Step::End if !self.o.gone() => { + self.p.invalidate(&mut self.o); + self.revoke(); + } + Step::Walk if self.walking && self.visiting.is_none() => { + let freed = &mut self.freed; + match self.p.walk(&mut self.o, &mut self.ring, &mut Nobody, &mut |f, _, _| assert!(freed.insert(f), "frame {f} given back twice")) { + Some(node) => self.visiting = Some(node), + None => self.walking = false, + } + } + Step::Visit if self.visiting.is_some() => { + let s = self.visiting.take().unwrap() as usize; + for page in 0..2 { + if let Some(f) = self.leaves[s][page] { + if self.p.lookup(&self.o, page as u64) != Lookup::Present(f) { + self.leaves[s][page] = None; + } + } + } + } + _ => return false, + } + self.check(step); + true + } + + fn revoke(&mut self) { + match self.o.revoke() { + Revoke::Walk => { + assert!(!self.walking, "a second walker"); + self.walking = true; + } + Revoke::Queued(_) => assert!(self.walking, "queued behind no walk"), + Revoke::Done => {} + } + } + + /// Every leaf maps the frame the index holds for its page, present + /// or revoking, and never one given back. + fn check(&self, step: Step) { + for (s, leaves) in self.leaves.iter().enumerate() { + for (page, leaf) in leaves.iter().enumerate() { + let Some(f) = *leaf else { continue }; + assert!(!self.freed.contains(&f), "after {step:?}: space {s} page {page} maps given-back frame {f}"); + let held = self.p.find(self.o.id, page as u64).map(|i| self.p.slot(i).frame); + assert_eq!(held, Some(Some(f)), "after {step:?}: space {s} page {page} maps {f}, the index holds {held:?}"); + } + } + assert!(self.walking || self.o.revoking == 0, "after {step:?}: pages revoking with no walk"); + } + } + } + + fn play(steps: &[mapped::Step]) -> Option { + let mut w = mapped::World::new(); + steps.iter().all(|&s| w.step(s)).then_some(w) + } + + /// From both spaces mapping both pages, every sequence of six steps of + /// faults, supplies, removals, mappings and the walk keeps each leaf on + /// its page's one frame. + #[test] + fn every_interleaving_keeps_each_leaf_on_its_pages_one_frame() { + use mapped::Step; + let prefix = [Step::Map(0), Step::Map(1), Step::Supply(0), Step::Supply(1), Step::Fault(0, 0), Step::Fault(1, 1)]; + let mut stack = std::vec![prefix.to_vec()]; + let mut played = 0u64; + while let Some(seq) = stack.pop() { + played += 1; + if seq.len() == prefix.len() + 6 { + continue; + } + for &step in &mapped::STEPS { + let mut next = seq.clone(); + next.push(step); + if play(&next).is_some() { + stack.push(next); + } + } + } + assert!(played > 50_000, "only {played} sequences played"); + } + + /// The same over long sequences, seeded: removals queued behind running + /// walks, ends meeting walks, unmaps while their node is visited. + #[test] + fn long_runs_keep_each_leaf_on_its_pages_one_frame() { + let mut x = 7u64; + for _ in 0..2_000 { + let mut w = mapped::World::new(); + for _ in 0..200 { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + w.step(mapped::STEPS[(x >> 33) as usize % mapped::STEPS.len()]); + } + } + } } From 2d15331e49c9718a4b7b1797aa09b2a8d29eb888 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 19:35:56 +0200 Subject: [PATCH 03/34] The model's parked faults all complete once removals stop The model of mapped paged objects gains its requests: a fault on an absent page parks with its request, the server reads its notices and answers each by a supply later, so a removal can land between a supply and the retry. The parked faults' wants are the waiters the index's withdraw and walk mark (`Waiters`), so the model drives the re-report itself. After any sequence, a fair drain (the walk to its end, the server answering what it hears, every parked fault retrying) leaves none parked. The exhaustive depth is five steps; the seeded runs keep the long ones. Not yet modelled: a fault waiter's wait ending on a version change, which step 2's faults need by design rev 5 (a shrink and a grow between a removal's wake and the waiter's look). Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 112 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 98f67f972b..03b089cca1 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -1771,9 +1771,15 @@ mod tests { Walk, /// The walker's visit of the node it took, under that space's lock. Visit, + /// The server reads its notices: each request not yet named. + Report, + /// The server answers the oldest request it read, by a supply. + Answer, + /// Every parked fault looks at its page again. + Retry, } - pub(super) const STEPS: [Step; 18] = [ + pub(super) const STEPS: [Step; 21] = [ Step::Map(0), Step::Map(1), Step::Unmap(0), @@ -1792,6 +1798,9 @@ mod tests { Step::End, Step::Walk, Step::Visit, + Step::Report, + Step::Answer, + Step::Retry, ]; pub(super) struct World { @@ -1804,6 +1813,22 @@ mod tests { freed: BTreeSet, walking: bool, visiting: Option, + /// Faults parked on an absent page: the space, the page, and the + /// request as the pager hears of it. + parked: std::vec::Vec<(u32, u64, Want)>, + /// Pages the server read requests for and has yet to supply. + asked: std::collections::VecDeque, + } + + /// The parked faults' wants, which removals name again. + struct Parked<'a>(&'a mut std::vec::Vec<(u32, u64, Want)>); + + impl Waiters for Parked<'_> { + fn each(&mut self, f: &mut dyn FnMut(&mut Want)) { + for (_, _, want) in self.0.iter_mut() { + f(want); + } + } } impl World { @@ -1820,6 +1845,8 @@ mod tests { freed: BTreeSet::new(), walking: false, visiting: None, + parked: std::vec::Vec::new(), + asked: std::collections::VecDeque::new(), } } @@ -1837,11 +1864,13 @@ mod tests { self.mapped[s as usize] = false; } Step::Fault(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_none() => { - let Lookup::Present(_) = self.p.lookup(&self.o, page) else { return false }; - for at in 0..2 { - if let (None, Lookup::Present(f)) = (self.leaves[s as usize][at], self.p.lookup(&self.o, at as u64)) { - self.leaves[s as usize][at] = Some(f); + if !self.touch(s, page) { + if self.p.lookup(&self.o, page) != Lookup::Absent || self.parked.iter().any(|&(t, q, _)| (t, q) == (s, page)) { + return false; } + let mut want = Want::NONE; + want.ask(page, 1); + self.parked.push((s, page, want)); } } Step::Supply(page) if !self.o.gone() => { @@ -1849,7 +1878,7 @@ mod tests { let _ = self.p.supply(&mut self.o, page, self.next, false, (page + 1) * PAGE); } Step::Withdraw(page) if !self.o.gone() => { - self.p.withdraw(&mut self.o, page, 1, &mut Nobody); + self.p.withdraw(&mut self.o, page, 1, &mut Parked(&mut self.parked)); self.revoke(); } Step::Fail(page) if !self.o.gone() => { @@ -1870,11 +1899,48 @@ mod tests { } Step::Walk if self.walking && self.visiting.is_none() => { let freed = &mut self.freed; - match self.p.walk(&mut self.o, &mut self.ring, &mut Nobody, &mut |f, _, _| assert!(freed.insert(f), "frame {f} given back twice")) { + let walked = self.p.walk(&mut self.o, &mut self.ring, &mut Parked(&mut self.parked), &mut |f, _, _| { + assert!(freed.insert(f), "frame {f} given back twice"); + }); + match walked { Some(node) => self.visiting = Some(node), None => self.walking = false, } } + Step::Report => { + let before = self.asked.len(); + for (_, _, want) in &mut self.parked { + if let Some((first, _)) = want.report() { + self.asked.push_back(first); + } + } + if self.asked.len() == before { + return false; + } + } + Step::Answer => { + let Some(page) = self.asked.pop_front() else { return false }; + if !self.o.gone() { + self.next += 1; + let _ = self.p.supply(&mut self.o, page, self.next, false, (page + 1) * PAGE); + } + } + Step::Retry if !self.parked.is_empty() => { + let parked = core::mem::take(&mut self.parked); + for (s, page, mut want) in parked { + if !self.mapped[s as usize] || self.touch(s, page) { + continue; + } + match self.p.lookup(&self.o, page) { + Lookup::Absent => { + want.ask(page, 1); + self.parked.push((s, page, want)); + } + // Refused: failed, past the end, or gone. + _ => {} + } + } + } Step::Visit if self.visiting.is_some() => { let s = self.visiting.take().unwrap() as usize; for page in 0..2 { @@ -1891,6 +1957,30 @@ mod tests { true } + /// A fault's hit: `page` and its neighbour written where present. + fn touch(&mut self, s: u32, page: u64) -> bool { + let Lookup::Present(_) = self.p.lookup(&self.o, page) else { return false }; + for at in 0..2 { + if let (None, Lookup::Present(f)) = (self.leaves[s as usize][at], self.p.lookup(&self.o, at as u64)) { + self.leaves[s as usize][at] = Some(f); + } + } + true + } + + /// Once nothing is removed any more, the walk ends, the server + /// answers every request it hears of, and every parked fault + /// retries: none is left parked. + pub(super) fn drain(&mut self, steps: &[Step]) { + let fair = [Step::Walk, Step::Visit, Step::Report, Step::Answer, Step::Retry]; + for _ in 0..64 { + if !fair.iter().fold(false, |moved, &step| self.step(step) | moved) { + break; + } + } + assert!(self.parked.is_empty(), "after {steps:?}: faults left parked {:?}", self.parked); + } + fn revoke(&mut self) { match self.o.revoke() { Revoke::Walk => { @@ -1923,7 +2013,7 @@ mod tests { steps.iter().all(|&s| w.step(s)).then_some(w) } - /// From both spaces mapping both pages, every sequence of six steps of + /// From both spaces mapping both pages, every sequence of five steps of /// faults, supplies, removals, mappings and the walk keeps each leaf on /// its page's one frame. #[test] @@ -1934,7 +2024,8 @@ mod tests { let mut played = 0u64; while let Some(seq) = stack.pop() { played += 1; - if seq.len() == prefix.len() + 6 { + play(&seq).expect("a sequence played before").drain(&seq); + if seq.len() == prefix.len() + 5 { continue; } for &step in &mapped::STEPS { @@ -1945,7 +2036,7 @@ mod tests { } } } - assert!(played > 50_000, "only {played} sequences played"); + assert!(played > 20_000, "only {played} sequences played"); } /// The same over long sequences, seeded: removals queued behind running @@ -1959,6 +2050,7 @@ mod tests { x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); w.step(mapped::STEPS[(x >> 33) as usize % mapped::STEPS.len()]); } + w.drain(&[]); } } } From 4aa615394bf0f4e75300f4919e8e5e090c8e1400 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 19:38:35 +0200 Subject: [PATCH 04/34] A fault's wait ends where its object's version moved, and it asks afresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design rev 5 §3 (after review r1 of #1233): a waiter's wait ends on a write or resize of its object as well as on its page, and a fault, which has no read to answer `WouldBlock`, then asks for its page afresh. The index gains the two pieces the fault path will use: `Object::version`, where a fault's wait begins, and `Want::anew`, a request named in a notice whatever the old one said. The model's parked faults record the version they parked at, look again whether or not their wake was theirs, and ask afresh where the version moved. `a_fault_answered_then_shrunk_and_grown_asks_afresh` plays the strand rev 5 names: a fault's request read and answered, then a shrink and a grow before it looks. Asking the same want instead leaves it parked for ever. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 80 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 03b089cca1..129f4468ad 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -86,6 +86,13 @@ impl Want { true } + /// The waiter asks afresh, its wait having ended on a change of the + /// object: named in a notice whatever the old request said, as a new + /// read's would be. + pub fn anew(&mut self, first: u64, count: u64) { + *self = Self { first, count, reported: false }; + } + /// `count` pages from `first` were withdrawn: whether that makes the /// range news again. pub fn withdrawn(&mut self, first: u64, count: u64) -> bool { @@ -380,6 +387,13 @@ impl Object { (self.writing == 0).then_some(Version(self.version)) } + /// Where a fault's wait for one of its pages begins: a write or resize + /// since ends it ([`Pages::wait_over`]), and the fault asks afresh + /// ([`Want::anew`]). + pub fn version(&self) -> Version { + Version(self.version) + } + /// Whether no write or resize has begun since the read began at `v`. pub fn unchanged(&self, v: Version) -> bool { self.version == v.0 @@ -1815,17 +1829,17 @@ mod tests { visiting: Option, /// Faults parked on an absent page: the space, the page, and the /// request as the pager hears of it. - parked: std::vec::Vec<(u32, u64, Want)>, + parked: std::vec::Vec<(u32, u64, Want, Version)>, /// Pages the server read requests for and has yet to supply. asked: std::collections::VecDeque, } /// The parked faults' wants, which removals name again. - struct Parked<'a>(&'a mut std::vec::Vec<(u32, u64, Want)>); + struct Parked<'a>(&'a mut std::vec::Vec<(u32, u64, Want, Version)>); impl Waiters for Parked<'_> { fn each(&mut self, f: &mut dyn FnMut(&mut Want)) { - for (_, _, want) in self.0.iter_mut() { + for (_, _, want, _) in self.0.iter_mut() { f(want); } } @@ -1865,12 +1879,12 @@ mod tests { } Step::Fault(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_none() => { if !self.touch(s, page) { - if self.p.lookup(&self.o, page) != Lookup::Absent || self.parked.iter().any(|&(t, q, _)| (t, q) == (s, page)) { + if self.p.lookup(&self.o, page) != Lookup::Absent || self.parked.iter().any(|&(t, q, _, _)| (t, q) == (s, page)) { return false; } let mut want = Want::NONE; want.ask(page, 1); - self.parked.push((s, page, want)); + self.parked.push((s, page, want, self.o.version())); } } Step::Supply(page) if !self.o.gone() => { @@ -1909,7 +1923,7 @@ mod tests { } Step::Report => { let before = self.asked.len(); - for (_, _, want) in &mut self.parked { + for (_, _, want, _) in &mut self.parked { if let Some((first, _)) = want.report() { self.asked.push_back(first); } @@ -1927,14 +1941,21 @@ mod tests { } Step::Retry if !self.parked.is_empty() => { let parked = core::mem::take(&mut self.parked); - for (s, page, mut want) in parked { + // A wake may be stale: each looks whether or not its + // wait is over, as a thread woken for another's page. + for (s, page, mut want, since) in parked { if !self.mapped[s as usize] || self.touch(s, page) { continue; } match self.p.lookup(&self.o, page) { + // A write or resize ended the wait: ask afresh. + Lookup::Absent if !self.o.unchanged(since) => { + want.anew(page, 1); + self.parked.push((s, page, want, self.o.version())); + } Lookup::Absent => { want.ask(page, 1); - self.parked.push((s, page, want)); + self.parked.push((s, page, want, since)); } // Refused: failed, past the end, or gone. _ => {} @@ -2013,19 +2034,15 @@ mod tests { steps.iter().all(|&s| w.step(s)).then_some(w) } - /// From both spaces mapping both pages, every sequence of five steps of - /// faults, supplies, removals, mappings and the walk keeps each leaf on - /// its page's one frame. - #[test] - fn every_interleaving_keeps_each_leaf_on_its_pages_one_frame() { - use mapped::Step; - let prefix = [Step::Map(0), Step::Map(1), Step::Supply(0), Step::Supply(1), Step::Fault(0, 0), Step::Fault(1, 1)]; + /// Every sequence of `depth` steps after `prefix`, each then drained: how + /// many were played. + fn explore(prefix: &[mapped::Step], depth: usize) -> u64 { let mut stack = std::vec![prefix.to_vec()]; let mut played = 0u64; while let Some(seq) = stack.pop() { played += 1; play(&seq).expect("a sequence played before").drain(&seq); - if seq.len() == prefix.len() + 5 { + if seq.len() == prefix.len() + depth { continue; } for &step in &mapped::STEPS { @@ -2036,9 +2053,40 @@ mod tests { } } } + played + } + + /// From both spaces mapping both pages, every sequence of five steps of + /// faults, supplies, removals, mappings and the walk keeps each leaf on + /// its page's one frame, and leaves no fault parked once drained. + #[test] + fn every_interleaving_keeps_each_leaf_on_its_pages_one_frame() { + use mapped::Step; + let played = explore(&[Step::Map(0), Step::Map(1), Step::Supply(0), Step::Supply(1), Step::Fault(0, 0), Step::Fault(1, 1)], 5); assert!(played > 20_000, "only {played} sequences played"); } + /// A fault whose request the pager read and answered, before the fault + /// looked again, meets a shrink and a grow: the version moved, so it + /// asks afresh and is heard. + #[test] + fn a_fault_answered_then_shrunk_and_grown_asks_afresh() { + use mapped::Step; + let steps = [ + Step::Map(0), + Step::Fault(0, 1), + Step::Report, + Step::Answer, + Step::Shrink, + Step::Walk, + Step::Visit, + Step::Walk, + Step::Grow, + Step::Retry, + ]; + play(&steps).expect("every step is taken").drain(&steps); + } + /// The same over long sequences, seeded: removals queued behind running /// walks, ends meeting walks, unmaps while their node is visited. #[test] From 806bcf6e780b88be352bffacd57f6cc149638cbb Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:10:59 +0200 Subject: [PATCH 05/34] A paged object's range in a space, linked on its object's list of mappers 7b step 2, the kernel's mapper list. An object range of a space maps either a memory object, as before, or a paged object (`mm::space::Mapped`). A paged range holds a counted handle (`PagedRange`) on its space's one node for the object (`MapNode`, charged to the mapper's job): a clone is another range of it, and the last one, which the space drops only after the range's leaves are cleared and shot down, unlinks the node. The node is on the object's list under the pager's lock, linked seen at the object's epoch, and holds a reference on the object's slot. A removal's walk now visits that list: under the pager's lock it takes the next node, holds it, drops the lock and visits the node's space, which clears every leaf of its ranges of the object the index no longer holds present as that frame, ends futex waits on those frames and shoots them down; the slot is held across the walk. A removal that finds another's walk running waits for the pass that takes its pages, reported past the pager's report time; a release and the pager's end never wait. A fault on a paged range writes the page and its present neighbours under the pager's lock, so no leaf maps a revoking page's frame. `vm_map` of a paged holder maps it read-only and shared, with `READ`, refused to a pager's process as its reads are. Not yet: a fault on an absent page is refused rather than parking, a space's end through one owner, and the SDK's call. Co-Authored-By: Claude Opus 5.5 --- kernel/src/mm/space.rs | 114 +++++++++++++- kernel/src/object/paged.rs | 287 +++++++++++++++++++++++++++++++----- kernel/src/syscall/paged.rs | 2 +- kernel/src/syscall/vm.rs | 23 ++- 4 files changed, 379 insertions(+), 47 deletions(-) diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 16ba4c85db..bfd8b898aa 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -33,13 +33,32 @@ use super::charge::{self, Charge}; use super::paging::{Leaf, Mapping, Root, Unlinked, UserTables}; use super::policy::{self, CachePolicy}; use super::{pmm, PAGE_2M, PAGE_SIZE}; +use crate::object::paged::{MapNode, PagedHolderObject, PagedRange}; use crate::object::shm::SharedMemObject; use crate::sync::{Lock, LockGuard}; use crate::UserAddr; -type Inventory = vm::Inventory, Arc>; -type SpaceEntry = Entry, Arc>; -type SpaceKind = Kind, Arc>; +type Inventory = vm::Inventory>; +type SpaceEntry = Entry>; +type SpaceKind = Kind>; + +/// What an object range maps: a memory object's pages, fixed for its life, +/// or a paged object's, which its pager's index holds and revokes. +#[derive(Clone)] +pub enum Mapped { + Memory(Arc), + Paged(PagedRange), +} + +impl vm::Object for Mapped { + fn same(&self, other: &Self) -> bool { + match (self, other) { + (Mapped::Memory(a), Mapped::Memory(b)) => Arc::ptr_eq(a, b), + (Mapped::Paged(a), Mapped::Paged(b)) => core::ptr::eq(a.node(), b.node()), + _ => false, + } + } +} /// The bytes a copied range's pages start as. A read runs under the space /// lock, from a fault or a user copy, so it may take no lock a user copy can @@ -502,10 +521,73 @@ impl AddressSpace { self.room_at(len, at)?; let start = self.place(len, at)?; self.build(start, start + len)?; - self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object, offset, max: prot } }); + self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object: Mapped::Memory(object), offset, max: prot } }); Ok(UserAddr::new(start)) } + /// `len` bytes of `holder`'s object from `offset`, as `prot` allows, + /// sharing the index's frames: one node per object in a space, so a + /// second range of the object holds the first's. `space` is this space, + /// whose lock the caller holds. All or nothing, as + /// [`reserve`](Self::reserve). + pub fn map_paged(&mut self, space: &Arc, holder: &PagedHolderObject, offset: u64, len: u64, prot: Prot, at: At) -> Result { + if !offset.is_multiple_of(PAGE_SIZE) || offset.checked_add(len).is_none_or(|end| end > kernel::paged::LAST_END) { + return Err(SyscallError::InvalidArgument); + } + self.room_at(len, at)?; + let start = self.place(len, at)?; + let held = self.inventory.meeting(FLOOR, TOP).find_map(|(_, entry)| match &entry.kind { + Kind::Object { object: Mapped::Paged(range), .. } if holder.maps(range) => Some(range.clone()), + _ => None, + }); + let range = match held { + Some(range) => range, + None => holder.link(Arc::downgrade(space), &self.job)?, + }; + self.build(start, start + len)?; + self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object: Mapped::Paged(range), offset, max: prot } }); + Ok(UserAddr::new(start)) + } + + /// The visit of a walk of `node`'s object: every leaf of this space's + /// ranges of it that the index no longer holds present as that frame is + /// cleared, every futex wait on its frame ends, and every CPU drops what + /// it cached of them. A fault meanwhile waits on this space's lock and + /// then finds the page as the index has it. + pub fn revoke_paged(&mut self, node: &MapNode) { + let tables = &mut self.tables; + let mut cleared = [0u64; 64]; + let mut n = 0; + node.with_present(|present| { + for (start, entry) in self.inventory.meeting(FLOOR, TOP) { + let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; + if !core::ptr::eq(range.node(), node) { + continue; + } + let mut stale = false; + tables.edit(start, entry.end, |va, leaf| match leaf { + Leaf::Mapped(m) if present((offset + (va - start)) / PAGE_SIZE) != Some(m.phys) => { + stale = true; + if n == cleared.len() { + cleared.sort_unstable(); + crate::sched::futex::revoke_sorted(&cleared, |&phys| phys); + n = 0; + } + cleared[n] = m.phys; + n += 1; + Some(Leaf::Empty) + } + _ => None, + }); + if stale { + let _reached = tables.shoot_down(start, entry.end, &Unlinked::none()); + } + } + }); + cleared[..n].sort_unstable(); + crate::sched::futex::revoke_sorted(&cleared[..n], |&phys| phys); + } + /// Take every page of `[start, start + len)` away, whatever holds it: the /// POSIX `munmap` of any run of pages. Nothing held there is not a refusal. pub fn release(&mut self, start: UserAddr, len: u64) { @@ -790,7 +872,7 @@ impl AddressSpace { let mut at = start; while at < end { let next = self.inventory.meeting(at, end).find_map(|(s, entry)| match &entry.kind { - Kind::Object { object: held, offset, .. } + Kind::Object { object: Mapped::Memory(held), offset, .. } if core::ptr::eq(Arc::as_ptr(held), object) && *offset + at.saturating_sub(s) == at.max(s) - start => { Some((at.max(s), entry.end.min(end))) @@ -886,11 +968,28 @@ impl AddressSpace { let (lo, hi) = vm::around(start, entry.end, page, |at| leaves.covers(at) && leaves.get(at) != Leaf::Empty); match &entry.kind { Kind::Private { committed: false } => unreachable!("space: a reservation at {page:#x} allows an access"), - Kind::Object { object, offset, .. } => { + Kind::Object { object: Mapped::Memory(object), offset, .. } => { let phys = |at: u64| object.page(offset + (at - start)); leaves.fill(lo, hi, |at| Leaf::Mapped(Mapping { phys: phys(at), prot, owned: false, cache: object.cache() })); phys(page) } + // The index's own frames, written under its lock: a + // revoking page reads absent, so no leaf maps a frame on + // its way out. + Kind::Object { object: Mapped::Paged(range), offset, .. } => { + let of = |at: u64| (offset + (at - start)) / PAGE_SIZE; + range.node().with_present(|present| { + let own = present(of(page))?; + let mut at = lo; + while at < hi { + if let (Leaf::Empty, Some(phys)) = (leaves.get(at), present(of(at))) { + leaves.set(at, Leaf::Mapped(Mapping { phys, prot, owned: false, cache: CachePolicy::Normal })); + } + at += PAGE_SIZE; + } + Some(own) + })? + } Kind::Private { committed: true } | Kind::Copy { .. } => { let (mut filled, mut own) = (0, 0); leaves.fill(lo, hi, |at| { @@ -924,7 +1023,8 @@ impl AddressSpace { /// Whether a range's pages are memory the direct map reaches. fn copyable(entry: &SpaceEntry) -> bool { match &entry.kind { - Kind::Object { object, .. } => object.is_ordinary(), + Kind::Object { object: Mapped::Memory(object), .. } => object.is_ordinary(), + Kind::Object { object: Mapped::Paged(_), .. } => true, Kind::Private { .. } | Kind::Copy { .. } => true, } } diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 071a635a59..2d58ba9a05 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -42,7 +42,7 @@ //! life (`issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md`), //! so making an object is the one call that allocates. -use alloc::sync::Arc; +use alloc::sync::{Arc, Weak}; use core::cell::{Cell, UnsafeCell}; use core::marker::PhantomPinned; use core::pin::Pin; @@ -55,10 +55,11 @@ use kernel::sched::task::WaitClass; use kernel::slots::Slots; use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; -use crate::mm::charge::Charge; +use crate::mm::charge::{self, Charge}; use crate::mm::pmm; +use crate::mm::space::Space; use crate::process::Pid; -use crate::sync::Lock; +use crate::sync::{Lock, LockGuard}; use crate::time::{Deadline, Duration}; use crate::user_ptr::{UserBytes, UserBytesMut}; use crate::watch::{self, Watch}; @@ -104,6 +105,14 @@ struct Entry { holders: u32, /// Not released: owner calls reach it. owned: bool, + /// The spaces mapping it, one node each ([`MapNode`]). + mappers: Ends, +} + +/// The ends of an object's list of mappers. +struct Ends { + head: *const MapNode, + tail: *const MapNode, } struct State { @@ -306,7 +315,8 @@ impl PagerObject { let State { pages, entries, .. } = &mut *state; let made = entries.insert_with(|slot| { let object = if anonymous { pages.anonymous(slot, size) } else { pages.object(slot, size) }; - Entry { object, refs: 1, holders: 0, owned: true } + let mappers = Ends { head: ptr::null(), tail: ptr::null() }; + Entry { object, refs: 1, holders: 0, owned: true, mappers } }); u64::from(made.expect("paged: a slot was made free")) } @@ -319,61 +329,97 @@ impl PagerObject { let mut waiters = Linked { head: state.requests, slot }; let (pages, object) = state.split(slot); let answer = f(pages, object, &mut waiters); - self.revoked(&mut state, slot); + let (mut state, queued) = self.revoked(state, slot); + let Some(epoch) = queued else { + drop(state); + self.changed.post(); + return Ok(answer); + }; + // A walk runs: its next pass takes these pages, and the call returns + // once it has, with the slot held meanwhile. + state.entry_mut(slot).refs += 1; drop(state); self.changed.post(); + let _held = Ref { pager: self, slot }; + let parkable = crate::scheduler::Parkable::at_entry(); + let over = || self.state.lock().entry(slot).object.finished(epoch); + // A kill ends either wait, and its thread never reads the answer. A + // walk takes no wait of its own, so one past the report time is a + // kernel fault worth a line, and the wait goes on. + let report = Deadline::at(crate::clock::now() + self.report); + if watch::wait_until(&parkable, &self.changed, 0, WaitClass::Io, report, over).is_ok() && !over() { + log!("paged: pid {}'s removal waits past {} ms on another's walk", self.pid, self.report.nanos() / 1_000_000); + let _ = watch::wait_until(&parkable, &self.changed, 0, WaitClass::Io, Deadline::never(), over); + } Ok(answer) } - /// Under the lock, `state`: the pages of the object at `slot` just marked - /// revoking leave once no mapping reaches them, each frame given back, - /// and every waiter's want a page that ends absent meets is named again - /// in a notice (`Pages::walk`). - fn revoked(&self, state: &mut State, slot: u32) { - let job = self.core.job(); - let mut waiters = Linked { head: state.requests, slot }; - let State { pages, entries, held, gone, .. } = &mut *state; - let object = &mut entries.get_mut(slot).expect("paged: a slot an object names is held").object; - match object.revoke() { - Revoke::Done => return, + /// The pages of the object at `slot` just marked revoking leave once no + /// mapping reaches them: this walks the object's mappers, the lock dropped + /// for each space's visit and the slot held meanwhile, gives each frame + /// back, and names again every waiter's want a page that ends absent meets + /// (`Pages::walk`). Answers the lock and, where another's walk runs and + /// takes the pages in its next pass, that pass's epoch. + fn revoked<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> (LockGuard<'a, State>, Option) { + match state.entry_mut(slot).object.revoke() { + Revoke::Done => return (state, None), + Revoke::Queued(epoch) => return (state, Some(epoch)), Revoke::Walk => {} - Revoke::Queued(_) => unreachable!("paged: a walk outlived the hold of the lock that began it"), } + let job = self.core.job(); + state.entry_mut(slot).refs += 1; let mut absent = false; - let next = pages.walk(object, &mut Unmapped, &mut waiters, &mut |phys, _, becomes| { - held.give_back(job, *gone, phys, true); - absent |= becomes == Revoked::Absent; - }); - debug_assert!(next.is_none()); + loop { + let mut waiters = Linked { head: state.requests, slot }; + let State { pages, entries, held, gone, .. } = &mut *state; + let Entry { object, mappers, .. } = entries.get_mut(slot).expect("paged: a walked slot is held"); + let next = pages.walk(object, &mut Ring(mappers), &mut waiters, &mut |phys, _, becomes| { + held.give_back(job, *gone, phys, true); + absent |= becomes == Revoked::Absent; + }); + let Some(node) = next else { break }; + // SAFETY: linked, so live, under the lock; a range holds it, so + // it upgrades, and the count kept holds it past its unlinking + // while the lock is dropped. + let node = unsafe { (*node).this.upgrade() }.expect("paged: a linked mapper is held by a range"); + drop(state); + if let Some(space) = node.space.upgrade() { + space.lock().revoke_paged(&node); + } + state = self.state.lock(); + } + self.unref_locked(&mut state, slot); if absent { self.watch.post(); } + (state, None) } /// The pager lets object `id` go, in one hold of the lock: it answers /// `Gone` from now on, its pages go, and its slot with its last holder. A /// second release, or one racing the pager's end, finds it released. pub fn release(&self, id: u64) -> u64 { - let mut state = self.state.lock(); + let state = self.state.lock(); let slot = match state.live(id) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; - self.let_go(&mut state, slot); - drop(state); + drop(self.let_go(state, slot)); self.changed.post(); 0 } /// Under the lock, `state`: the unreleased object at `slot` released. - fn let_go(&self, state: &mut State, slot: u32) { + fn let_go<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { let State { pages, entries, .. } = &mut *state; let entry = entries.get_mut(slot).expect("paged: a slot an object names is held"); pages.invalidate(&mut entry.object); assert!(core::mem::take(&mut entry.owned), "paged: an object released twice"); entries.unmark(slot); - self.revoked(state, slot); - self.unref_locked(state, slot); + // A walk already running takes the pages; nothing here waits. + let (mut state, _) = self.revoked(state, slot); + self.unref_locked(&mut state, slot); + state } /// A holder of object `id`, which counts as one once its handle is @@ -1147,7 +1193,7 @@ impl ZeroHandles for PagerObject { state.gone = true; for slot in 0..state.entries.len() as u32 { if state.entries.get(slot).is_some_and(|e| e.owned) { - self.let_go(&mut state, slot); + state = self.let_go(state, slot); } } let job = self.core.job(); @@ -1172,18 +1218,172 @@ impl Drop for PagerObject { } } -/// No space maps a paged object yet: every walk ends at its first step. -struct Unmapped; +/// One space's mappings of one paged object, on the object's list from its +/// space's first range of the object until its last, which unlinks it only +/// once that range's leaves are cleared and shot down: the space drops a +/// removed range after its shootdown. It holds a reference on the object's +/// slot, so a mapped object outlives its release, and is charged to the +/// mapper's job. +pub struct MapNode { + pager: Arc, + slot: u32, + space: Weak, + /// The ranges holding it ([`PagedRange`]), each under its space's lock. + ranges: AtomicU32, + /// Under the pager's lock. + link: UnsafeCell, + /// Itself, for a walk to hold while it visits. + this: Weak, + _charge: Charge, +} + +struct Link { + /// The epoch of the walk that last visited it, or the one current at + /// its linking ([`kernel::paged::Mappers`]). + seen: u64, + prev: *const MapNode, + next: *const MapNode, +} -impl Mappers for Unmapped { - type Node = (); - fn head(&self) -> Option<()> { - None +// SAFETY: `link` is reached only under its pager's lock, and the rest is +// immutable or atomic. +unsafe impl Send for MapNode {} +// SAFETY: as for `Send`. +unsafe impl Sync for MapNode {} + +impl MapNode { + /// Under the pager's lock, through `present`, which answers the frame the + /// index holds present for a page of the object, or `None`: `f`'s answer. + pub fn with_present(&self, f: impl FnOnce(&dyn Fn(u64) -> Option) -> R) -> R { + let state = self.pager.state.lock(); + let object = &state.entry(self.slot).object; + let present = |page: u64| match state.pages.lookup(object, page) { + Lookup::Present(phys) => Some(phys), + _ => None, + }; + f(&present) } - fn seen(&self, (): ()) -> u64 { - 0 +} + +/// A range's hold on its space's [`MapNode`]: a clone is another range of +/// the same object in the space, and the last one dropped unlinks the node. +pub struct PagedRange(Arc); + +impl PagedRange { + pub fn node(&self) -> &MapNode { + &self.0 + } +} + +impl Clone for PagedRange { + fn clone(&self) -> Self { + self.0.ranges.fetch_add(1, Ordering::Relaxed); + Self(Arc::clone(&self.0)) + } +} + +impl Drop for PagedRange { + fn drop(&mut self) { + if self.0.ranges.fetch_sub(1, Ordering::Relaxed) == 1 { + self.0.pager.unlink(&self.0); + } + } +} + +impl PagerObject { + /// The first range of object `slot` in `space`, whose memory is counted + /// to `job`: its node linked, seen at the object's epoch now, under + /// the space's lock the caller holds. `Gone` where the object or its + /// pager is. + pub fn link(self: &Arc, slot: u32, space: Weak, job: &Arc) -> Result { + let charge = Charge::new(job, charge::arc::()).map_err(|_| SyscallError::ResourceExhausted)?; + let link = UnsafeCell::new(Link { seen: 0, prev: ptr::null(), next: ptr::null() }); + let node = Arc::new_cyclic(|this| MapNode { + pager: Arc::clone(self), + slot, + space, + ranges: AtomicU32::new(1), + link, + this: this.clone(), + _charge: charge, + }); + let mut state = self.state.lock(); + let entry = state.entry_mut(slot); + if entry.object.gone() { + return Err(SyscallError::Gone); + } + let at = Arc::as_ptr(&node); + // SAFETY: a fresh node no list holds yet, and the tail's, linked, + // under the lock. + unsafe { + *(*at).link.get() = Link { seen: entry.object.epoch(), prev: entry.mappers.tail, next: ptr::null() }; + match entry.mappers.tail.is_null() { + true => entry.mappers.head = at, + false => (*(*entry.mappers.tail).link.get()).next = at, + } + } + entry.mappers.tail = at; + entry.refs += 1; + Ok(PagedRange(node)) + } + + /// The last range of `node`'s space went, its leaves cleared and shot + /// down: the node leaves the list, and its reference on the slot goes. + fn unlink(&self, node: &MapNode) { + let mut state = self.state.lock(); + let entry = state.entry_mut(node.slot); + // SAFETY: linked, as are its neighbours, under the lock. + unsafe { + let Link { prev, next, .. } = *node.link.get(); + match prev.is_null() { + true => entry.mappers.head = next, + false => (*(*prev).link.get()).next = next, + } + match next.is_null() { + true => entry.mappers.tail = prev, + false => (*(*next).link.get()).prev = prev, + } + } + self.unref_locked(&mut state, node.slot); + } +} + +/// An object's list of mappers, as its walk visits them, under the pager's +/// lock. +struct Ring<'a>(&'a mut Ends); + +impl Mappers for Ring<'_> { + type Node = *const MapNode; + + fn head(&self) -> Option<*const MapNode> { + (!self.0.head.is_null()).then_some(self.0.head) + } + + fn seen(&self, node: *const MapNode) -> u64 { + // SAFETY: linked, so live, under the lock. + unsafe { (*(*node).link.get()).seen } + } + + fn visited(&mut self, node: *const MapNode, epoch: u64) { + // SAFETY: linked, as are its neighbours, under the lock; it moves to + // the tail. + unsafe { + let link = &mut *(*node).link.get(); + link.seen = epoch; + if self.0.tail == node { + return; + } + match link.prev.is_null() { + true => self.0.head = link.next, + false => (*(*link.prev).link.get()).next = link.next, + } + (*(*link.next).link.get()).prev = link.prev; + (*(*self.0.tail).link.get()).next = node; + link.prev = self.0.tail; + link.next = ptr::null(); + self.0.tail = node; + } } - fn visited(&mut self, (): (), _: u64) {} } /// One call's reference to its pager's object. @@ -1227,6 +1427,17 @@ impl PagedHolderObject { pub fn read(&self, out: &mut UserBytesMut, offset: u64) -> u64 { self.named.pager.read(self.named.slot, out, offset, true) } + + /// Whether `range` maps this holder's object. + pub fn maps(&self, range: &PagedRange) -> bool { + ptr::eq(Arc::as_ptr(&range.0.pager), Arc::as_ptr(&self.named.pager)) && range.0.slot == self.named.slot + } + + /// The first range of its object in `space`, whose memory is counted to + /// `job` ([`PagerObject::link`]). + pub fn link(&self, space: Weak, job: &Arc) -> Result { + self.named.pager.link(self.named.slot, space, job) + } } impl ZeroHandles for PagedHolderObject { diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index 953861cdf0..dbbe960f22 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -77,7 +77,7 @@ pub(super) fn sys_paged_read(h: RawHandle, out: &mut UserBytesMut, offset: u64) } } -fn holder(h: RawHandle) -> Result, u64> { +pub(super) fn holder(h: RawHandle) -> Result, u64> { let handles = process::handles(); let resolved = readable_holder(&handles.lock(), h, handles.is_pager()); match resolved { diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index f9fdf51b71..1c24114b0f 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -11,6 +11,7 @@ use toyos_abi::handle::{RawHandle, Rights}; use crate::mm::space::At; use crate::object::shm::SharedMemObject; +use crate::object::KObjectRef; use crate::user_ptr::SyscallContext; use crate::UserAddr; use crate::process; @@ -90,11 +91,16 @@ pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) Ok(asked) => asked, Err(e) => return SyscallError::to_u64(e), }; + let space = process::current_address_space(); + match process::with_handles(|handles| handles.get_ref(h, Rights::NONE).map(|o| matches!(o, KObjectRef::Paged(_)))) { + Ok(true) => return map_paged(h, &space, args.offset, len, prot, at), + Ok(false) => {} + Err(e) => return e.refuse(), + } let object = match process::with_handles(|handles| handles.get::(h, map_rights(prot))) { Ok(object) => object, Err(e) => return e.refuse(), }; - let space = process::current_address_space(); let mapped = space.lock().map(alloc::sync::Arc::clone(&object), args.offset, len, prot, at); let at = match mapped { Ok(at) => at, @@ -109,6 +115,21 @@ pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) at.raw() } +/// A paged object's holder `h`: its pages shared, read-only, as `READ` +/// allows, and refused a pager's process as its reads are +/// (`syscall::paged`). +fn map_paged(h: RawHandle, space: &alloc::sync::Arc, offset: u64, len: u64, prot: Prot, at: At) -> u64 { + if prot != Prot::Read { + return SyscallError::InvalidArgument.to_u64(); + } + let holder = match super::paged::holder(h) { + Ok(holder) => holder, + Err(refused) => return refused, + }; + let mapped = space.lock().map_paged(space, &holder, offset, len, prot, at); + mapped.map_or_else(SyscallError::to_u64, UserAddr::raw) +} + /// Take every page of `[addr, addr + len)` away, as POSIX `munmap` does. pub(super) fn sys_vm_release(addr: u64, len: u64) -> u64 { answer(release(addr, len)) From bc017e4386a4bf8a934597acdcc7f2bb67efda5f Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:14:42 +0200 Subject: [PATCH 06/34] A fault on an absent paged page parks on its pager and touches it again 7b step 2, the fault path. A touch of a paged range's absent page answers `Populated::Wait` with the object's page and its version, read under the pager's lock. `handle_page_fault` then waits holding no lock (`MapNode::wait`): the page's aligned 64 KiB run is named in a fresh request, so a fault whose last wait a write or resize ended asks afresh (design rev 5), and the wait lasts until the page is no longer absent, the version moved, or the pager is gone (`Pages::wait_over`), reported overdue past the pager's report time and never ended by a timeout (D6). It touches the page again, under the space's lock, until it is served or refused. While it waits the CPU's fault mark is down, and it is up again on whichever CPU the thread wakes on, so a fault that parks is no recursion on either architecture. A kill that ends the wait answers `Faulted::Killed`: a user fault returns and the thread exits on its way back to user code; a user copy stops at its fixup instead of faulting again. A kernel reach of a paged page that is absent is refused; only a fault waits. Not yet: a fault refused on a paged page names its cause (a copy answers `BadAddress`, not `Io`), and a hole of an anonymous object is refused rather than given a zeroed frame (N4). Co-Authored-By: Claude Opus 5.5 --- kernel/src/arch/aarch64/trap.rs | 5 +- kernel/src/arch/x86_64/idt/exceptions.rs | 5 +- kernel/src/mm/space.rs | 75 ++++++++++++++++------ kernel/src/object/paged.rs | 81 +++++++++++++++++++++--- kernel/src/process.rs | 47 +++++++++++--- 5 files changed, 174 insertions(+), 39 deletions(-) diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 546bb2eb2a..4707e2827d 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -164,7 +164,7 @@ fn user_copy_abort(frame: &mut Frame) -> bool { let served = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); percpu::set_fault_state(CpuFaultState::Normal); - if served { + if served == crate::process::Faulted::Served { return true; } } @@ -356,7 +356,8 @@ fn user_abort(frame: &mut Frame) { }; let served = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); - if served { + // A kill that ended a wait is acted on as the thread returns to EL0. + if served != crate::process::Faulted::Refused { percpu::set_fault_state(CpuFaultState::Normal); percpu::preempt_count_down(); return; diff --git a/kernel/src/arch/x86_64/idt/exceptions.rs b/kernel/src/arch/x86_64/idt/exceptions.rs index a67e742e6a..f3b26905b2 100644 --- a/kernel/src/arch/x86_64/idt/exceptions.rs +++ b/kernel/src/arch/x86_64/idt/exceptions.rs @@ -532,7 +532,8 @@ pub(super) fn page_fault_handler(frame: &mut TrapFrame) { } else { kernel::vm::Access::Read }; - if is_user && process::handle_page_fault(fault_addr, access) { + // A kill that ended a wait is acted on as the thread returns to Ring 3. + if is_user && process::handle_page_fault(fault_addr, access) != process::Faulted::Refused { percpu::set_fault_state(percpu::CpuFaultState::Normal); return; } @@ -563,7 +564,7 @@ fn user_copy_fault(frame: &mut TrapFrame, addr: u64, fixup: u64) { percpu::preempt_count_down(); let served = process::handle_page_fault(addr, access); percpu::preempt_count_up(); - if served { + if served == process::Faulted::Served { return; } } diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index bfd8b898aa..fd75591d25 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -33,7 +33,7 @@ use super::charge::{self, Charge}; use super::paging::{Leaf, Mapping, Root, Unlinked, UserTables}; use super::policy::{self, CachePolicy}; use super::{pmm, PAGE_2M, PAGE_SIZE}; -use crate::object::paged::{MapNode, PagedHolderObject, PagedRange}; +use crate::object::paged::{MapNode, Missing, PagedHolderObject, PagedRange}; use crate::object::shm::SharedMemObject; use crate::sync::{Lock, LockGuard}; use crate::UserAddr; @@ -558,7 +558,7 @@ impl AddressSpace { let tables = &mut self.tables; let mut cleared = [0u64; 64]; let mut n = 0; - node.with_present(|present| { + node.with_present(|object| { for (start, entry) in self.inventory.meeting(FLOOR, TOP) { let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; if !core::ptr::eq(range.node(), node) { @@ -566,7 +566,7 @@ impl AddressSpace { } let mut stale = false; tables.edit(start, entry.end, |va, leaf| match leaf { - Leaf::Mapped(m) if present((offset + (va - start)) / PAGE_SIZE) != Some(m.phys) => { + Leaf::Mapped(m) if object.page((offset + (va - start)) / PAGE_SIZE) != Ok(m.phys) => { stale = true; if n == cleared.len() { cleared.sort_unstable(); @@ -932,27 +932,32 @@ impl AddressSpace { } /// Serve a fault at `va` for `access`: populate its page if its range - /// allows the access and the page is empty. `false` ends the thread. - pub fn fault(&mut self, va: u64, access: Access) -> bool { - self.populate(va, access, |_| true).is_some() + /// allows the access and the page is empty, or say which paged page to + /// wait for. `Refused` ends the thread. + pub fn fault(&mut self, va: u64, access: Access) -> Populated { + self.populate(va, access, |_| true) } /// Where the byte at `va` is, for the kernel to reach through the direct /// map, populating its page where its range allows `access`; `None` for - /// an aperture's page, which the direct map does not reach. + /// an aperture's page, which the direct map does not reach, and for a + /// paged page that is absent, which only a fault waits for. pub fn reach(&mut self, va: u64, access: Access) -> Option { - self.populate(va, access, copyable) + match self.populate(va, access, copyable) { + Populated::At(phys) => Some(phys), + Populated::Refused | Populated::Wait(_) => None, + } } /// The frame behind `va`, populated where its range allows `access` and /// is one `admits`. - fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Option { - let (start, entry) = self.inventory.find(va)?; + fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Populated { + let Some((start, entry)) = self.inventory.find(va) else { return Populated::Refused }; if !admits(entry) || !entry.prot.allows(access) { - return None; + return Populated::Refused; } let page = va & !(PAGE_SIZE - 1); - let prot = leaf_prot(entry.prot)?; + let Some(prot) = leaf_prot(entry.prot) else { return Populated::Refused }; // A range that allows an access is backed, so its tables are built. let mut leaves = self.tables.leaf_table(page).unwrap_or_else(|| panic!("space: no tables under {page:#x}")); let phys = match leaves.get(page) { @@ -978,17 +983,22 @@ impl AddressSpace { // its way out. Kind::Object { object: Mapped::Paged(range), offset, .. } => { let of = |at: u64| (offset + (at - start)) / PAGE_SIZE; - range.node().with_present(|present| { - let own = present(of(page))?; + let found = range.node().with_present(|object| { + let own = object.page(of(page))?; let mut at = lo; while at < hi { - if let (Leaf::Empty, Some(phys)) = (leaves.get(at), present(of(at))) { + if let (Leaf::Empty, Ok(phys)) = (leaves.get(at), object.page(of(at))) { leaves.set(at, Leaf::Mapped(Mapping { phys, prot, owned: false, cache: CachePolicy::Normal })); } at += PAGE_SIZE; } - Some(own) - })? + Ok(own) + }); + match found { + Ok(phys) => phys, + Err(Missing::Absent(since)) => return Populated::Wait(PageWait { range: range.clone(), page: of(page), since }), + Err(Missing::Refused) => return Populated::Refused, + } } Kind::Private { committed: true } | Kind::Copy { .. } => { let (mut filled, mut own) = (0, 0); @@ -1007,7 +1017,7 @@ impl AddressSpace { } } }; - Some(phys + va % PAGE_SIZE) + Populated::At(phys + va % PAGE_SIZE) } /// The frame and protection a present leaf maps `va` with, populating @@ -1020,6 +1030,35 @@ impl AddressSpace { } } +/// What a touch of a page found. +pub enum Populated { + /// The byte's physical address. + At(u64), + /// No range allows the access there, or its paged page failed, lies past + /// the object's end or its pager is gone. + Refused, + /// A paged page that is absent: the thread waits for it, holding nothing, + /// and touches it again. + Wait(PageWait), +} + +/// A paged page a fault waits for: the object's page, and its version when +/// it was found absent ([`MapNode::wait`]). +pub struct PageWait { + range: PagedRange, + page: u64, + since: kernel::paged::Version, +} + +impl PageWait { + /// Wait, holding no lock, until the page is no longer absent, the + /// object's version moved, or its pager is gone: `false` where a kill + /// ended the wait. + pub fn wait(self) -> bool { + self.range.node().wait(self.page, self.since) + } +} + /// Whether a range's pages are memory the direct map reaches. fn copyable(entry: &SpaceEntry) -> bool { match &entry.kind { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 2d58ba9a05..09b2531e02 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -1251,17 +1251,82 @@ unsafe impl Send for MapNode {} // SAFETY: as for `Send`. unsafe impl Sync for MapNode {} +/// Why the index holds no frame a fault may map for a page. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Missing { + /// Absent, or revoking, at this version of the object: the fault waits. + Absent(Version), + /// Failed, past the end, a hole of an anonymous object, or the pager is + /// gone: the fault is refused. + Refused, +} + +/// A mapped object's pages as its index holds them, under the pager's lock. +pub struct Index<'a> { + state: &'a State, + slot: u32, +} + +impl Index<'_> { + /// The frame present for the object's `page`, or why there is none. + pub fn page(&self, page: u64) -> Result { + let object = &self.state.entry(self.slot).object; + match self.state.pages.lookup(object, page) { + _ if self.state.gone => Err(Missing::Refused), + Lookup::Present(phys) => Ok(phys), + Lookup::Absent => Err(Missing::Absent(object.version())), + Lookup::Zero | Lookup::Failed | Lookup::PastEnd | Lookup::Gone => Err(Missing::Refused), + } + } +} + +/// The pages a fault's request names: the aligned run around its page. +const FAULT_RUN: u64 = 16; + impl MapNode { - /// Under the pager's lock, through `present`, which answers the frame the - /// index holds present for a page of the object, or `None`: `f`'s answer. - pub fn with_present(&self, f: impl FnOnce(&dyn Fn(u64) -> Option) -> R) -> R { + /// `f` over the object's pages, under the pager's lock. + pub fn with_present(&self, f: impl FnOnce(&Index<'_>) -> R) -> R { let state = self.pager.state.lock(); - let object = &state.entry(self.slot).object; - let present = |page: u64| match state.pages.lookup(object, page) { - Lookup::Present(phys) => Some(phys), - _ => None, + f(&Index { state: &state, slot: self.slot }) + } + + /// A fault's wait for the object's `page`, found absent at `since`, its + /// space's lock not held: the page's aligned run is requested in a fresh + /// request, so a fault whose last wait a write or resize ended asks + /// afresh, and the wait lasts until the page is no longer absent, the + /// version moved, or the pager is gone. Reported overdue past the pager's + /// report time, as a holder's read is, and never ended by a timeout; a + /// file's page behaves as memory does (7b's D6). `false` where a kill + /// ended it. + pub fn wait(&self, page: u64, since: Version) -> bool { + let (pager, slot) = (&*self.pager, self.slot); + let over = |state: &State| state.gone || state.pages.wait_over(&state.entry(slot).object, page, since); + let waiting = core::pin::pin!(Waiting::new(pager, slot)); + let (first, count) = { + let mut state = pager.state.lock(); + if over(&state) { + return true; + } + // Absent, so inside the object. + let end = state.entry(slot).object.size().div_ceil(PAGE); + let first = page - page % FAULT_RUN; + let count = (first + FAULT_RUN).min(end) - first; + waiting.as_ref().request(&mut state, first, count); + (first, count) }; - f(&present) + let ready = || over(&pager.state.lock()); + let parkable = crate::scheduler::Parkable::at_entry(); + let began = crate::clock::now(); + if watch::wait_until(&parkable, &pager.changed, 0, WaitClass::Io, Deadline::at(began + pager.report), ready).is_err() { + return false; + } + if !ready() { + let _overdue = Overdue::begin(pager, slot, first, count, began); + if watch::wait_until(&parkable, &pager.changed, 0, WaitClass::Io, Deadline::never(), ready).is_err() { + return false; + } + } + true } } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 3c72a6ab2b..bf82d6c36d 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -958,17 +958,46 @@ pub fn ask_join(join: &join::Join, tid: Tid, parent_pid: Pid) -> Option bool { - let Some(space) = scheduler::current_address_space() else { return false }; +/// What serving a fault came to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Faulted { + /// The page is populated, or already held a leaf allowing the access. + Served, + /// No range allows the access, or its paged page cannot be had. + Refused, + /// A kill ended a wait for a paged page: the thread never runs its user + /// code again, and a copy stops where it is. + Killed, +} + +/// Serve a fault at `fault_addr` for `access` in the running thread's space, +/// at its trap's depth with interrupts on: its page populated under the +/// space's lock, or, for a paged page that is absent, waited for holding no +/// lock and touched again. While it waits the CPU's fault mark is down, so +/// another fault on this CPU meanwhile is no recursion; it is up again on +/// whichever CPU the thread wakes on. +pub fn handle_page_fault(fault_addr: u64, access: kernel::vm::Access) -> Faulted { + use crate::arch::percpu::{self, CpuFaultState}; + let Some(space) = scheduler::current_address_space() else { return Faulted::Refused }; let began = crate::clock::nanos_since_boot(); - let served = space.lock().fault(fault_addr, access); - if served { - space.usage().faulted(crate::clock::nanos_since_boot().saturating_sub(began)); + loop { + let populated = space.lock().fault(fault_addr, access); + match populated { + crate::mm::space::Populated::At(_) => break, + crate::mm::space::Populated::Refused => return Faulted::Refused, + crate::mm::space::Populated::Wait(page) => { + percpu::set_fault_state(CpuFaultState::Normal); + let woken = page.wait(); + let prev = percpu::swap_fault_state(CpuFaultState::PageFault); + assert_eq!(prev, CpuFaultState::Normal, "a fault woke on a CPU marked {prev:?}"); + if !woken { + return Faulted::Killed; + } + } + } } - served + space.usage().faulted(crate::clock::nanos_since_boot().saturating_sub(began)); + Faulted::Served } /// Dump memory around `fault_addr` and the instruction pointer for the From 3659ef26480f4bd004d623d2cd3b39b579fb47e2 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:15:33 +0200 Subject: [PATCH 07/34] A process's end empties its space, so a revoker's last reference frees nothing 7b step 2, a space's end through one owner. `AddressSpace::end` takes every range away as a release does, its leaves cleared and shot down before a paged range lets its node go; a process's teardown calls it after closing its handles. A walk that holds a space it visits then drops, at worst, the last reference to an empty space, never a teardown inside its pager's call. A space dropped still holding a paged range is a kernel fault: a spawn lays out none in this step. Co-Authored-By: Claude Opus 5.5 --- kernel/src/mm/space.rs | 14 +++++++++++++- kernel/src/process.rs | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index fd75591d25..b729b26104 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -613,6 +613,15 @@ impl AddressSpace { self.settle(); } + /// Empty the space as its process ends, every range taken away as + /// [`release`](Self::release) takes one: a paged range lets its node go + /// only after its leaves are cleared and shot down, so the space's last + /// reference, wherever it drops (a revoker's visit among them), frees + /// nothing but its empty tables. + pub fn end(&mut self) { + self.release(UserAddr::new(FLOOR), TOP - FLOOR); + } + /// Give every page of `[start, start + len)` protection `prot`: a reserved /// private page is committed unless `prot` is `None`, promised its frame /// here, and a committed one keeps its frame and commitment whatever @@ -1096,9 +1105,12 @@ fn copy_in(source: &dyn CopySource, offset: u64, len: u64, phys: u64) { /// Not through [`AddressSpace::clear`]: no thread runs in a space that drops, /// no CPU holds its root ([`Space::retire`]), and a private frame is mapped in /// no other, so no CPU caches its leaves, no futex waiter is keyed on its -/// frames and no copy pins them. +/// frames and no copy pins them. A paged range is gone by then: its process's +/// end took every range ([`AddressSpace::end`]), and a spawn lays out none. impl Drop for AddressSpace { fn drop(&mut self) { + let paged = self.inventory.meeting(FLOOR, TOP).any(|(_, entry)| matches!(entry.kind, Kind::Object { object: Mapped::Paged(_), .. })); + assert!(!paged, "space: dropped holding a paged range its end did not take"); let committed = self.committed_pages(FLOOR, TOP); self.tables.edit(FLOOR, TOP, |_, leaf| matches!(leaf, Leaf::Mapped(Mapping { owned: false, .. })).then_some(Leaf::Empty)); let mut freed = 0; diff --git a/kernel/src/process.rs b/kernel/src/process.rs index bf82d6c36d..47bfcb5bb3 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -663,6 +663,7 @@ fn teardown_resources( let handles = Arc::clone(&process_data_arc.lock().handles); ops::close_all(&mut handles.lock()); + process_data_arc.lock().space.lock().end(); // Every thread has left, and none returns to Ring 3 to use them. crate::isa::process_ends(pid); let space = Arc::clone(&process_data_arc.lock().space); From 7e62271600d7970936f77da29c9a04fef21b67d6 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:18:05 +0200 Subject: [PATCH 08/34] A served file is mapped read-only through its holder, and libc maps one 7b step 2, the SDK's and libc's half. `toyos::paged::Paged::map` maps a holder's object read-only and shared; `toyos::files::map` does so for an open file through the paged holder its server supplied, and answers `NotSupported` for a file with none. libc's `mmap` of a file, read-only and shared or private (the same pages until a write, which a read-only mapping never makes), now maps it; `offset` off a page is `EINVAL`, a file with no pages to map `ENODEV`, and a file asked for anything but `PROT_READ` stays `ENODEV` until 7b's later steps (a private copy on write or execute, and writable shared mappings). Co-Authored-By: Claude Opus 5.5 --- sdk/libc/arch/src/memory_refusals.rs | 9 +++++++-- sdk/libc/src/memreq.rs | 5 +++-- sdk/libc/src/posix_io.rs | 21 ++++++++++++++++++++- sdk/toyos/src/files.rs | 17 +++++++++++++++++ sdk/toyos/src/files/owner.rs | 23 +++++++++++++++++++++++ sdk/toyos/src/paged.rs | 13 +++++++++++++ 6 files changed, 83 insertions(+), 5 deletions(-) diff --git a/sdk/libc/arch/src/memory_refusals.rs b/sdk/libc/arch/src/memory_refusals.rs index d69741ecd5..bed36ad9a4 100644 --- a/sdk/libc/arch/src/memory_refusals.rs +++ b/sdk/libc/arch/src/memory_refusals.rs @@ -1,5 +1,5 @@ //! What libc's memory calls refuse before the kernel or the allocator is -//! asked: `mmap` a file, memory writable and executable at once, no bytes, a fixed place off a +//! asked: `mmap` a file other than read-only, memory writable and executable at once, no bytes, a fixed place off a //! page, or no one disposition; `posix_madvise` anything but its five; `madvise` the same, an //! address off a page, and `MADV_DONTNEED`'s discard. @@ -22,8 +22,13 @@ fn mmap_refuses_what_the_kernel_cannot_map_and_nothing_else() { (0, 4096, rw, MAP_ANONYMOUS, Some(MapRefusal::Invalid)), (0, 4096, rw, MAP_SHARED | MAP_PRIVATE | MAP_ANONYMOUS, Some(MapRefusal::Invalid)), (0, 4096, PROT_READ, 0, Some(MapRefusal::Invalid)), - (0, 4096, PROT_READ, MAP_PRIVATE, Some(MapRefusal::File)), + // A file read-only, shared or private, and nothing else of one. + (0, 4096, PROT_READ, MAP_PRIVATE, None), + (0, 4096, PROT_READ, MAP_SHARED, None), (0, 4096, rw, MAP_SHARED, Some(MapRefusal::File)), + (0, 4096, rw, MAP_PRIVATE, Some(MapRefusal::File)), + (0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, Some(MapRefusal::File)), + (0, 4096, 0, MAP_SHARED, Some(MapRefusal::File)), (0, 4096, PROT_READ | PROT_EXEC, anonymous, None), (0, 4096, PROT_WRITE | PROT_EXEC, anonymous, Some(MapRefusal::WriteExec)), (0, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, anonymous, Some(MapRefusal::WriteExec)), diff --git a/sdk/libc/src/memreq.rs b/sdk/libc/src/memreq.rs index 1b3f1123db..8c19848e0b 100644 --- a/sdk/libc/src/memreq.rs +++ b/sdk/libc/src/memreq.rs @@ -26,7 +26,8 @@ pub(crate) enum MapRefusal { /// No bytes, a fixed place off a page boundary, or not exactly one of /// `MAP_SHARED` and `MAP_PRIVATE`: `EINVAL`. Invalid, - /// A file's bytes, which the kernel maps for no process: `ENODEV`. + /// A file's bytes other than read-only, which no mapping of a file gives + /// yet: `ENODEV`. File, /// Memory writable and executable at once, which no page is: `ENOTSUP`. WriteExec, @@ -43,7 +44,7 @@ pub(crate) fn mmap_refusal(addr: usize, len: usize, prot: i32, flags: i32) -> Op { return Some(MapRefusal::Invalid); } - if flags & MAP_ANONYMOUS == 0 { + if flags & MAP_ANONYMOUS == 0 && prot != PROT_READ { return Some(MapRefusal::File); } if prot & PROT_EXEC != 0 && prot & PROT_WRITE != 0 { diff --git a/sdk/libc/src/posix_io.rs b/sdk/libc/src/posix_io.rs index 7a96cba49d..e6e52e9eae 100644 --- a/sdk/libc/src/posix_io.rs +++ b/sdk/libc/src/posix_io.rs @@ -823,7 +823,7 @@ fn range_errno(e: syscall::SyscallError) -> i32 { #[no_mangle] pub unsafe extern "C" fn mmap( - addr: *mut u8, len: usize, prot: i32, flags: i32, _fd: i32, _offset: i64, + addr: *mut u8, len: usize, prot: i32, flags: i32, fd: i32, offset: i64, ) -> *mut u8 { use crate::memreq::{self, MapRefusal}; @@ -845,6 +845,25 @@ pub unsafe extern "C" fn mmap( // POSIX takes a null address with `MAP_FIXED` as the address zero, which // no process holds; anything else without it is only a hint. let at = if flags & memreq::MAP_FIXED != 0 { syscall::VmAt::Fixed(addr) } else { syscall::VmAt::ANYWHERE }; + if flags & memreq::MAP_ANONYMOUS == 0 { + // Read-only, so private and shared are the same pages until a write. + let Some(offset) = u64::try_from(offset).ok().filter(|o| o.is_multiple_of(syscall::PAGE_SIZE as u64)) else { + crate::errno::set(EINVAL); + return MAP_FAILED; + }; + // SAFETY: a `MAP_FIXED` caller gives up what those pages held, as POSIX says. + return match unsafe { files::map(fd, offset, len, at) } { + Ok(ptr) => ptr, + Err(files::Error::Kernel(syscall::SyscallError::NotSupported)) => { + crate::errno::set(crate::errcode::ENODEV); + MAP_FAILED + } + Err(e) => { + refuse_files(e); + MAP_FAILED + } + }; + } // SAFETY: a `MAP_FIXED` caller gives up what those pages held, as POSIX says. match unsafe { syscall::vm_reserve(len, prot, at) } { Ok(ptr) => ptr, diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index 492b7111a8..e7ae79ba4e 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -223,6 +223,7 @@ extern "C" { fn toyos_fs_write(fd: Fd, buf: *const u8, len: usize) -> i64; fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u64) -> i64; fn toyos_fs_pwrite(fd: Fd, buf: *const u8, len: usize, offset: u64) -> i64; + fn toyos_fs_map(fd: Fd, offset: u64, len: usize, place: u64, fixed: bool) -> i64; fn toyos_fs_seek(fd: Fd, whence: u32, delta: i64) -> i64; fn toyos_fs_truncate(fd: Fd, size: u64) -> i64; fn toyos_fs_fsync(fd: Fd) -> i64; @@ -276,6 +277,22 @@ pub fn pread(fd: Fd, buf: &mut [u8], offset: u64) -> Result { decode(unsafe { toyos_fs_pread(fd, buf.as_mut_ptr(), buf.len(), offset) }).map(|n| n as usize) } +/// `len` bytes of the file `fd` from the page boundary `offset`, mapped +/// read-only and shared ([`crate::paged::Paged::map`]) where `at` says. +/// `Kernel(NotSupported)` for a file its server maps no pages of. +/// +/// # Safety +/// A `Fixed` range replaces what those pages held, so nothing may still +/// reach that. +pub unsafe fn map(fd: Fd, offset: u64, len: usize, at: toyos_abi::syscall::VmAt) -> Result<*mut u8, Error> { + let (place, fixed) = match at { + toyos_abi::syscall::VmAt::Fixed(addr) => (addr as u64, true), + toyos_abi::syscall::VmAt::Anywhere { .. } => (0, false), + }; + // SAFETY: the caller's. + decode(unsafe { toyos_fs_map(fd, offset, len, place, fixed) }).map(|addr| core::ptr::with_exposed_provenance_mut(addr as usize)) +} + /// Write at `offset`, leaving the descriptor's offset where it is, whether or /// not it appends. pub fn pwrite(fd: Fd, buf: &[u8], offset: u64) -> Result { diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index e0c89c61c1..c7b276fe89 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -553,6 +553,29 @@ unsafe extern "C" fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u6 answer(n.map(|n| n as u64)) } +/// `len` bytes of the file `fd` names from the page boundary `offset`, mapped +/// read-only and shared through its paged holder, at `place` where `fixed`, +/// else anywhere: the address. `NotSupported` for a file its server supplied +/// no holder for, and a descriptor that names no served file. +/// +/// # Safety +/// A fixed range replaces what those pages held, so nothing may still reach +/// that. +#[no_mangle] +unsafe extern "C" fn toyos_fs_map(fd: Fd, offset: u64, len: usize, place: u64, fixed: bool) -> i64 { + let at = if fixed { syscall::VmAt::Fixed(place as usize) } else { syscall::VmAt::ANYWHERE }; + let refused = || Ok(Err(SyscallError::NotSupported)); + let mapped = on_fd_held( + fd, + |_| refused(), + is_directory, + // SAFETY: the caller's. + |holder, _| Some(unsafe { holder.map(offset, len, at) }.map(|ptr| ptr.addr() as u64)), + |_, _, _| refused(), + ); + answer(mapped.and_then(|inner| inner.map_err(Error::Kernel))) +} + #[no_mangle] unsafe extern "C" fn toyos_fs_pwrite(fd: Fd, buf: *const u8, len: usize, offset: u64) -> i64 { // SAFETY: the caller's. diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 98aa2d10ff..81daa4924c 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -226,4 +226,17 @@ impl Paged { pub fn read(&self, out: &mut [u8], offset: u64) -> Result { syscall::paged_read(self.0.raw(), out, offset) } + + /// `len` bytes of the object from the page boundary `offset`, mapped + /// read-only and shared where `at` says: the pager's own frames, so a + /// write of its lands in them, and a touch of a page not yet supplied + /// waits for it as a read does. A touch of a page that failed, past the + /// object's end, or after its pager ended, ends the process. + /// + /// # Safety + /// As [`syscall::vm_map`]'s: a `Fixed` range replaces what those pages held. + pub unsafe fn map(&self, offset: u64, len: usize, at: syscall::VmAt) -> Result<*mut u8, SyscallError> { + // SAFETY: the caller's. + unsafe { syscall::vm_map(self.0.raw(), offset, len, syscall::VmProt::Read, at) } + } } From cf4086fa5c2952c6f2c18d62b1c2f0872a89dcb5 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:27:22 +0200 Subject: [PATCH 09/34] The file mapping entry names a fixed place as a pointer, and its refusals in each closure's type The ToyOS-only path of eab7f28c7 did not build: `VmAt::Fixed` takes a pointer, and the served closure answers `fs::Error`. The image builds now. Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/src/files.rs | 2 +- sdk/toyos/src/files/owner.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index e7ae79ba4e..018f8c0da7 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -286,7 +286,7 @@ pub fn pread(fd: Fd, buf: &mut [u8], offset: u64) -> Result { /// reach that. pub unsafe fn map(fd: Fd, offset: u64, len: usize, at: toyos_abi::syscall::VmAt) -> Result<*mut u8, Error> { let (place, fixed) = match at { - toyos_abi::syscall::VmAt::Fixed(addr) => (addr as u64, true), + toyos_abi::syscall::VmAt::Fixed(addr) => (addr.expose_provenance() as u64, true), toyos_abi::syscall::VmAt::Anywhere { .. } => (0, false), }; // SAFETY: the caller's. diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index c7b276fe89..1f0aa42df3 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -563,15 +563,17 @@ unsafe extern "C" fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u6 /// that. #[no_mangle] unsafe extern "C" fn toyos_fs_map(fd: Fd, offset: u64, len: usize, place: u64, fixed: bool) -> i64 { - let at = if fixed { syscall::VmAt::Fixed(place as usize) } else { syscall::VmAt::ANYWHERE }; - let refused = || Ok(Err(SyscallError::NotSupported)); + let at = match fixed { + true => syscall::VmAt::Fixed(core::ptr::with_exposed_provenance_mut(place as usize)), + false => syscall::VmAt::ANYWHERE, + }; let mapped = on_fd_held( fd, - |_| refused(), + |_| Ok(Err(SyscallError::NotSupported)), is_directory, // SAFETY: the caller's. - |holder, _| Some(unsafe { holder.map(offset, len, at) }.map(|ptr| ptr.addr() as u64)), - |_, _, _| refused(), + |holder, _| Some(unsafe { holder.map(offset, len, at) }.map(|ptr| ptr.expose_provenance() as u64)), + |_, _, _| Ok(Err(SyscallError::NotSupported)), ); answer(mapped.and_then(|inner| inner.map_err(Error::Kernel))) } From 1e796da46d5ad0e400a4f669c5486d5a8735b9c7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:44:24 +0200 Subject: [PATCH 10/34] A lost mapped page ends a program as SIGBUS and a copy as Io, and no pinned copy waits A touch of a paged range now says why it was refused. `Populated` splits `Refused` into `Unreachable` (no range allows the access, or the page lies past the object's end) and `Lost`, whose record names the pager, the object, the page and the cause: failed by its pager, released, the pager ended, or no room for a hole's page. - A Ring 3 touch of a lost page ends the process with the new `Fault::Mapping` (word 8, `SIGBUS` in the SDK and libc) and one log line naming what was lost, on both architectures. - A user copy's fault leaves its cause in a per-thread word (`Copies`, on the thread's `TaskHandle`), so `user_ptr::Faulted` is `Unreachable` or `Lost` and a syscall answers `BadAddress` or `Io`. Pipe reads and writes, paged reads, writes, supplies and notices carry it instead of flattening it. - A futex wait or wake on a mapped file's page that is absent or lost is a spurious wake, as the manual says of swapped and lost words. No wait holds a pin. A holder's read and a pager's write copy through `user_ptr::Pinned`, under which a copy's fault on an absent paged page stops the copy instead of waiting; the call lets its pins go, waits for the window's pages holding nothing (`wait_in`) and copies the rest of the chunk again. A touched hole of an anonymous object (a MEMORY volume's) gets a zeroed frame of its pager's, drawn and charged as a supply's, installed clean under the pager's lock; a full table or a refused promise refuses the fault with `NoRoom`. Fault-around maps only present pages. A mapping counts as one of its object's holders, so a file mapped and closed, or unlinked, is not collected on the last handle's notice while mapped. The range count's last drop, which may be a parked fault's with no space lock, takes `Arc`'s release/acquire ordering before it unlinks. The module header says what it now holds: mappings, revoking pages, the lock order space, pager, allocator. Co-Authored-By: Claude Opus 5.5 --- kernel/src/arch/aarch64/trap.rs | 26 ++- kernel/src/arch/aarch64/uaccess.rs | 12 +- kernel/src/arch/x86_64/idt/exceptions.rs | 33 +++- kernel/src/arch/x86_64/uaccess.rs | 12 +- kernel/src/mm/space.rs | 53 +++--- kernel/src/object/ops.rs | 4 +- kernel/src/object/paged.rs | 223 ++++++++++++++++++----- kernel/src/pipe.rs | 18 +- kernel/src/process.rs | 116 +++++++++--- kernel/src/sched/driver.rs | 5 + kernel/src/sched/payload.rs | 7 + kernel/src/user_ptr.rs | 183 +++++++++++++++++-- sdk/abi/src/end.rs | 12 +- sdk/libc/arch/src/spawn_requests.rs | 1 + sdk/toyos/src/process.rs | 3 +- 15 files changed, 559 insertions(+), 149 deletions(-) diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 4707e2827d..2cd9a45f01 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -151,6 +151,7 @@ fn user_copy_abort(frame: &mut Frame) -> bool { if frame.esr >> 26 != EC_DABT_SAME || !far_valid || !toyos_userbound::is_user_addr(frame.far) { return false; } + let mut faulted = crate::process::Faulted::Unreachable; if is_translation_fault(frame.esr) { let prev = percpu::swap_fault_state(CpuFaultState::PageFault); if prev != CpuFaultState::Normal { @@ -161,13 +162,14 @@ fn user_copy_abort(frame: &mut Frame) -> bool { cpu::enable_interrupts(); // `WnR`, bit 6: the access was a write. let access = if frame.esr & (1 << 6) != 0 { kernel::vm::Access::Write } else { kernel::vm::Access::Read }; - let served = crate::process::handle_page_fault(frame.far, access); + faulted = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); percpu::set_fault_state(CpuFaultState::Normal); - if served == crate::process::Faulted::Served { + if faulted == crate::process::Faulted::Served { return true; } } + crate::user_ptr::stopped(faulted.stop()); frame.elr = fixup; true } @@ -354,13 +356,21 @@ fn user_abort(frame: &mut Frame) { (false, true) => kernel::vm::Access::Write, (false, false) => kernel::vm::Access::Read, }; - let served = crate::process::handle_page_fault(frame.far, access); + let faulted = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); - // A kill that ended a wait is acted on as the thread returns to EL0. - if served != crate::process::Faulted::Refused { - percpu::set_fault_state(CpuFaultState::Normal); - percpu::preempt_count_down(); - return; + match faulted { + crate::process::Faulted::Unreachable => {} + crate::process::Faulted::Lost(lost) => { + percpu::set_fault_state(CpuFaultState::Normal); + crate::process::lost(frame.far, lost); + } + crate::process::Faulted::Absent => unreachable!("an EL0 fault while its thread's copies hold pins"), + // A kill that ended a wait is acted on as the thread returns to EL0. + crate::process::Faulted::Served | crate::process::Faulted::Killed => { + percpu::set_fault_state(CpuFaultState::Normal); + percpu::preempt_count_down(); + return; + } } log!( "#PF UNHANDLED: far={:#x} pc={:#x} esr={:#x} tid={:?}", diff --git a/kernel/src/arch/aarch64/uaccess.rs b/kernel/src/arch/aarch64/uaccess.rs index 38580fd699..3646cd71b7 100644 --- a/kernel/src/arch/aarch64/uaccess.rs +++ b/kernel/src/arch/aarch64/uaccess.rs @@ -23,7 +23,7 @@ use toyos_userbound::USER_TOP; -use crate::user_ptr::Faulted; +use crate::user_ptr::Stopped; core::arch::global_asm!( ".pushsection .text.toyos_uaccess, \"ax\"", @@ -165,8 +165,8 @@ unsafe extern "C" { } /// A routine's answer: 0 where it copied every byte. -fn whole(left: usize) -> Result<(), Faulted> { - if left == 0 { Ok(()) } else { Err(Faulted) } +fn whole(left: usize) -> Result<(), Stopped> { + if left == 0 { Ok(()) } else { Err(crate::user_ptr::last_stop()) } } /// Copy `len` bytes from user address `user` to `dst`. On `Err` some or none @@ -175,7 +175,7 @@ fn whole(left: usize) -> Result<(), Faulted> { /// # Safety /// `dst` is writable kernel memory for `len` bytes; `[user, user + len)` lies /// below [`USER_TOP`]; no lock is held (`crate::user_ptr`). -pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_copy_from_user(dst, user, len) }) } @@ -184,7 +184,7 @@ pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`], with `src` readable. -pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_copy_to_user(user, src, len) }) } @@ -193,7 +193,7 @@ pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`]. -pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_clear_user(user, len) }) } diff --git a/kernel/src/arch/x86_64/idt/exceptions.rs b/kernel/src/arch/x86_64/idt/exceptions.rs index f3b26905b2..4c37252e69 100644 --- a/kernel/src/arch/x86_64/idt/exceptions.rs +++ b/kernel/src/arch/x86_64/idt/exceptions.rs @@ -532,10 +532,21 @@ pub(super) fn page_fault_handler(frame: &mut TrapFrame) { } else { kernel::vm::Access::Read }; - // A kill that ended a wait is acted on as the thread returns to Ring 3. - if is_user && process::handle_page_fault(fault_addr, access) != process::Faulted::Refused { - percpu::set_fault_state(percpu::CpuFaultState::Normal); - return; + if is_user { + match process::handle_page_fault(fault_addr, access) { + process::Faulted::Unreachable => {} + process::Faulted::Lost(lost) => { + percpu::set_fault_state(percpu::CpuFaultState::Normal); + process::lost(fault_addr, lost); + } + process::Faulted::Absent => unreachable!("a Ring 3 fault while its thread's copies hold pins"), + // A kill that ended a wait is acted on as the thread returns + // to Ring 3. + process::Faulted::Served | process::Faulted::Killed => { + percpu::set_fault_state(percpu::CpuFaultState::Normal); + return; + } + } } log!("#PF UNHANDLED: cr2={:#x} rip={:#x} err={:#x} user={} tid={:?}", fault_addr, frame.rip, frame.error_code, is_user, percpu::current_tid()); @@ -560,14 +571,18 @@ fn user_copy_fault(frame: &mut TrapFrame, addr: u64, fixup: u64) { const AC: u64 = 1 << 18; assert!(frame.rflags & AC != 0, "uaccess: a copy faulted at {:#x} with RFLAGS.AC clear", frame.rip); let access = if frame.error_code & PF_WRITE != 0 { kernel::vm::Access::Write } else { kernel::vm::Access::Read }; - if frame.error_code & PF_PRESENT == 0 { + let faulted = if frame.error_code & PF_PRESENT == 0 { percpu::preempt_count_down(); - let served = process::handle_page_fault(addr, access); + let faulted = process::handle_page_fault(addr, access); percpu::preempt_count_up(); - if served == process::Faulted::Served { - return; - } + faulted + } else { + process::Faulted::Unreachable + }; + if faulted == process::Faulted::Served { + return; } + crate::user_ptr::stopped(faulted.stop()); frame.rip = fixup; } diff --git a/kernel/src/arch/x86_64/uaccess.rs b/kernel/src/arch/x86_64/uaccess.rs index 5d85738367..62009a7077 100644 --- a/kernel/src/arch/x86_64/uaccess.rs +++ b/kernel/src/arch/x86_64/uaccess.rs @@ -23,7 +23,7 @@ use toyos_userbound::USER_TOP; use super::copy::{self, copy_down_asm}; -use crate::user_ptr::Faulted; +use crate::user_ptr::Stopped; core::arch::global_asm!( ".pushsection .text.toyos_uaccess, \"ax\"", @@ -139,8 +139,8 @@ unsafe extern "C" { const REP_STRING_LEN: u64 = 2; /// A routine's answer: 0 where it copied every byte. -fn whole(left: usize) -> Result<(), Faulted> { - if left == 0 { Ok(()) } else { Err(Faulted) } +fn whole(left: usize) -> Result<(), Stopped> { + if left == 0 { Ok(()) } else { Err(crate::user_ptr::last_stop()) } } /// Copy `len` bytes from user address `user` to `dst`. On `Err` some or none @@ -149,7 +149,7 @@ fn whole(left: usize) -> Result<(), Faulted> { /// # Safety /// `dst` is writable kernel memory for `len` bytes; `[user, user + len)` lies /// below [`USER_TOP`]; no lock is held (`crate::user_ptr`). -pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; each routine touches nothing else, and // the rule sends down only a copy of at least 8 bytes. whole(unsafe { @@ -164,7 +164,7 @@ pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`], with `src` readable. -pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Stopped> { // SAFETY: as in `copy_from_user`. whole(unsafe { match copy::rule().goes_down(user as usize, src as usize, len) { @@ -178,7 +178,7 @@ pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`]. -pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_clear_user(user, len) }) } diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index b729b26104..186cc8655e 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -33,7 +33,7 @@ use super::charge::{self, Charge}; use super::paging::{Leaf, Mapping, Root, Unlinked, UserTables}; use super::policy::{self, CachePolicy}; use super::{pmm, PAGE_2M, PAGE_SIZE}; -use crate::object::paged::{MapNode, Missing, PagedHolderObject, PagedRange}; +use crate::object::paged::{Lost, MapNode, Missing, PagedHolderObject, PagedRange}; use crate::object::shm::SharedMemObject; use crate::sync::{Lock, LockGuard}; use crate::UserAddr; @@ -245,17 +245,23 @@ impl FutexWait { /// The frame behind the futex word at `addr`, populated where its range /// allows a read ([`AddressSpace::reach`]), as a translation the wait can - /// ask about later without the lock. - pub fn translate(&self, addr: crate::UserAddr) -> Option> { + /// ask about later without the lock; why there is none, waiting for no + /// mapped file's page. + pub fn translate(&self, addr: crate::UserAddr) -> Result, crate::user_ptr::Untranslated> { + use crate::user_ptr::Untranslated; let space: &Space = &self.0; let (phys, at) = { let mut held = space.lock(); - let phys = held.reach(addr.raw(), Access::Read)?; + let phys = match held.reach(addr.raw(), Access::Read) { + Populated::At(phys) => phys, + Populated::Unreachable => return Err(Untranslated::Unreachable), + Populated::Lost(_) | Populated::Wait(_) => return Err(Untranslated::Away), + }; // Counted under this hold: a change made before it is in `at`, // and one after it moves the count. (phys, space.usage.shot_down.load(Relaxed)) }; - Some(Translation { space, va: addr.raw(), phys, at: Cell::new(at) }) + Ok(Translation { space, va: addr.raw(), phys, at: Cell::new(at) }) } } @@ -558,7 +564,7 @@ impl AddressSpace { let tables = &mut self.tables; let mut cleared = [0u64; 64]; let mut n = 0; - node.with_present(|object| { + node.with_index(|object| { for (start, entry) in self.inventory.meeting(FLOOR, TOP) { let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; if !core::ptr::eq(range.node(), node) { @@ -942,31 +948,27 @@ impl AddressSpace { /// Serve a fault at `va` for `access`: populate its page if its range /// allows the access and the page is empty, or say which paged page to - /// wait for. `Refused` ends the thread. + /// wait for. pub fn fault(&mut self, va: u64, access: Access) -> Populated { self.populate(va, access, |_| true) } /// Where the byte at `va` is, for the kernel to reach through the direct - /// map, populating its page where its range allows `access`; `None` for - /// an aperture's page, which the direct map does not reach, and for a - /// paged page that is absent, which only a fault waits for. - pub fn reach(&mut self, va: u64, access: Access) -> Option { - match self.populate(va, access, copyable) { - Populated::At(phys) => Some(phys), - Populated::Refused | Populated::Wait(_) => None, - } + /// map, populating its page where its range allows `access`; an + /// aperture's page, which the direct map does not reach, is unreachable. + pub fn reach(&mut self, va: u64, access: Access) -> Populated { + self.populate(va, access, copyable) } /// The frame behind `va`, populated where its range allows `access` and /// is one `admits`. fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Populated { - let Some((start, entry)) = self.inventory.find(va) else { return Populated::Refused }; + let Some((start, entry)) = self.inventory.find(va) else { return Populated::Unreachable }; if !admits(entry) || !entry.prot.allows(access) { - return Populated::Refused; + return Populated::Unreachable; } let page = va & !(PAGE_SIZE - 1); - let Some(prot) = leaf_prot(entry.prot) else { return Populated::Refused }; + let Some(prot) = leaf_prot(entry.prot) else { return Populated::Unreachable }; // A range that allows an access is backed, so its tables are built. let mut leaves = self.tables.leaf_table(page).unwrap_or_else(|| panic!("space: no tables under {page:#x}")); let phys = match leaves.get(page) { @@ -992,8 +994,8 @@ impl AddressSpace { // its way out. Kind::Object { object: Mapped::Paged(range), offset, .. } => { let of = |at: u64| (offset + (at - start)) / PAGE_SIZE; - let found = range.node().with_present(|object| { - let own = object.page(of(page))?; + let found = range.node().with_index(|object| { + let own = object.fault(of(page))?; let mut at = lo; while at < hi { if let (Leaf::Empty, Ok(phys)) = (leaves.get(at), object.page(of(at))) { @@ -1006,7 +1008,8 @@ impl AddressSpace { match found { Ok(phys) => phys, Err(Missing::Absent(since)) => return Populated::Wait(PageWait { range: range.clone(), page: of(page), since }), - Err(Missing::Refused) => return Populated::Refused, + Err(Missing::PastEnd | Missing::Hole) => return Populated::Unreachable, + Err(Missing::Lost(lost)) => return Populated::Lost(lost), } } Kind::Private { committed: true } | Kind::Copy { .. } => { @@ -1043,9 +1046,11 @@ impl AddressSpace { pub enum Populated { /// The byte's physical address. At(u64), - /// No range allows the access there, or its paged page failed, lies past - /// the object's end or its pager is gone. - Refused, + /// No range allows the access there, or its paged page lies past the + /// object's end. + Unreachable, + /// A paged page its pager failed or can no longer supply. + Lost(Lost), /// A paged page that is absent: the thread waits for it, holding nothing, /// and touches it again. Wait(PageWait), diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index cb9d7ca2de..dc1ffedad6 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -341,7 +341,7 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { KObjectRef::PipeRead(r) => match r.pipe().read(&crate::scheduler::Parkable::at_entry(), buf) { Ok(read) => read.map(|n| n as u64), Err(pipe::ReadStopped::Cancelled) => Some(SyscallError::Gone.to_u64()), - Err(pipe::ReadStopped::Faulted) => Some(SyscallError::BadAddress.to_u64()), + Err(pipe::ReadStopped::Faulted(faulted)) => Some(SyscallError::from(faulted).to_u64()), }, KObjectRef::Device(_) => unreachable!("a device claim is read by `read_device`"), KObjectRef::Null(_) => Some(0), @@ -376,7 +376,7 @@ pub fn pipe_written(wrote: pipe::PipeWrite) -> Option { // The reader is gone, not absent: `NotFound` means the name is not there and nothing else may say so. pipe::PipeWrite::BrokenPipe => Some(SyscallError::Gone.to_u64()), pipe::PipeWrite::NoMemory => Some(SyscallError::ResourceExhausted.to_u64()), - pipe::PipeWrite::Faulted => Some(SyscallError::BadAddress.to_u64()), + pipe::PipeWrite::Faulted(faulted) => Some(SyscallError::from(faulted).to_u64()), pipe::PipeWrite::Wrote(n) => Some(n as u64), pipe::PipeWrite::NoRoom => None, } diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 09b2531e02..4e1a964309 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -1,6 +1,6 @@ //! Paged objects (memory design §7, stage 7a): a pager process's memory, -//! one object per file node, which holders read only through -//! `SYS_PAGED_READ`. Nothing here is mapped. +//! one object per file node, which holders read through `SYS_PAGED_READ` or +//! map read-only. //! //! **Two kinds.** A [`PagerObject`] is its process's one pager: its index of //! `capacity` pages ([`kernel::paged::Pages`]), its objects by id, its @@ -11,9 +11,16 @@ //! A [`PagedHolderObject`] reads one object, and may be narrowed and sent. //! //! **Supply copies into fresh frames**, so no frame the pager still maps -//! enters an object. A frame leaves at withdrawal, failure, shrink or the -//! object's release: out of the index under the pager's lock, then freed, -//! which a reader's pin defers to its last unpin. +//! enters an object. A frame leaves at withdrawal, failure, shrink, the +//! object's release or the pager's end: its page is revoking, keeping the +//! frame, until a walk of the object's mappers ([`MapNode`]) has cleared +//! every space's leaves of it; then it leaves the index under the pager's +//! lock and is freed, which a reader's pin defers to its last unpin. +//! +//! **A mapping shares the index's frames.** A space mapping an object holds +//! one [`MapNode`] on the object's list while it has a range of it, and +//! counts as one of its holders. A fault on an absent page requests it and +//! waits as a reader does, holding nothing. //! //! **A reader waits only on its pager.** An absent page is requested in the //! waiter's own record, linked on its pager's list, and the wait ends on a @@ -28,9 +35,10 @@ //! `WouldBlock`, and the holder asks the pager, whose own reads its writes //! order. //! -//! **Lock order.** The pager's lock is innermost but for the allocator's. A -//! copy to or from user memory holds pins, never the lock; no wait holds a -//! pin. +//! **Lock order.** A space's lock, then the pager's, then the allocator's. +//! A copy to or from user memory holds pins, never the lock; no wait holds a +//! pin, so such a copy stops at an absent page (`user_ptr::Pinned`), lets its +//! pins go, and waits for the page holding nothing. //! //! **What a pager holds** (`issues/a-pager-holds-a-fixed-capacity-until-the-memory-service-reclaims-its-clean-pages.md`): //! its core — `dirty` pages and a read's headroom — committed at its making @@ -61,7 +69,7 @@ use crate::mm::space::Space; use crate::process::Pid; use crate::sync::{Lock, LockGuard}; use crate::time::{Deadline, Duration}; -use crate::user_ptr::{UserBytes, UserBytesMut}; +use crate::user_ptr::{Pinned, Stopped, UserBytes, UserBytesMut}; use crate::watch::{self, Watch}; use crate::DirectMap; @@ -435,7 +443,12 @@ impl PagerObject { /// A holder object's handles went. fn holder_gone(&self, slot: u32) { - let mut state = self.state.lock(); + self.let_hold_go(&mut self.state.lock(), slot); + } + + /// One of the object's holders, a handle's or a space's mapping, went: + /// the last is said in a notice. + fn let_hold_go(&self, state: &mut State, slot: u32) { let gone = state.gone; let entry = state.entry_mut(slot); entry.holders -= 1; @@ -497,9 +510,9 @@ impl PagerObject { }; // SAFETY: a `PagedNotice` is `UserSafe`: every byte of it is a // field, and the batch outlives the copy. - if unsafe { out.write_raw(filled * size, batch.as_ptr().cast::(), n * size) }.is_err() { + if let Err(faulted) = unsafe { out.write_raw(filled * size, batch.as_ptr().cast::(), n * size) } { // The notices taken are lost with the window that could not hold them. - return if filled > 0 { filled as u64 } else { SyscallError::BadAddress.to_u64() }; + return if filled > 0 { filled as u64 } else { SyscallError::from(faulted).to_u64() }; } filled += n; if n < want { @@ -558,20 +571,23 @@ impl PagerObject { }; match step { Step::Copy { count, last } => { - let mut faulted = false; + let (first, chunk) = (done, pieces[..count].iter().map(|piece| piece.len).sum::()); + let mut stopped = None; + let pinned = Pinned::begin(); for piece in &pieces[..count] { let copied = match piece.phys { // SAFETY: `phys` is pinned by `pin_chunk`, and // `within + len` lies inside its frame. - Some(phys) => unsafe { out.write_raw(done, DirectMap::from_phys(phys).as_ptr::().add(piece.within), piece.len) }, - None => out.fill_zero(done, piece.len), + Some(phys) => unsafe { pinned.write(out, done, DirectMap::from_phys(phys).as_ptr::().add(piece.within), piece.len) }, + None => pinned.zero(out, done, piece.len), }; - if copied.is_err() { - faulted = true; + if let Err(why) = copied { + stopped = Some(why); break; } done += piece.len; } + drop(pinned); for piece in &pieces[..count] { if let Some(phys) = piece.phys { pmm::unpin_range(phys, PAGE as usize); } } @@ -588,11 +604,18 @@ impl PagerObject { return SyscallError::WouldBlock.to_u64(); } // After the checks: bytes copied before the fault count only - // as every copied chunk's do. - if faulted { - return if done > 0 { done as u64 } else { SyscallError::BadAddress.to_u64() }; + // as every copied chunk's do. A page of the buffer that is + // absent is waited for with nothing pinned, and the rest of + // the chunk read again. + let faulted = match stopped { + None => None, + Some(Stopped::Absent) => out.wait_in(done, first + chunk - done).err(), + Some(Stopped::Faulted(faulted)) => Some(faulted), + }; + if let Some(faulted) = faulted { + return if done > 0 { done as u64 } else { SyscallError::from(faulted).to_u64() }; } - if last { + if last && stopped.is_none() { return done as u64; } } @@ -783,6 +806,7 @@ impl PagerObject { let mut ends = [0u64; CHUNK]; // The frames filled from `src`, in order: a fault stops the fill. let mut filled = 0; + let mut faulted = None; for k in 0..reserved { let page_at = (first + k as u64) * PAGE; let from = done + k * PAGE as usize; @@ -803,8 +827,12 @@ impl PagerObject { frames[k] = phys; ends[k] = page_at + (src.len() - from).min(PAGE as usize) as u64; // SAFETY: a frame just drawn, which nothing else reaches yet. - if filled == k && unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::(), n) }.is_ok() { - filled += 1; + if filled == k { + // SAFETY: as above. + match unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::(), n) } { + Ok(()) => filled += 1, + Err(why) => faulted = Some(why), + } } } let mut refused = None; @@ -845,8 +873,8 @@ impl PagerObject { } } self.changed.post(); - if filled < reserved && !stopped { - refused.get_or_insert(SyscallError::BadAddress); + if let Some(faulted) = faulted.filter(|_| !stopped) { + refused.get_or_insert(faulted.into()); } if let Some(why) = refused { return refusal(installed, why); @@ -878,7 +906,7 @@ impl PagerObject { let mut pinned = [(0u64, 0u64, 0usize, 0usize); CHUNK]; let mut count = 0; let mut stopped = None; - let mut faulted = false; + let mut stopped_at = None; let slot = { let mut state = self.state.lock(); let slot = match state.live(id) { @@ -909,15 +937,18 @@ impl PagerObject { let mut from = done; // The pages whose bytes all came: a fault in the owner's buffer stops there. let mut copied = 0; + let copies = Pinned::begin(); for &(_, phys, within, n) in &pinned[..count] { // SAFETY: a frame of the object, pinned and write-pinned above. - if unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::().add(within), n) }.is_err() { - faulted = true; + if let Err(why) = unsafe { copies.read(src, from, DirectMap::from_phys(phys).as_mut_ptr::().add(within), n) } { + stopped_at = Some(why); break; } from += n; copied += 1; } + drop(copies); + let chunk: usize = pinned[..count].iter().map(|&(_, _, _, n)| n).sum(); { let mut state = self.state.lock(); for (k, &(page, phys, within, n)) in pinned[..count].iter().enumerate() { @@ -936,9 +967,19 @@ impl PagerObject { for &(_, phys, _, _) in &pinned[..count] { pmm::unpin_range(phys, PAGE as usize); } + // A page of the owner's buffer that is absent is waited for with + // nothing pinned, and the rest of the chunk written again. + let faulted = match stopped_at { + None => None, + Some(Stopped::Absent) => src.wait_in(from, done + chunk - from).err(), + Some(Stopped::Faulted(faulted)) => Some(faulted), + }; done = from; - if faulted { - return refusal(done as u64, SyscallError::BadAddress); + if let Some(faulted) = faulted { + return refusal(done as u64, faulted.into()); + } + if stopped_at.is_some() { + continue; } match stopped { None => {} @@ -1256,14 +1297,51 @@ unsafe impl Sync for MapNode {} pub enum Missing { /// Absent, or revoking, at this version of the object: the fault waits. Absent(Version), - /// Failed, past the end, a hole of an anonymous object, or the pager is - /// gone: the fault is refused. - Refused, + /// At or past the object's end: nothing is there. + PastEnd, + /// A hole of an anonymous object, which only [`Index::fault`] fills. + Hole, + /// Its pager cannot let a mapping have it. + Lost(Lost), +} + +/// A page a fault was refused, as the record of its end names it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Lost { + pub pager: Pid, + pub object: u32, + pub page: u64, + pub why: Why, +} + +/// Why a pager cannot let a mapping have a page. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Why { + /// The pager failed it. + Failed, + /// The pager released the object. + Released, + /// The pager ended. + Ended, + /// A hole of an anonymous object, and the pager has no room for its page. + NoRoom, +} + +impl Why { + pub fn name(self) -> &'static str { + match self { + Why::Failed => "failed by its pager", + Why::Released => "released by its pager", + Why::Ended => "its pager ended", + Why::NoRoom => "no room in its pager", + } + } } /// A mapped object's pages as its index holds them, under the pager's lock. pub struct Index<'a> { - state: &'a State, + pager: &'a PagerObject, + state: &'a mut State, slot: u32, } @@ -1272,12 +1350,66 @@ impl Index<'_> { pub fn page(&self, page: u64) -> Result { let object = &self.state.entry(self.slot).object; match self.state.pages.lookup(object, page) { - _ if self.state.gone => Err(Missing::Refused), + _ if self.state.gone => Err(self.lost(page, Why::Ended)), Lookup::Present(phys) => Ok(phys), Lookup::Absent => Err(Missing::Absent(object.version())), - Lookup::Zero | Lookup::Failed | Lookup::PastEnd | Lookup::Gone => Err(Missing::Refused), + Lookup::PastEnd => Err(Missing::PastEnd), + Lookup::Zero => Err(Missing::Hole), + Lookup::Failed => Err(self.lost(page, Why::Failed)), + Lookup::Gone => Err(self.lost(page, Why::Released)), } } + + /// As [`Self::page`], for the page a fault touched: a hole of an + /// anonymous object, a MEMORY volume's, gets a zeroed frame of its + /// pager's, installed clean and charged as a supply's is, and is refused + /// where the pager has no room for it (7b's N4). + pub fn fault(&mut self, page: u64) -> Result { + match self.page(page) { + Err(Missing::Hole) => self.zeroed(page), + found => found, + } + } + + fn zeroed(&mut self, page: u64) -> Result { + let no_room = self.lost(page, Why::NoRoom); + let job = self.pager.core.job(); + let State { pages, entries, held, gone, .. } = &mut *self.state; + held.settle(); + let phys = match held.core.draw(!held.spare.is_empty()) { + Draw::Aside => { + let phys = held.spare.pop().expect("paged: a frame drawn aside is held aside"); + // SAFETY: a frame held aside, out of the index and unpinned, + // which nothing else reaches. + unsafe { ptr::write_bytes(DirectMap::from_phys(phys).as_mut_ptr::(), 0, PAGE as usize) }; + phys + } + Draw::Pool => pmm::alloc_promised().into_raw(), + Draw::Beyond if pmm::promise(job, 1) => { + held.core.promised(); + pmm::alloc_promised().into_raw() + } + Draw::Beyond => return Err(no_room), + }; + let object = &mut entries.get_mut(self.slot).expect("paged: a mapped slot is held").object; + match pages.supply(object, page, phys, false, (page + 1) * PAGE) { + Ok(Supplied::Installed) => { + held.core.installed(); + Ok(phys) + } + Ok(Supplied::Present) => unreachable!("paged: a hole found under the lock held a page"), + // Inside the object and the pager alive, as the lookup found it: + // only a full table refuses. + Err(_) => { + held.give_back(job, *gone, phys, false); + Err(no_room) + } + } + } + + fn lost(&self, page: u64, why: Why) -> Missing { + Missing::Lost(Lost { pager: self.pager.pid, object: self.slot, page, why }) + } } /// The pages a fault's request names: the aligned run around its page. @@ -1285,9 +1417,9 @@ const FAULT_RUN: u64 = 16; impl MapNode { /// `f` over the object's pages, under the pager's lock. - pub fn with_present(&self, f: impl FnOnce(&Index<'_>) -> R) -> R { - let state = self.pager.state.lock(); - f(&Index { state: &state, slot: self.slot }) + pub fn with_index(&self, f: impl FnOnce(&mut Index<'_>) -> R) -> R { + let mut state = self.pager.state.lock(); + f(&mut Index { pager: &self.pager, state: &mut state, slot: self.slot }) } /// A fault's wait for the object's `page`, found absent at `since`, its @@ -1349,7 +1481,11 @@ impl Clone for PagedRange { impl Drop for PagedRange { fn drop(&mut self) { - if self.0.ranges.fetch_sub(1, Ordering::Relaxed) == 1 { + // The last may be a parked fault's, holding no space lock: as `Arc`'s + // count, each drop releases what its holder cleared, and the last + // acquires every one before the node leaves the list. + if self.0.ranges.fetch_sub(1, Ordering::Release) == 1 { + core::sync::atomic::fence(Ordering::Acquire); self.0.pager.unlink(&self.0); } } @@ -1388,12 +1524,16 @@ impl PagerObject { } } entry.mappers.tail = at; + // A mapping holds the object as a handle does: a file mapped and + // closed, or unlinked, stays until the last mapping goes. entry.refs += 1; + entry.holders += 1; Ok(PagedRange(node)) } /// The last range of `node`'s space went, its leaves cleared and shot - /// down: the node leaves the list, and its reference on the slot goes. + /// down: the node leaves the list, and its hold and reference on the + /// slot go. fn unlink(&self, node: &MapNode) { let mut state = self.state.lock(); let entry = state.entry_mut(node.slot); @@ -1409,6 +1549,7 @@ impl PagerObject { false => (*(*next).link.get()).prev = prev, } } + self.let_hold_go(&mut state, node.slot); self.unref_locked(&mut state, node.slot); } } diff --git a/kernel/src/pipe.rs b/kernel/src/pipe.rs index 9581228fa6..a24d0b49c8 100644 --- a/kernel/src/pipe.rs +++ b/kernel/src/pipe.rs @@ -17,7 +17,7 @@ use alloc::sync::Arc; use crate::scheduler::Parkable; use crate::sync::Lock; -use crate::user_ptr::{UserBytes, UserBytesMut}; +use crate::user_ptr::{Faulted, UserBytes, UserBytesMut}; use crate::watch::{Cancelled, Watch}; /// A reference to a pipe that holds neither end: what an end's object, a @@ -173,16 +173,16 @@ pub enum PipeWrite { /// The memory its bytes need could not be taken; no wait for room makes /// it appear, so a caller must not park on it. NoMemory, - /// A byte of the writer's buffer it cannot reach: nothing was written. - Faulted, + /// A byte of the writer's buffer it cannot have: nothing was written. + Faulted(Faulted), } /// Why a read answered no count. pub enum ReadStopped { /// A kill ended the wait for another read's claim. Cancelled, - /// A byte of the reader's buffer it cannot reach: the bytes it took are lost. - Faulted, + /// A byte of the reader's buffer it cannot have: the bytes it took are lost. + Faulted(Faulted), } impl From for ReadStopped { @@ -219,7 +219,7 @@ impl PipeRef { if boost { crate::scheduler::boost_current_rt_inherited(); } - buf.write_at(0, bounce.bytes()).map_err(|_| ReadStopped::Faulted)?; + buf.write_at(0, bounce.bytes()).map_err(ReadStopped::Faulted)?; return Ok(Some(n)); } pipe.held.claim_read(n); @@ -238,7 +238,7 @@ impl PipeRef { frame_bytes::give_back(&self.0.state, |s| &mut s.held, spares); drop(unread); frame_bytes::wake(&self.0.read_watch, ended); - copied.map_err(|_| ReadStopped::Faulted)?; + copied.map_err(ReadStopped::Faulted)?; Ok(Some(n)) } @@ -249,7 +249,7 @@ impl PipeRef { let mut bounce = PipeBounce::new(); return match bounce.read(buf) { Ok(src) => self.write_bounced(p, src), - Err(_) => Ok(PipeWrite::Faulted), + Err(faulted) => Ok(PipeWrite::Faulted(faulted)), }; } let (n, first) = { @@ -273,7 +273,7 @@ impl PipeRef { let wrote = match supplied { Ok(runs) => match frame_bytes::fill(&self.0.state, |s| &mut s.held, buf, n, runs) { Ok(()) => PipeWrite::Wrote(n), - Err(_) => PipeWrite::Faulted, + Err(faulted) => PipeWrite::Faulted(faulted), }, Err(Exhausted) => PipeWrite::NoMemory, }; diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 47bfcb5bb3..1c54c791c9 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -924,9 +924,14 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { }; let wait = crate::user_ptr::begin_futex_wait(); - // Physical, so a futex in shared memory works across processes. - let Some(word) = futex_word(addr).and_then(|addr| wait.translate(addr)) else { - return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); + // Physical, so a futex in shared memory works across processes. A word + // of a mapped file's page that is absent or lost is a spurious wake, as + // swapped memory's is. + let unaligned = crate::user_ptr::Untranslated::Unreachable; + let word = match futex_word(addr).ok_or(unaligned).and_then(|addr| wait.translate(addr)) { + Ok(word) => word, + Err(crate::user_ptr::Untranslated::Away) => return 0, + Err(crate::user_ptr::Untranslated::Unreachable) => return toyos_abi::syscall::SyscallError::BadAddress.to_u64(), }; // 0 covers both woken and never-blocked (the caller re-checks the word either way); 1 means only the caller's own deadline ended it. @@ -938,8 +943,12 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { /// Wake up to `count` threads blocked on the same physical address as `addr`. pub fn futex_wake(addr: UserAddr, count: u64) -> u64 { - let Some(phys) = futex_word(addr).and_then(|addr| crate::user_ptr::translate_user(addr, crate::user_ptr::Access::Read)) else { - return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); + // No wait is keyed on a page that is not there. + let unaligned = crate::user_ptr::Untranslated::Unreachable; + let phys = match futex_word(addr).ok_or(unaligned).and_then(|addr| crate::user_ptr::translate_user(addr, crate::user_ptr::Access::Read)) { + Ok(phys) => phys, + Err(crate::user_ptr::Untranslated::Away) => return 0, + Err(crate::user_ptr::Untranslated::Unreachable) => return toyos_abi::syscall::SyscallError::BadAddress.to_u64(), }; scheduler::futex_wake(phys, count as usize) } @@ -964,43 +973,104 @@ pub fn ask_join(join: &join::Join, tid: Tid, parent_pid: Pid) -> Option crate::user_ptr::Stop { + match self { + Faulted::Lost(_) => crate::user_ptr::Stop::Lost, + Faulted::Absent => crate::user_ptr::Stop::Absent, + Faulted::Served | Faulted::Unreachable | Faulted::Killed => crate::user_ptr::Stop::Unreachable, + } + } +} + /// Serve a fault at `fault_addr` for `access` in the running thread's space, /// at its trap's depth with interrupts on: its page populated under the /// space's lock, or, for a paged page that is absent, waited for holding no -/// lock and touched again. While it waits the CPU's fault mark is down, so -/// another fault on this CPU meanwhile is no recursion; it is up again on -/// whichever CPU the thread wakes on. +/// lock and touched again, unless the fault is a copy's that holds page pins. +/// While it waits the CPU's fault mark is down, so another fault on this CPU +/// meanwhile is no recursion; it is up again on whichever CPU the thread +/// wakes on. pub fn handle_page_fault(fault_addr: u64, access: kernel::vm::Access) -> Faulted { use crate::arch::percpu::{self, CpuFaultState}; - let Some(space) = scheduler::current_address_space() else { return Faulted::Refused }; + touch(fault_addr, access, |page| { + if crate::user_ptr::pinned() { + return Faulted::Absent; + } + percpu::set_fault_state(CpuFaultState::Normal); + let woken = page.wait(); + let prev = percpu::swap_fault_state(CpuFaultState::PageFault); + assert_eq!(prev, CpuFaultState::Normal, "a fault woke on a CPU marked {prev:?}"); + if woken { Faulted::Served } else { Faulted::Killed } + }) +} + +/// The page at `va` populated for `access`, a paged page that is absent left +/// to `wait`, which answers `Served` to touch it again. +fn touch(va: u64, access: kernel::vm::Access, mut wait: impl FnMut(crate::mm::space::PageWait) -> Faulted) -> Faulted { + use crate::mm::space::Populated; + let Some(space) = scheduler::current_address_space() else { return Faulted::Unreachable }; let began = crate::clock::nanos_since_boot(); loop { - let populated = space.lock().fault(fault_addr, access); + let populated = space.lock().fault(va, access); match populated { - crate::mm::space::Populated::At(_) => break, - crate::mm::space::Populated::Refused => return Faulted::Refused, - crate::mm::space::Populated::Wait(page) => { - percpu::set_fault_state(CpuFaultState::Normal); - let woken = page.wait(); - let prev = percpu::swap_fault_state(CpuFaultState::PageFault); - assert_eq!(prev, CpuFaultState::Normal, "a fault woke on a CPU marked {prev:?}"); - if !woken { - return Faulted::Killed; - } - } + Populated::At(_) => break, + Populated::Unreachable => return Faulted::Unreachable, + Populated::Lost(lost) => return Faulted::Lost(lost), + Populated::Wait(page) => match wait(page) { + Faulted::Served => {} + stopped => return stopped, + }, } } space.usage().faulted(crate::clock::nanos_since_boot().saturating_sub(began)); Faulted::Served } +/// Populate the running thread's pages over `[start, end)` for `access` from +/// a syscall holding no lock and no pin, waiting for a mapped file's absent +/// page as a fault does. +pub fn wait_in(start: u64, end: u64, access: kernel::vm::Access) -> Result<(), crate::user_ptr::Faulted> { + let mut page = start & !(crate::mm::PAGE_SIZE - 1); + while page < end { + let touched = touch(page, access, |page| if page.wait() { Faulted::Served } else { Faulted::Killed }); + match touched { + Faulted::Served => {} + Faulted::Lost(_) => return Err(crate::user_ptr::Faulted::Lost), + Faulted::Unreachable | Faulted::Absent | Faulted::Killed => return Err(crate::user_ptr::Faulted::Unreachable), + } + page += crate::mm::PAGE_SIZE; + } + Ok(()) +} + +/// A Ring 3 touch at `fault_addr` of a mapped file's page its pager cannot +/// let the process have ends the process as `SIGBUS`, with one record of what +/// was lost. The caller has put the CPU's fault mark down. +pub fn lost(fault_addr: u64, lost: crate::object::paged::Lost) -> ! { + log!( + "fault: pid {} touched {fault_addr:#x}, page {} of pid {}'s object {}: {}", + current_process(), + lost.page, + lost.pager, + lost.object, + lost.why.name() + ); + exit(End::Fault(Fault::Mapping)) +} + /// Dump memory around `fault_addr` and the instruction pointer for the /// current process; called from the exception handler on user-mode crashes. pub fn dump_crash_diagnostics(fault_addr: u64, rip: u64) { diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 7cab127cb8..805f86a1d4 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -753,6 +753,11 @@ pub fn pass_on_every_other_cpu() { } } +/// `f` over the running thread's user copies; `None` off-task. +pub fn with_current_copies(f: impl FnOnce(&crate::user_ptr::Copies) -> R) -> Option { + try_with_cpu(|cpu| cpu.running().map(|t| f(t.ext().handle.copies()))).flatten() +} + /// Count the running thread's syscall `num`, which took `ticks` of the counter. pub fn note_syscall(num: u64, ticks: u64) { try_with_cpu(|cpu| cpu.running().map(|t| t.ext().handle.syscalls().note(num, ticks))); diff --git a/kernel/src/sched/payload.rs b/kernel/src/sched/payload.rs index 99c531a987..3bc6fba146 100644 --- a/kernel/src/sched/payload.rs +++ b/kernel/src/sched/payload.rs @@ -115,6 +115,7 @@ pub struct TaskHandle { /// The releases this thread owns while its process is torn down (`object::Releases`); kept here, as `operation` is, so they follow the thread across a migration. releases: Lock>>, syscalls: SyscallProfile, + copies: crate::user_ptr::Copies, } /// What a thread's syscalls cost it. Written only by the thread, at the end of @@ -168,6 +169,7 @@ impl TaskHandle { cancels: AtomicU32::new(0), releases: Lock::new(None), syscalls: SyscallProfile::new(), + copies: crate::user_ptr::Copies::new(), } } @@ -175,6 +177,11 @@ impl TaskHandle { &self.syscalls } + /// Its user copies, as their faults leave them. + pub fn copies(&self) -> &crate::user_ptr::Copies { + &self.copies + } + pub(crate) fn publish(&self, acct: &TaskAccounting, running_since: Option) { self.cpu_ns.store(acct.cpu_ns, Ordering::Relaxed); self.running_since diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 5fe35d4d08..ff470f951c 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -1,8 +1,9 @@ //! User memory, copied in and out through the running process's own mapping //! (`arch::uaccess`): a copy reaches only what a load or store the process //! made itself could reach, and a byte it cannot reach ends the copy as -//! [`Faulted`], which a syscall answers as `BadAddress`. Values are copied -//! out, never referenced. +//! [`Faulted`], which a syscall answers as `BadAddress`, or as `Io` where it +//! is a mapped file's page its server failed or lost. Values are copied out, +//! never referenced. //! //! **A window is a range of the user half**, checked for its end, overflow //! included, when it is made, and by nothing else: whether its pages are @@ -13,10 +14,12 @@ //! only the accesses the process could make through that mapping itself //! (`arch::uaccess`). //! -//! **A copy holds no kernel lock**: its fault takes its space's lock, and in -//! 7b will wait on a pager. Every spin lock raises the preemption count, so -//! each copy refuses to run above the depth its syscall's entry runs at -//! ([`gate`]), whether or not a page faults. +//! **A copy holds no kernel lock**: its fault takes its space's lock and may +//! wait on a pager. Every spin lock raises the preemption count, so each copy +//! refuses to run above the depth its syscall's entry runs at ([`gate`]), +//! whether or not a page faults. **Nor does it wait holding a page pin**: a +//! copy made holding one goes through [`Pinned`], which stops where a page +//! would be waited for. //! //! Every single-word user read is `read_volatile`, including the futex word //! and the crash dump's walk, both outside this module. @@ -24,6 +27,7 @@ use alloc::vec; use alloc::vec::Vec; use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use kernel::vm::Access as Asked; use toyos_abi::syscall::SyscallError; @@ -41,9 +45,24 @@ fn asked(access: Access) -> Asked { } } -/// The frame behind a user virtual address, populating its page if needed. -pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { - unlocked_space().lock().reach(addr.raw(), asked(access)) +/// Why a user address has no frame. +pub(crate) enum Untranslated { + /// Nothing the access may reach is there. + Unreachable, + /// A mapped file's page that is absent or lost. + Away, +} + +/// The frame behind a user virtual address, populating its page if needed, +/// but waiting for no mapped file's page. +pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Result { + use crate::mm::space::Populated; + let reached = unlocked_space().lock().reach(addr.raw(), asked(access)); + match reached { + Populated::At(phys) => Ok(phys), + Populated::Unreachable => Err(Untranslated::Unreachable), + Populated::Lost(_) | Populated::Wait(_) => Err(Untranslated::Away), + } } /// This thread's futex wait in its space. @@ -67,13 +86,129 @@ fn gate() { assert!(crate::arch::cpu::interrupts_enabled(), "user_ptr: a user copy with interrupts masked"); } -/// A copy met a user byte its process could not reach. -#[derive(Debug)] -pub struct Faulted; +/// A copy met a user byte its process could not have. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Faulted { + /// Nothing the access may reach is there. + Unreachable, + /// A mapped file's page its server failed, or can no longer supply. + Lost, +} impl From for SyscallError { - fn from(_: Faulted) -> Self { - SyscallError::BadAddress + fn from(faulted: Faulted) -> Self { + match faulted { + Faulted::Unreachable => SyscallError::BadAddress, + Faulted::Lost => SyscallError::Io, + } + } +} + +/// What stopped a copy made through [`Pinned`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Stopped { + Faulted(Faulted), + /// A mapped file's page that is absent: the copy did not wait for it. + Absent, +} + +/// What stopped a thread's last copy short, as its fault left it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Stop { + Unreachable, + Lost, + /// Only under [`Pinned`]. + Absent, +} + +/// A thread's user copies: whether it holds page pins across them, and what +/// stopped its last one short. Written only by the thread itself, by its +/// copies and their faults. +pub struct Copies { + pinned: AtomicBool, + stop: AtomicU8, +} + +impl Copies { + pub const fn new() -> Self { + Self { pinned: AtomicBool::new(false), stop: AtomicU8::new(Stop::Unreachable as u8) } + } +} + +/// Whether the running thread's copies hold page pins ([`Pinned`]), so its +/// copy's fault must not wait. +pub fn pinned() -> bool { + crate::sched::driver::with_current_copies(|c| c.pinned.load(Ordering::Relaxed)).unwrap_or(false) +} + +/// A copy's fault could not serve its page, for `why`: the copy resumes at +/// its fixup, and [`last_stop`] reads this. +pub fn stopped(why: Stop) { + crate::sched::driver::with_current_copies(|c| c.stop.store(why as u8, Ordering::Relaxed)); +} + +/// What stopped the running thread's last copy short, as its fault left it: +/// what a copy routine answers where it left bytes behind (`arch::uaccess`). +pub fn last_stop() -> Stopped { + match crate::sched::driver::with_current_copies(|c| c.stop.load(Ordering::Relaxed)) { + Some(x) if x == Stop::Lost as u8 => Stopped::Faulted(Faulted::Lost), + Some(x) if x == Stop::Absent as u8 => Stopped::Absent, + _ => Stopped::Faulted(Faulted::Unreachable), + } +} + +/// A copy holding no pin: what stopped it is a fault. +fn faulted(copied: Result<(), Stopped>) -> Result<(), Faulted> { + copied.map_err(|why| match why { + Stopped::Faulted(faulted) => faulted, + Stopped::Absent => panic!("user_ptr: a copy holding no pin stopped short of an absent page"), + }) +} + +/// The running thread's copies while it holds page pins, which no wait may +/// hold: a fault on a mapped file's absent page stops the copy +/// ([`Stopped::Absent`]) instead of waiting. The holder lets its pins go, +/// waits for the window's pages holding nothing (`wait_in`), and copies +/// again. Not `Send`: it marks the thread that made it. +pub struct Pinned(PhantomData<*const ()>); + +impl Pinned { + pub fn begin() -> Self { + crate::sched::driver::with_current_copies(|c| c.pinned.store(true, Ordering::Relaxed)); + Self(PhantomData) + } + + /// [`UserBytesMut::write_raw`], stopping at an absent page. + /// + /// # Safety + /// As [`UserBytesMut::write_raw`]. + pub unsafe fn write(&self, out: &mut UserBytesMut, off: usize, src: *const u8, len: usize) -> Result<(), Stopped> { + let user = out.0.at("UserBytesMut::write", off, len); + // SAFETY: as `View::write_raw`. + unsafe { uaccess::copy_to_user(user, src, len) } + } + + /// [`UserBytesMut::fill_zero`], stopping at an absent page. + pub fn zero(&self, out: &mut UserBytesMut, off: usize, len: usize) -> Result<(), Stopped> { + let user = out.0.at("UserBytesMut::fill_zero", off, len); + // SAFETY: as `View::fill_zero`. + unsafe { uaccess::clear_user(user, len) } + } + + /// [`UserBytes::read_raw`], stopping at an absent page. + /// + /// # Safety + /// As [`UserBytes::read_raw`]. + pub unsafe fn read(&self, from: &UserBytes, off: usize, dst: *mut u8, len: usize) -> Result<(), Stopped> { + let user = from.0.at("UserBytes::read", off, len); + // SAFETY: as `View::read_raw`. + unsafe { uaccess::copy_from_user(dst, user, len) } + } +} + +impl Drop for Pinned { + fn drop(&mut self) { + crate::sched::driver::with_current_copies(|c| c.pinned.store(false, Ordering::Relaxed)); } } @@ -176,7 +311,7 @@ impl View { let user = self.at("UserBytes::read", off, len); // SAFETY: the caller's `dst`; `at` keeps the user range in the half, // and `gate` holds no lock across the copy. - unsafe { uaccess::copy_from_user(dst, user, len) } + faulted(unsafe { uaccess::copy_from_user(dst, user, len) }) } /// # Safety @@ -184,13 +319,13 @@ impl View { unsafe fn write_raw(&self, off: usize, src: *const u8, len: usize) -> Result<(), Faulted> { let user = self.at("UserBytesMut::write", off, len); // SAFETY: as `read_raw`. - unsafe { uaccess::copy_to_user(user, src, len) } + faulted(unsafe { uaccess::copy_to_user(user, src, len) }) } fn fill_zero(&self, off: usize, len: usize) -> Result<(), Faulted> { let user = self.at("UserBytesMut::fill_zero", off, len); // SAFETY: as `read_raw`, with no kernel memory touched. - unsafe { uaccess::clear_user(user, len) } + faulted(unsafe { uaccess::clear_user(user, len) }) } } @@ -202,6 +337,14 @@ impl UserBytes<'_> { self.0.len } + /// Populate the window's pages under `[off, off + len)` for reading, + /// waiting for a mapped file's absent page holding nothing: what a copy + /// [`Pinned`] stopped at needs before it copies again. + pub fn wait_in(&self, off: usize, len: usize) -> Result<(), Faulted> { + let start = self.0.at("UserBytes::wait_in", off, len); + crate::process::wait_in(start, start + len as u64, Asked::Read) + } + /// Copy `dst.len()` bytes out of the window at `off`; panics if out of bounds — the kernel's own arithmetic, never user input. pub fn read_at(&self, off: usize, dst: &mut [u8]) -> Result<(), Faulted> { self.0.read_into(off, dst) @@ -249,4 +392,10 @@ impl UserBytesMut<'_> { pub fn fill_zero(&mut self, off: usize, len: usize) -> Result<(), Faulted> { self.0.fill_zero(off, len) } + + /// As [`UserBytes::wait_in`], for writing. + pub fn wait_in(&mut self, off: usize, len: usize) -> Result<(), Faulted> { + let start = self.0.at("UserBytesMut::wait_in", off, len); + crate::process::wait_in(start, start + len as u64, Asked::Write) + } } diff --git a/sdk/abi/src/end.rs b/sdk/abi/src/end.rs index fa025dba65..660207c67c 100644 --- a/sdk/abi/src/end.rs +++ b/sdk/abi/src/end.rs @@ -32,6 +32,9 @@ pub enum Fault { /// A syscall named a handle the process does not hold, which ends it /// (`kernel::object::HandleError`'s rule). Handle, + /// A touch of a mapped file's page its server failed or can no longer + /// supply (`SIGBUS`). + Mapping, } const EXITED: u64 = 0; @@ -47,6 +50,7 @@ const fn fault_kind(fault: Fault) -> u64 { Fault::Arithmetic => 5, Fault::Trap => 6, Fault::Handle => 7, + Fault::Mapping => 8, } } @@ -74,6 +78,7 @@ impl End { (5, 0) => Fault::Arithmetic, (6, 0) => Fault::Trap, (7, 0) => Fault::Handle, + (8, 0) => Fault::Mapping, _ => return None, }; (fault_kind(fault) == kind).then_some(End::Fault(fault)) @@ -102,6 +107,7 @@ impl Fault { Fault::Arithmetic => "arithmetic", Fault::Trap => "trap", Fault::Handle => "handle", + Fault::Mapping => "mapping", } } } @@ -123,8 +129,8 @@ mod tests { use super::*; use crate::syscall::SyscallError; - const FAULTS: [Fault; 6] = - [Fault::Memory, Fault::Alignment, Fault::Instruction, Fault::Arithmetic, Fault::Trap, Fault::Handle]; + const FAULTS: [Fault; 7] = + [Fault::Memory, Fault::Alignment, Fault::Instruction, Fault::Arithmetic, Fault::Trap, Fault::Handle, Fault::Mapping]; fn every_end() -> impl Iterator { let exits = [0, 1, 137, 139, -1, i32::MIN, i32::MAX].map(End::Exited); @@ -158,7 +164,7 @@ mod tests { /// or fault carrying a code. #[test] fn a_word_no_end_makes_names_none() { - let past = 8 << 32; + let past = 9 << 32; for word in [past, (KILLED << 32) | 1, End::Fault(Fault::Trap).to_word() | 7] { assert_eq!(End::from_word(word), None, "{word:#x}"); } diff --git a/sdk/libc/arch/src/spawn_requests.rs b/sdk/libc/arch/src/spawn_requests.rs index d459820bab..e39c4e613f 100644 --- a/sdk/libc/arch/src/spawn_requests.rs +++ b/sdk/libc/arch/src/spawn_requests.rs @@ -175,6 +175,7 @@ fn each_end_reads_as_its_wait_status() { (End::Fault(Fault::Memory), "SIGSEGV"), (End::Fault(Fault::Handle), "SIGSEGV"), (End::Fault(Fault::Alignment), "SIGBUS"), + (End::Fault(Fault::Mapping), "SIGBUS"), (End::Fault(Fault::Instruction), "SIGILL"), (End::Fault(Fault::Arithmetic), "SIGFPE"), (End::Fault(Fault::Trap), "SIGTRAP"), diff --git a/sdk/toyos/src/process.rs b/sdk/toyos/src/process.rs index 11f20821f4..875bf9dc6f 100644 --- a/sdk/toyos/src/process.rs +++ b/sdk/toyos/src/process.rs @@ -28,7 +28,7 @@ pub fn signal(end: End) -> Option { End::Exited(_) => return None, End::Killed => SIGKILL, End::Fault(Fault::Memory | Fault::Handle) => SIGSEGV, - End::Fault(Fault::Alignment) => SIGBUS, + End::Fault(Fault::Alignment | Fault::Mapping) => SIGBUS, End::Fault(Fault::Instruction) => SIGILL, End::Fault(Fault::Arithmetic) => SIGFPE, End::Fault(Fault::Trap) => SIGTRAP, @@ -250,6 +250,7 @@ mod tests { (End::Fault(Fault::Memory), "SIGSEGV"), (End::Fault(Fault::Handle), "SIGSEGV"), (End::Fault(Fault::Alignment), "SIGBUS"), + (End::Fault(Fault::Mapping), "SIGBUS"), (End::Fault(Fault::Instruction), "SIGILL"), (End::Fault(Fault::Arithmetic), "SIGFPE"), (End::Fault(Fault::Trap), "SIGTRAP"), From e9a13d543f24ecfc3d0dcd6b8e0bffa86eb6d700 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:44:24 +0200 Subject: [PATCH 11/34] The manual says how a file mapping shares the cache, waits and ends The memory page: a mapping maps the cache's own pages and holds its file as an open handle does; a page leaves the cache only once no mapping reaches it; a wait for a file's page behaves like swapped memory's (D6); a touch of a lost page ends the program with a diagnostic and a copy answers an I/O error; a touched hole of a file held in RAM is resident within its server's capacity. The files and services pages: a file server's end revokes every mapping of its files, in the words of the ruling (D1). Co-Authored-By: Claude Opus 5.5 --- docs/src/architecture/files.md | 6 ++++++ docs/src/architecture/memory.md | 14 ++++++++++++++ docs/src/architecture/services.md | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/docs/src/architecture/files.md b/docs/src/architecture/files.md index beae25f3c2..80b875bb59 100644 --- a/docs/src/architecture/files.md +++ b/docs/src/architecture/files.md @@ -77,5 +77,11 @@ directory, which the child resolves in its own view. that server's open files, which then answer that they are gone; a fresh open reaches the restarted server. Unsaved changes may be lost; data already confirmed durable stays durable. +- A file server's end revokes every mapping of its files, so a program + executing from, or reading a mapping of, that server's files ends at its + next access to that mapping, a system service mapping a data file included. + A file server's death is the loss of that file system: its parent restarts + it, and programs that depended on it fail clearly rather than run on with + stale memory. - Per-session temporary storage is an ordinary file server holding its contents in RAM. diff --git a/docs/src/architecture/memory.md b/docs/src/architecture/memory.md index 32577bc855..a466152f19 100644 --- a/docs/src/architecture/memory.md +++ b/docs/src/architecture/memory.md @@ -53,6 +53,20 @@ structures. - Mapped files may be larger than RAM, partially mapped, and coherently shared and written; their length is never committed in RAM. +- A file mapping maps the cache's own pages, so a mapper sees a write as it + lands, and a private mapping shares them until it is written. A mapping + holds its file as an open handle does: a file mapped and then closed or + deleted stays until its last mapping goes. A page leaves the cache only once + no mapping reaches it. +- A wait for a file's page behaves like one for swapped memory: it advances + with its file server's progress, is reported when overdue and never becomes + an error by timeout. The server failing what it cannot read, and its device + waits' own timeouts, bound it. +- A touch of a mapped page its file server failed, or of a file whose server + ended, ends the program with a diagnostic naming the file's page and the + cause; a copying system call answers an I/O error instead. A touched + unwritten page of a file held in RAM becomes resident within its server's + capacity and is refused once that is full. - Clean pages are reclaimed; dirty pages are written back first, without losing concurrent writes. Dirty data growth is bounded, and writeback starts ahead of pressure. diff --git a/docs/src/architecture/services.md b/docs/src/architecture/services.md index 803cc57f08..de9ea83706 100644 --- a/docs/src/architecture/services.md +++ b/docs/src/architecture/services.md @@ -67,6 +67,10 @@ Every server admits its clients the same way, through one shared mechanism: - **Recovery restores durable state, not sessions.** File servers recover what was on disk, not old opens; network sessions may fail; the memory service keeps recoverable swapped contents. +- **A file server's end is the loss of its file system.** Every mapping of its + files is revoked, so a program executing from or reading a mapping of them, + a service included, ends at its next access to that mapping rather than run + on with stale memory. - **Nothing is silently replayed.** An operation with an uncertain outcome is reported, and retried only where safe; a write whose acknowledgement was lost is never replayed behind the client's back. From a8596b1ec8b4ac6a962682942dd6680bf3f28bbf Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:57:41 +0200 Subject: [PATCH 12/34] A guest probe maps a pager's objects from a child, on both architectures `mapprobe`, a pager row (a pager may not map), starts itself again as a mapper per case, endowed a holder and pipe ends, and judges each by its end and by its notices: - coherent: the pager's write lands in the page the mapper maps, and the mapping holds the object after the mapper closed its handle (no last-holder notice until the mapper ends); - parks: a fault waits for its run, a withdrawn run is asked for again and comes back with the same bytes, and a supply held past the report time is waited out; - unmapped: a range taken away under a parked thread is nothing when the page comes (`Fault::Memory`); - fault, copy: a kill ends a thread parked in its fault and one parked in a pipe write's copy from the mapping, which writes nothing; - hole: an anonymous object's unwritten page reads zero with no request and costs the pager one page; with no room left, the touch ends the mapper (`Fault::Mapping`); - shrunk: a page the object shrank past is nothing (`Fault::Memory`); - ends, last: the pager's end answers a write from the mapping `Io` and ends the mapper at its next touch (`Fault::Mapping`). `mapped_file` launches it. It is a shared-boot member on the benches, the x86-64 QEMU row `a_mapped_file_follows_its_pager` (which also reads the kernel's overdue report and its end off the serial line), and `virt_mapped_file` on AArch64's job case, where a fault parks with the CPU's fault mark down. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 8 + Cargo.toml | 1 + tests/guest/mapprobe/Cargo.toml | 10 + tests/guest/mapprobe/src/main.rs | 337 ++++++++++++++++++ tests/guest/rust-tests/src/bin/mapped_file.rs | 20 ++ tests/images/testcases.toml | 9 +- tests/images/virtjobcase.toml | 8 +- tests/toyos.rs | 55 +++ 8 files changed, 445 insertions(+), 3 deletions(-) create mode 100644 tests/guest/mapprobe/Cargo.toml create mode 100644 tests/guest/mapprobe/src/main.rs create mode 100644 tests/guest/rust-tests/src/bin/mapped_file.rs diff --git a/Cargo.lock b/Cargo.lock index 7845d6cd69..32e258fed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1714,6 +1714,14 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "mapprobe" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", +] + [[package]] name = "matchers" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 5ae48789a2..04ca85a94a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ members = [ "tests/guest/osbench/compare", "tests/guest/osbench/lan", "tests/guest/osbench/rules", + "tests/guest/mapprobe", "tests/guest/pagerprobe", "tests/guest/pciholder", "tests/guest/starveling", diff --git a/tests/guest/mapprobe/Cargo.toml b/tests/guest/mapprobe/Cargo.toml new file mode 100644 index 0000000000..708c5c3129 --- /dev/null +++ b/tests/guest/mapprobe/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mapprobe" +description = "Test-only: a pager whose objects a child of its own maps, each fault answered, withheld or refused as a case needs." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dependencies] +toyos-abi = { path = "../../../sdk/abi" } +toyos = { path = "../../../sdk/toyos" } diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs new file mode 100644 index 0000000000..87c0fddcbb --- /dev/null +++ b/tests/guest/mapprobe/src/main.rs @@ -0,0 +1,337 @@ +//! A pager whose objects a child of its own maps, started by its row's +//! `pager`: what a mapping of a paged object does that only a booted kernel +//! has, its tables, its faults and a second process. A pager may not map, +//! so each case's mapper is this program again, `--map `, endowed a +//! holder and pipe ends, and judged by its end and by what the pager hears. +//! +//! - `coherent`: a mapper sees the pager's write land in the page it maps, +//! and its mapping holds the object after it closed its handle. +//! - `parks`: a touch of an absent page waits until it is supplied; a +//! withdrawn page is asked for again and comes back with the same bytes; +//! a supply held past the report time is reported overdue, and the mapper +//! then goes on. +//! - `unmapped`: a range taken away under a thread parked in it ends the +//! process as a touch of nothing once the page comes. +//! - `fault`, `copy`: a kill ends a thread parked in its own fault, and one +//! parked in a write's copy from the mapping, which writes nothing. +//! - `hole`: an unwritten page of an anonymous object reads zero with no +//! request, and one the pager has no room for ends the mapper. +//! - `shrunk`: a page the object shrank past ends its mapper as nothing. +//! - `ends`, last: the pager's end answers a write from the mapping `Io` +//! and ends the mapper at its next touch. + +use std::os::toyos::process::{CommandExt, ExitStatusExt}; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use toyos::endow::Endowments; +use toyos::paged::{Notice, Paged, PagedNotice, Pager, PAGE}; +use toyos::poller::{Poller, READABLE}; +use toyos::{OwnedHandle, Pipe}; +use toyos_abi::end::{End, Fault}; +use toyos_abi::syscall::{self, SyscallError, VmAt}; + +const SELF: &str = "/system/bin/mapprobe"; +const HOLDER: &str = "mapprobe-holder"; +const READY: &str = "mapprobe-ready"; +const GO: &str = "mapprobe-go"; +const OUT: &str = "mapprobe-out"; + +/// Every page the pager holds at once. +const CAPACITY: u64 = 256; +/// The pager's report time, which `parks` holds a supply past. +const REPORT: Duration = Duration::from_secs(1); +/// How long a step may take before the probe gives up on it: a failure +/// bound, never what ends a passing run. +const BOUND: Duration = Duration::from_secs(60); + +/// The aligned run a fault's request names, in pages. +const RUN: u64 = 16; + +fn main() { + let mut args = std::env::args().skip(1); + if args.next().as_deref() == Some("--map") { + return map(&args.next().expect("mapprobe --map: a case")); + } + let pager = Pager::create(CAPACITY, 0, REPORT).expect("mapprobe: its row starts it as a pager"); + let mut served = Served { pager, poller: Poller::new(1), unheld: Vec::new() }; + coherent(&mut served); + parks(&mut served); + killed(&mut served); + hole(&mut served); + shrunk(&mut served); + ends(served); + println!("mapprobe: ok"); +} + +/// The pager, and the objects whose last holder its notices said went. +struct Served { + pager: Pager, + poller: Poller, + unheld: Vec, +} + +impl Served { + /// Wait for a request for `object` naming page `first`. A request for + /// another run is a duplicate of one answered: the kernel names a run + /// again after a withdraw meets its waiter. + fn requested(&mut self, object: u64, first: u64) { + let began = Instant::now(); + while !self.heard(|n| matches!(n, Notice::Requested { object: o, first: f, .. } if o == object && f == first)) { + self.wait(began, &format!("a request for object {object} page {first}")); + } + } + + /// Wait for the notice that `object`'s last holder went. + fn unheld(&mut self, object: u64) { + let began = Instant::now(); + while !self.unheld.contains(&object) { + self.heard(|_| false); + if !self.unheld.contains(&object) { + self.wait(began, &format!("object {object}'s last holder to go")); + } + } + } + + /// Whether a pending notice is one `want` names, the rest read and kept. + fn heard(&mut self, want: impl Fn(Notice) -> bool) -> bool { + let mut buf = [PagedNotice::default(); 8]; + let mut found = false; + for notice in self.pager.notices(&mut buf).expect("mapprobe: its notices") { + if let Notice::NoHolders { object } = notice { + self.unheld.push(object); + } + found |= want(notice); + } + found + } + + fn wait(&self, began: Instant, what: &str) { + let left = BOUND.checked_sub(began.elapsed()).unwrap_or_else(|| panic!("mapprobe: no {what} within {BOUND:?}")); + self.poller.watch(&self.pager, READABLE, 0); + self.poller.wait(1, left.as_nanos() as u64, |_| {}); + } + + fn supply(&self, object: u64, first: u64, count: u64, byte: u8) { + let bytes = vec![byte; (count * PAGE) as usize]; + // A page already present was a duplicate's: the kernel installs nothing. + self.pager.supply(object, first * PAGE, &bytes, false).expect("mapprobe: a supply"); + } + + /// A mapper of `object` for `case`, endowed a holder and `ends`. + fn mapper(&self, object: u64, case: &str, ends: Vec<(&str, OwnedHandle)>) -> Child { + let holder = self.pager.holder(object).expect("mapprobe: a holder to hand on"); + let mut command = Command::new(SELF); + command.arg("--map").arg(case).endow(HOLDER, holder.into()); + for (label, end) in ends { + command.endow(label, end.into()); + } + command.spawn().expect("mapprobe: a mapper starts") + } +} + +/// A pipe: the pager's end, and the mapper's. +fn pipe() -> (Pipe, OwnedHandle) { + let (read, write) = toyos::pipe_pair().expect("mapprobe: a pipe"); + (read, OwnedHandle::from(write)) +} + +/// A pipe the other way: the mapper reads. +fn go_pipe() -> (Pipe, OwnedHandle) { + let (read, write) = toyos::pipe_pair().expect("mapprobe: a pipe"); + (write, OwnedHandle::from(read)) +} + +fn ended(child: &mut Child) -> End { + End::from_word(child.wait().expect("mapprobe: a mapper's end").into_raw()).expect("mapprobe: an end the kernel makes") +} + +fn coherent(served: &mut Served) { + let object = served.pager.object(2 * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, 1, 0x11); + let (ready, theirs) = pipe(); + let mut child = served.mapper(object, "coherent", vec![(READY, theirs)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the coherent mapper never read the supplied page"); + assert!(!served.heard(|n| n == Notice::NoHolders { object }), "mapprobe: a mapped object's last holder was said to go"); + assert_eq!(served.pager.write(object, 0, &[0x22; PAGE as usize]), Ok(PAGE as usize)); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the coherent mapper"); + served.unheld(object); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a mapper sees the pager's write, and its mapping holds the object it closed"); +} + +fn parks(served: &mut Served) { + let object = served.pager.object(4 * RUN * PAGE).expect("mapprobe: an object"); + let mut child = served.mapper(object, "parks", Vec::new()); + served.requested(object, 0); + served.supply(object, 0, RUN, 0x33); + served.requested(object, RUN); + assert_eq!(served.pager.withdraw(object, 0, RUN), Ok(RUN), "mapprobe: the first run withdrawn under its mapper"); + // A stall is the stimulus here, not a wait for an event: the mapper's + // fault waits past the report time while the pager does nothing. + std::thread::sleep(2 * REPORT); + served.supply(object, RUN, RUN, 0x44); + served.requested(object, 0); + served.supply(object, 0, RUN, 0x33); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the parked mapper"); + println!("mapprobe: a fault waits for its page, a withdrawn page comes back, and a stalled supply is waited out"); + + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let mut child = served.mapper(object, "unmapped", vec![(READY, theirs), (GO, mine)]); + served.requested(object, 3 * RUN); + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never took its range away"); + served.supply(object, 3 * RUN, RUN, 0x55); + assert_eq!(ended(&mut child), End::Fault(Fault::Memory), "mapprobe: a thread woken in a range taken away"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a range taken away under a parked thread is nothing when its page comes"); +} + +fn killed(served: &mut Served) { + let object = served.pager.object(2 * RUN * PAGE).expect("mapprobe: an object"); + let mut child = served.mapper(object, "fault", Vec::new()); + served.requested(object, 0); + child.kill().expect("mapprobe: kill the parked mapper"); + assert_eq!(ended(&mut child), End::Killed, "mapprobe: a mapper parked in its fault"); + let (out, theirs) = pipe(); + let mut child = served.mapper(object, "copy", vec![(OUT, theirs)]); + served.requested(object, RUN); + child.kill().expect("mapprobe: kill the parked writer"); + assert_eq!(ended(&mut child), End::Killed, "mapprobe: a mapper parked in a write's copy"); + assert_eq!(out.read(&mut [0; 16]), Ok(0), "mapprobe: a write killed in its copy wrote"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing"); +} + +fn hole(served: &mut Served) { + let object = served.pager.anonymous(2 * PAGE).expect("mapprobe: anonymous memory"); + let room = served.pager.room().expect("mapprobe: room"); + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let mut child = served.mapper(object, "hole", vec![(READY, theirs), (GO, mine)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never read its hole"); + assert!(!served.heard(|n| matches!(n, Notice::Requested { object: o, .. } if o == object)), "mapprobe: a hole was requested"); + assert_eq!(served.pager.room(), Ok(room - 1), "mapprobe: a touched hole is not one page of the pager's"); + let filler = served.pager.object((room - 1) * PAGE).expect("mapprobe: an object to fill the pager"); + served.supply(filler, 0, room - 1, 0x99); + assert_eq!(served.pager.room(), Ok(0)); + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a hole touched with no room"); + served.pager.release(filler).expect("mapprobe: release"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room"); +} + +fn shrunk(served: &mut Served) { + let object = served.pager.object(4 * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, 4, 0x66); + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let mut child = served.mapper(object, "shrunk", vec![(READY, theirs), (GO, mine)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never read its last page"); + served.pager.resize(object, PAGE).expect("mapprobe: a shrink"); + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ended(&mut child), End::Fault(Fault::Memory), "mapprobe: a touch past a shrunk object"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a page the object shrank past is nothing to its mapper"); +} + +fn ends(mut served: Served) { + let object = served.pager.object(2 * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, 1, 0x77); + let (out, theirs) = pipe(); + let mut child = served.mapper(object, "ends", vec![(OUT, theirs)]); + served.requested(object, 0); + drop(served); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a mapper of an ended pager's object"); + assert_eq!(out.read(&mut [0; 16]), Ok(0), "mapprobe: a write from an ended pager's page wrote"); + println!("mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch"); +} + +/// The object a mapper was endowed, mapped whole. +fn mapped(pages: u64) -> *mut u8 { + let handle: OwnedHandle = Endowments::get().take(HOLDER).expect("mapprobe --map: its holder"); + let holder = Paged::received(handle).expect("mapprobe --map: a holder it may read"); + // SAFETY: placed anywhere, so it replaces nothing; the holder goes and + // the mapping holds the object. + unsafe { holder.map(0, (pages * PAGE) as usize, VmAt::Anywhere { align: PAGE as usize }) }.expect("mapprobe --map: the object mapped") +} + +fn byte(at: *mut u8, page: u64) -> u8 { + // SAFETY: inside the mapping this case made, read-only. + unsafe { at.add((page * PAGE) as usize).read_volatile() } +} + +fn pipe_end(label: &str) -> Pipe { + Endowments::get().take(label).expect("mapprobe --map: a pipe end") +} + +fn map(case: &str) { + match case { + "coherent" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0x11, "mapprobe --map: the supplied page"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + let began = Instant::now(); + while byte(at, 0) != 0x22 { + assert!(began.elapsed() < BOUND, "mapprobe --map: the pager's write never reached the mapping"); + std::thread::yield_now(); + } + } + "parks" => { + let at = mapped(4 * RUN); + assert_eq!(byte(at, 4), 0x33); + assert_eq!(byte(at, RUN + 4), 0x44); + assert_eq!(byte(at, 4), 0x33, "mapprobe --map: a withdrawn page came back as other bytes"); + } + "unmapped" => { + let at = mapped(4 * RUN) as usize; + let parked = std::thread::spawn(move || byte(at as *mut u8, 3 * RUN + 2)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + // SAFETY: the mapping this case made, which the parked thread + // touches again only to fault. + unsafe { syscall::vm_release(at as *mut u8, (4 * RUN * PAGE) as usize) }.expect("mapprobe --map: unmap"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + let read = parked.join(); + panic!("mapprobe --map: a thread woken in a range taken away read {read:?}"); + } + "fault" => { + let read = byte(mapped(2 * RUN), 2); + panic!("mapprobe --map: a page nobody supplies read {read}"); + } + "copy" => { + let at = mapped(2 * RUN); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at.add((RUN * PAGE) as usize), PAGE as usize) }; + let wrote = pipe_end(OUT).write(page); + panic!("mapprobe --map: a write from a page nobody supplies answered {wrote:?}"); + } + "hole" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0, "mapprobe --map: an unwritten page"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + let read = byte(at, 1); + panic!("mapprobe --map: a hole the pager had no room for read {read}"); + } + "shrunk" => { + let at = mapped(4); + assert_eq!(byte(at, 3), 0x66); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + let read = byte(at, 3); + panic!("mapprobe --map: a page past the object read {read}"); + } + "ends" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0x77); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at.add(PAGE as usize), PAGE as usize) }; + assert_eq!(pipe_end(OUT).write(page), Err(SyscallError::Io), "mapprobe --map: a write from an ended pager's page"); + let read = byte(at, 0); + panic!("mapprobe --map: a page of an ended pager read {read}"); + } + other => panic!("mapprobe --map: no case {other:?}"), + } +} diff --git a/tests/guest/rust-tests/src/bin/mapped_file.rs b/tests/guest/rust-tests/src/bin/mapped_file.rs new file mode 100644 index 0000000000..1996b04eb4 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/mapped_file.rs @@ -0,0 +1,20 @@ +//! A paged object mapped by another process, on both architectures: +//! `mapprobe`, launched as its row's pager, maps its objects in children of +//! its own, answers or withholds each fault and judges each child by its end +//! (`tests/guest/mapprobe`). The probe says each claim as it holds and exits +//! 0 once every one did. +//! +//! Needs the guest: the faults, waits, revocations and refusals are a booted +//! kernel's, between two processes. + +use std::os::toyos::process::CommandExt; +use std::process::Command; + +fn main() { + let status = Command::new("/system/bin/mapprobe") + .independent() + .status() + .expect("the mapping probe launches: its image's test-runner starts it"); + assert!(status.success(), "the mapping probe ended {status:?}"); + println!("mapped_file: a mapped paged object follows its pager"); +} diff --git a/tests/images/testcases.toml b/tests/images/testcases.toml index 378112b5c3..564f5897eb 100644 --- a/tests/images/testcases.toml +++ b/tests/images/testcases.toml @@ -76,8 +76,9 @@ syscap = ["rt"] # `/state/test-runner` and nothing else of `/state`. [programs.test-runner] state = true -# `pagerprobe`, a pager row, which `a_pager_s_end_answers_its_waiting_holders_gone` launches. -starts = ["pagerprobe"] +# `pagerprobe` and `mapprobe`, pager rows, which `a_pager_s_end_answers_its_waiting_holders_gone` +# and `mapped_file` launch. +starts = ["pagerprobe", "mapprobe"] receives = ["soundserver", "volume", "power", "log"] syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "inventory", "jobs"] @@ -85,6 +86,10 @@ syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "i [programs.pagerprobe] pager = true +# A pager whose objects its own children map (`tests/guest/mapprobe`). +[programs.mapprobe] +pager = true + [programs.toybox] receives = ["soundserver"] diff --git a/tests/images/virtjobcase.toml b/tests/images/virtjobcase.toml index 64a46a17f6..ef63723253 100644 --- a/tests/images/virtjobcase.toml +++ b/tests/images/virtjobcase.toml @@ -20,12 +20,18 @@ restart = true [programs.test-runner] receives = ["power"] +# `mapprobe`, a pager row, which `test_rs_mapped_file` launches. +starts = ["mapprobe"] # Four minutes from boot rather than one: a job here may run on an emulated # CPU, and a job past the bound reboots the guest with the job named. -args = ["--bound-ms=240000", "preempt", "fp_isolation", "first_entry", "unmap_touch", "debug_refused", "test_rs_ring0_timer_in_syscall", "test_rs_random_draws", "test_rs_wall_clock_now", "test_rs_user_copy_faults", "test_rs_vm_map_shares", "test_rs_relro_sealed", "test_rs_abuse_readonly_copyout"] +args = ["--bound-ms=240000", "preempt", "fp_isolation", "first_entry", "unmap_touch", "debug_refused", "test_rs_ring0_timer_in_syscall", "test_rs_random_draws", "test_rs_wall_clock_now", "test_rs_user_copy_faults", "test_rs_vm_map_shares", "test_rs_mapped_file", "test_rs_relro_sealed", "test_rs_abuse_readonly_copyout"] [programs.kernelprobe] +# A pager whose objects its own children map (`tests/guest/mapprobe`). +[programs.mapprobe] +pager = true + [symlinks] "bin/preempt" = "/system/bin/kernelprobe" "bin/fp_isolation" = "/system/bin/kernelprobe" diff --git a/tests/toyos.rs b/tests/toyos.rs index 7d266e9deb..3e3ab30d8c 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -401,6 +401,10 @@ const DRIVEN_AND_SHARED: &[&str] = &[ // Its shared run is the x86-64 verdict; `virt_jobs_at_el2` builds it for // AArch64 and runs it on that architecture's job case. "relro_sealed", + // Its shared run is the x86-64 verdict on its cases; + // `a_mapped_file_follows_its_pager` runs it again for the kernel's report + // on serial, and `virt_mapped_file` on AArch64's job case. + "mapped_file", ]; /// What the boot's audit of the kernel's tables says when every leaf passed @@ -456,6 +460,9 @@ const SCREEN_TESTS: &[ScreenTest] = &[ // `tmp_empty_files` on AArch64: the paged objects' table and its notices // are shared memory-management code. ("virt_tmp_holds_empty_files_by_what_each_costs", qemu::Profile::Virt, |profile, _| virt_tmp_holds_empty_files_by_what_each_costs(profile)), + // `a_mapped_file_follows_its_pager` on AArch64: a fault that parks on its + // pager there, the CPU's fault mark down while it waits. + ("virt_mapped_file", qemu::Profile::Virt, |profile, _| virt_job(profile, "test_rs_mapped_file")), ("virt_ring0_timer_in_syscall", qemu::Profile::Virt, |profile, _| virt_job(profile, &format!("test_rs_{VIRT_RING0_TIMER}"))), // Emulated, with `virt_el1_smp`, `virt_off_names_the_cpus_left_on` and // `virt_reboot`: under HVF a woken thread can go undispatched for seconds @@ -731,6 +738,15 @@ const MACHINE_TESTS: &[MachineTest] = &[ ("a_user_copy_highest_first_faults_or_ends_as_bad_address", |test_config| { a_user_copy_faults_in_mid_copy_or_ends_as_bad_address(test_config, &["copies-go-down"]) }), + // A paged object mapped by a second process (`tests/guest/mapprobe`), and + // the kernel's report that a mapper's fault waited past its pager's + // report time and then no longer did. That report is on serial and not + // the job's, and `testcases_members_exit_0` judges the member by its exit + // alone, so this boot of QEMU's x86-64 machine is its own. The job's + // cases, two processes, real tables and the kernel's fault path, which no + // host test has, are the shared run's too, and `virt_mapped_file` runs + // them on AArch64. + ("a_mapped_file_follows_its_pager", |test_config| a_mapped_file_follows_its_pager(test_config)), // The boot's log written through usbd onto the stick it booted from, and // still there once the machine has stopped: the path every metal boot's // `/log` takes, which no other QEMU row reads, because their console is @@ -3105,6 +3121,7 @@ const VIRT_JOBS: &[(&str, &str)] = &[ ("test_rs_user_copy_faults", "user copies: all checks passed"), ("test_rs_vm_map_shares", "vm_map_shares: an object's pages are mapped where asked"), ("test_rs_relro_sealed", "relro_sealed: a store to a relocated word of PT_GNU_RELRO ended its program as a memory fault"), + ("test_rs_mapped_file", "mapped_file: a mapped paged object follows its pager"), ("test_rs_abuse_readonly_copyout", "a syscall writes only where its caller could store"), ]; @@ -3576,6 +3593,44 @@ fn a_user_copy_faults_in_mid_copy_or_ends_as_bad_address(test_config: &Path, ker Ok(()) } +/// What `mapprobe` says as each of its cases holds (`tests/guest/mapprobe`). +const MAPPED_FILE_SAID: &[&str] = &[ + "mapprobe: a mapper sees the pager's write, and its mapping holds the object it closed", + "mapprobe: a fault waits for its page, a withdrawn page comes back, and a stalled supply is waited out", + "mapprobe: a range taken away under a parked thread is nothing when its page comes", + "mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing", + "mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room", + "mapprobe: a page the object shrank past is nothing to its mapper", + "mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch", + "mapped_file: a mapped paged object follows its pager", +]; + +/// `mapped_file` alone: each of the probe's claims said, and the kernel's +/// report that a mapper's fault waited past its pager's report time and then +/// no longer did, which is on the serial line and not the job's own. +fn a_mapped_file_follows_its_pager(test_config: &Path) -> Result<(), String> { + const JOB: &str = "mapped_file"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], BootOptions::default()); + serial::Serial::boot(&qemu).must_be_clean()?; + let result = qemu.run_test("test_rs_mapped_file", Duration::from_secs(120)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + if let Some(claim) = MAPPED_FILE_SAID.iter().find(|claim| !result.stdout.contains(*claim)) { + return Err(format!("the job never said {claim:?}:\n{}", result.stdout)); + } + for report in ["'s pager has a read overdue: object ", "'s pager has no read overdue"] { + if !result.serial.contains(report) { + return Err(format!("the kernel never said {report:?}:\n{}", result.serial)); + } + } + Ok(()) +} + /// A boot whose marked slot's supervisor is bent, beside a slot that /// verifies: the loader refuses the marked slot by the section, reads nothing /// of the other, hands no kernel the machine, and the machine stops. From 70562f94d84c022a9ff71253e5b755c66cb85289 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:59:21 +0200 Subject: [PATCH 13/34] A mapped object's range count is the kernel's own code under loom The count of a space's ranges holding its node moves to `object/paged/ranges.rs`, which the loom crate compiles as it does `shootdown.rs`. `paged_ranges.rs` races an unmap, which clears its leaf and lets its range go, against a parked fault letting its hold go with no space lock: whichever is last must see the leaf cleared before the node leaves its object's list, since the walk then stops visiting the space and frees the frame. It passes with the count's release and acquire, and the negative control `ranges-less-relaxed` reds it with loom's "Causality violation: Concurrent read and write accesses". The SDK's last-holder notice says a mapping is a holder. Co-Authored-By: Claude Opus 5.5 --- kernel/Cargo.toml | 3 ++ kernel/loom/Cargo.toml | 11 +++++ kernel/loom/src/lib.rs | 5 +++ kernel/loom/tests/paged_ranges.rs | 69 +++++++++++++++++++++++++++++++ kernel/src/object/paged.rs | 17 ++++---- kernel/src/object/paged/ranges.rs | 50 ++++++++++++++++++++++ sdk/toyos/src/paged.rs | 2 +- 7 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 kernel/loom/tests/paged_ranges.rs create mode 100644 kernel/src/object/paged/ranges.rs diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index e92c4755b7..446ee0c765 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -58,6 +58,9 @@ reap-raise-relaxed = [] # `src/shootdown.rs`'s `serve` reads what it owes `Relaxed`, and # `tlb_shootdown` reds. shootdown-serve-relaxed = [] +# `src/object/paged/ranges.rs`'s `less` goes `Relaxed`, and `paged_ranges` +# reds. +ranges-less-relaxed = [] # `src/shootdown.rs`'s `served` reads `Relaxed`, and `counters_round` reds. shootdown-served-relaxed = [] # `src/shootdown.rs`'s `Asks::serve` takes its asks `Relaxed`, and diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index 7774945863..46c24d3dea 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -221,6 +221,17 @@ log-ring-loads-swapped = [] tlb-tags-enter-relaxed = [] # The same for `Tags::resume`, which must red the idle holder's model. tlb-tags-resume-relaxed = [] +# The negative control for a mapped object's range count. It makes +# `object/paged/ranges.rs`'s `less` `Relaxed` with no fence, so the last range +# to let go can unlink its node without seeing another's leaf cleared, and +# `paged_ranges.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features ranges-less-relaxed \ +# --test paged_ranges +# +# Never on by default and never reachable from a kernel build, which declares +# the same name only so `cfg` checking knows it. +ranges-less-relaxed = [] # The negative control for the panic console's published descriptor. It drops # the publisher's `Release` fence between its odd mark and its words, so a diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index 74955aab90..6bccece3b5 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -158,6 +158,11 @@ pub mod shootdown; #[path = "../../src/tlb_tags.rs"] pub mod tlb_tags; +// A mapped object's count of the ranges holding a space's node: its last +// letting go sees every other's clearing (`tests/paged_ranges.rs`). +#[path = "../../src/object/paged/ranges.rs"] +pub mod paged_ranges; + /// The CPU roster and the release/answer word, driven by `tests/smp_bringup.rs`; /// under `loom` alone, as is the [`arch::cpu`] shim it names. #[cfg(feature = "loom")] diff --git a/kernel/loom/tests/paged_ranges.rs b/kernel/loom/tests/paged_ranges.rs new file mode 100644 index 0000000000..d97cbcd3cb --- /dev/null +++ b/kernel/loom/tests/paged_ranges.rs @@ -0,0 +1,69 @@ +//! Loom: a mapped object's node leaves its list only after every range of +//! the space let go of it, each having cleared its leaves first. +//! +//! A space's node of a mapped object is held by each of its ranges of the +//! object and by a fault parked on one of its pages, which holds no space +//! lock. Whoever lets go last unlinks the node, and from then on a walk of +//! the object's mappers does not visit the space: it frees a revoked frame +//! once its pass is done. So the last to let go must see every other's +//! clearing of its leaves, or a frame is freed under a leaf that may still +//! reach it. A leaf here is one cell; `CLEARED` read by the last is the +//! property, and loom reports a read the clearing does not happen before as a +//! causality violation. +//! +//! ```text +//! cargo test --manifest-path kernel/loom/Cargo.toml --features ranges-less-relaxed \ +//! --test paged_ranges +//! ``` +//! +//! relaxes the count's letting go, and this file must red. + +#![cfg(feature = "loom")] + +use kernel_loom::paged_ranges::Ranges; +use loom::cell::UnsafeCell; +use loom::sync::Arc; + +const MAPPED: u64 = 1; +const CLEARED: u64 = 0; + +/// What the last to let go does before the node leaves the list: the walk +/// it stops visiting will free whatever this leaf maps. +fn unlink(leaf: &UnsafeCell) { + // SAFETY: loom checks the access; the model's claim is that it is ordered. + let seen = leaf.with(|leaf| unsafe { *leaf }); + assert_eq!(seen, CLEARED, "the node left its list while a leaf of its space still mapped the page"); +} + +/// An unmap in the space clears its leaf and lets its range go; a fault +/// parked on the range's page lets its hold go as it wakes. Either may be +/// last. +#[test] +fn the_last_to_let_go_sees_every_leaf_cleared() { + loom::model(|| { + let ranges = Arc::new(Ranges::one()); + ranges.more(); + let leaf = Arc::new(UnsafeCell::new(MAPPED)); + + let unmap = { + let (ranges, leaf) = (ranges.clone(), leaf.clone()); + loom::thread::spawn(move || { + // SAFETY: the space's lock is the unmapper's; loom checks it. + leaf.with_mut(|leaf| unsafe { *leaf = CLEARED }); + if ranges.less() { + unlink(&leaf); + } + }) + }; + let parked = { + let (ranges, leaf) = (ranges.clone(), leaf.clone()); + loom::thread::spawn(move || { + if ranges.less() { + unlink(&leaf); + } + }) + }; + unmap.join().unwrap(); + parked.join().unwrap(); + }); +} diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 4e1a964309..0a315f4b5b 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -71,6 +71,9 @@ use crate::sync::{Lock, LockGuard}; use crate::time::{Deadline, Duration}; use crate::user_ptr::{Pinned, Stopped, UserBytes, UserBytesMut}; use crate::watch::{self, Watch}; + +mod ranges; +use ranges::Ranges; use crate::DirectMap; use super::{KObjectVariant, ObjectCore, ZeroHandles}; @@ -1269,8 +1272,8 @@ pub struct MapNode { pager: Arc, slot: u32, space: Weak, - /// The ranges holding it ([`PagedRange`]), each under its space's lock. - ranges: AtomicU32, + /// The ranges holding it ([`PagedRange`]). + ranges: Ranges, /// Under the pager's lock. link: UnsafeCell, /// Itself, for a walk to hold while it visits. @@ -1474,18 +1477,14 @@ impl PagedRange { impl Clone for PagedRange { fn clone(&self) -> Self { - self.0.ranges.fetch_add(1, Ordering::Relaxed); + self.0.ranges.more(); Self(Arc::clone(&self.0)) } } impl Drop for PagedRange { fn drop(&mut self) { - // The last may be a parked fault's, holding no space lock: as `Arc`'s - // count, each drop releases what its holder cleared, and the last - // acquires every one before the node leaves the list. - if self.0.ranges.fetch_sub(1, Ordering::Release) == 1 { - core::sync::atomic::fence(Ordering::Acquire); + if self.0.ranges.less() { self.0.pager.unlink(&self.0); } } @@ -1503,7 +1502,7 @@ impl PagerObject { pager: Arc::clone(self), slot, space, - ranges: AtomicU32::new(1), + ranges: Ranges::one(), link, this: this.clone(), _charge: charge, diff --git a/kernel/src/object/paged/ranges.rs b/kernel/src/object/paged/ranges.rs new file mode 100644 index 0000000000..4243c0f29b --- /dev/null +++ b/kernel/src/object/paged/ranges.rs @@ -0,0 +1,50 @@ +//! How many ranges of one space hold its node of a mapped object +//! (`object::paged::MapNode`). A range clears and shoots down its leaves +//! before it lets go; the last to let go, which may be a parked fault's hold +//! with no space lock, sees every other's clearing before the node leaves its +//! object's list, so a walk that no longer visits the space frees no frame a +//! leaf there may still reach. `kernel/loom/tests/paged_ranges.rs` checks it. + +#[cfg(not(feature = "loom"))] +use core::sync::atomic::{fence, AtomicU32, Ordering}; + +#[cfg(feature = "loom")] +use loom::sync::atomic::{fence, AtomicU32, Ordering}; + +/// The order a letting go publishes its clearing with, and the last reads +/// every one with: `Arc`'s. Its negative control relaxes both. +#[cfg(not(feature = "ranges-less-relaxed"))] +const LESS: Ordering = Ordering::Release; +#[cfg(not(feature = "ranges-less-relaxed"))] +const LAST: Ordering = Ordering::Acquire; + +#[cfg(feature = "ranges-less-relaxed")] +const LESS: Ordering = Ordering::Relaxed; +#[cfg(feature = "ranges-less-relaxed")] +const LAST: Ordering = Ordering::Relaxed; + +pub struct Ranges(AtomicU32); + +impl Ranges { + /// A node's first range. + pub fn one() -> Self { + Self(AtomicU32::new(1)) + } + + /// Another range of a node a range holds. + pub fn more(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + + /// A range lets go: whether it was the last. + #[must_use] + pub fn less(&self) -> bool { + if self.0.fetch_sub(1, LESS) != 1 { + return false; + } + if LAST != Ordering::Relaxed { + fence(LAST); + } + true + } +} diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 81daa4924c..e6f56e27d4 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -54,7 +54,7 @@ impl Refused { pub enum Notice { /// A reader waits for `count` pages from `first`. Requested { object: u64, first: u64, count: u64 }, - /// The last holder handle went. + /// The last holder, a handle or a space's mapping, went. NoHolders { object: u64 }, } From 1f48d68a869f22eaaf13caf3b238c03a14c9d825 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:01:32 +0200 Subject: [PATCH 14/34] osbench: a cached file's pages through a mapping, against Linux's mmap Two rows of every file target, on its 16 MiB file once its pages are cached: `map_touch_16m` maps the file read-only and shared, touches each page once and unmaps it (20 after 2), so its time is the faults fault-around leaves; `map_read_4k` copies 4 KiB out of a page of one mapping whose every page was touched (2,000 after 100), the read path a mapping replaces. Linux maps with `mmap(PROT_READ, MAP_SHARED)`; ToyOS opens the file through the SDK and maps its holder, since its std `File` names no descriptor. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/files.rs | 35 ++++++++++++++++++++++++++++ tests/guest/osbench/src/sys/linux.rs | 13 +++++++++++ tests/guest/osbench/src/sys/toyos.rs | 15 ++++++++++++ 3 files changed, 63 insertions(+) diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index a0e6e1e450..bc2e507613 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -117,6 +117,40 @@ fn parts( each(&name("pread_8"), 100, 2000, fields, |i| io("pread", f.read_at(&mut buf[..8], page_at(i, pages))).map(drop)); } +/// The 16 MiB file `big`, its pages cached, mapped read-only and shared. +/// Mapped, each page touched once and unmapped, so each touch is a fault +/// unless fault-around mapped its page already: 20 after 2. Then 4 KiB copied +/// out of a page of one mapping every page of which was touched: 2,000 after +/// 100. +fn mapped(name: &dyn Fn(&str) -> String, fields: &[(&'static str, String)], big: &Path, pages: usize) { + let len = pages * 4096; + let touch = |at: *mut u8| (0..pages).fold(0u8, |sum, page| { + // SAFETY: a page of the mapping, read-only. + sum.wrapping_add(unsafe { at.add(page * 4096).read_volatile() }) + }); + each(&name("map_touch_16m"), 2, 20, fields, |_| { + let at = crate::sys::map_file(big, len)?; + std::hint::black_box(touch(at)); + // SAFETY: the mapping just made, which nothing else names. + unsafe { crate::sys::unmap(at, len) }; + Ok(()) + }); + let at = match crate::sys::map_file(big, len) { + Ok(at) => at, + Err(why) => return refuse(&name("map_read_4k"), why), + }; + std::hint::black_box(touch(at)); + let mut buf = [0u8; 4096]; + each(&name("map_read_4k"), 100, 2000, fields, |i| { + // SAFETY: a page of the mapping, every one of which is mapped. + unsafe { core::ptr::copy_nonoverlapping(at.add(page_at(i, pages) as usize), buf.as_mut_ptr(), buf.len()) }; + std::hint::black_box(&buf); + Ok(()) + }); + // SAFETY: the mapping made above, which nothing else names. + unsafe { crate::sys::unmap(at, len) }; +} + /// Every benchmark under `target`, each line naming what served it and its /// medium. Each one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { @@ -203,6 +237,7 @@ pub fn suite(target: &Target) { } Err(e) => refuse(&name("read_4k_at"), e), } + mapped(&name, &with(&[("file_bytes", (16 * MIB).to_string())]), &big, pages); let _ = fs::remove_file(&big); // Listing a directory of 1,000 files, every name: 50 after 5. diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index 4c056d177d..73c528b014 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -61,6 +61,19 @@ pub fn map_anon(len: usize) -> *mut u8 { /// # Safety /// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing /// reaches it afterwards. +/// `len` bytes of the file at `path` from its start, mapped read-only and +/// shared. +pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { + use std::os::fd::AsRawFd; + let file = std::fs::File::open(path).map_err(|e| format!("open: {e}"))?; + // SAFETY: a fresh read-only mapping of a file this process opened. + let ptr = unsafe { libc::mmap(std::ptr::null_mut(), len, libc::PROT_READ, libc::MAP_SHARED, file.as_raw_fd(), 0) }; + if ptr == libc::MAP_FAILED { + return Err(format!("mmap: {}", std::io::Error::last_os_error())); + } + Ok(ptr.cast()) +} + pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. assert_eq!(unsafe { libc::munmap(ptr.cast(), len) }, 0); diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index b562f789d0..71374012e0 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -105,6 +105,21 @@ pub fn huge_backing() -> String { /// # Safety /// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing /// reaches it afterwards. +/// `len` bytes of the file at `path` from its start, mapped read-only and +/// shared: its file server's own cached pages. +pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { + use toyos::files::{At, Open}; + let path = path.to_str().ok_or("a path that is not UTF-8")?; + let fd = toyos::files::open(At::Cwd, path.as_bytes(), Open { read: true, cloexec: true, ..Open::default() }) + .map_err(|e| format!("open: {e:?}"))?; + // SAFETY: placed anywhere, so it replaces nothing. + let mapped = unsafe { toyos::files::map(fd, 0, len, VmAt::Anywhere { align: 4096 }) }; + // SAFETY: the descriptor this opened, which nothing else names; the + // mapping holds the file. + let _ = unsafe { toyos::fd::close(fd) }; + mapped.map_err(|e| format!("map: {e:?}")) +} + pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. unsafe { syscall::vm_release(ptr, len) }.expect("vm_release of a whole mapping"); From 27a826a841a5b2d61524c0ff187a188d4ad87785 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:02:44 +0200 Subject: [PATCH 15/34] issues: what step 2 of 7b built, and what it owes libc maps a file read-only, shared or private; other protections still answer `ENODEV`, now for what is not built (step 5's write faults, R4's private copy ranges of a file). The memory track says step 2 is built but for R4's copy ranges, which its item 5 owes; that only a pager's pages are revoked, so ROOT stays non-reclaimable until a pager serves it; and which mapping measurements remain owed after the two osbench rows step 2 adds. Co-Authored-By: Claude Opus 5.5 --- ...bc-refuses-what-toyos-cannot-yet-answer.md | 6 ++++- ...b-pages-still-copies-each-programs-text.md | 26 +++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index 5c6014790d..63fcf431a4 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -18,7 +18,11 @@ These answer failure in their POSIX form and do nothing the failure and reads the `rlimit` it did not get. - `gethostname`, `ENOSYS`: no host name is published to a process. - `msync`, `ENOSYS`: no mapping is of a file, so there is nothing to write back. -- `mmap` of a file, `ENODEV`: the kernel maps no file. +- `mmap` of a file other than `PROT_READ`, `ENODEV`: the kernel maps a file's + pages read-only and shared, and a private one shares them until it is written, + which waits for 7b's write faults (step 5); a private copy range of a file + (`PROT_WRITE` or `PROT_EXEC` with `MAP_PRIVATE`, 7b's R4) is not built either. + A file whose server maps no pages answers `ENODEV` too. - `fcntl` (`sdk/libc/src/fdreq.rs`): a record lock, `EINVAL`, POSIX's answer for a file that supports no locking; `F_GETFL`, `F_SETFL`, `F_GETOWN` and `F_SETOWN`, `ENOSYS`. diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index ab461129bd..3eeb4838ba 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -95,10 +95,12 @@ live split of a user leaf. at its first touch. A program anywhere else is still read whole into memory of the spawner's and copied a page at a time, until 7b's step 3. A shared segment's mapping is never widened, so a program can no longer - make its own read-only segment writable (`mprotect`). Exit for that: 7b's - step 2 converts on protect, so a protect that adds write to such a range - commits its pages and makes it a copy range of the same object; only a - page shared until its first write waits for step 5's write faults. Exit: + make its own read-only segment writable (`mprotect`). Exit for that: a + protect that adds write to such a range commits its pages and makes it a + copy range of the same object (7b's R4, which step 2 did not build: it + maps a paged object read-only only, and its range refuses a protect past + that); only a page shared until its first write waits for step 5's write + faults. Exit: two processes of one binary hold its text once, wherever it is served from. 6. **`/apps` image sharing.** Decided by the orchestrator under the owner's @@ -118,7 +120,8 @@ live split of a user leaf. **7b lands in steps**, each a pull request: (1) a range of a memory object mapped where the caller says, an immutable one shared and executable, which is stage 5 for `/system`: built; (2) a paged object mapped read-only and -shared: faults that park on the pager, reverse records, revocation; (3) a +shared: faults that park on the pager, reverse records, revocation: built, +but for R4's copy ranges of a paged object, which item 5 owes; (3) a spawn maps its image from any pager-backed volume (stage 6), sealed against writes in place while it runs; (4) the storage track's attributes and names without a request; (5) writable shared @@ -137,9 +140,16 @@ mappings and write-back; (6) the #1077 retry before; a spawn no longer copies a `/system` program's text, and whether that makes it faster is not measured on metal. -Until step 2's revocation exists, an object a space maps must keep its frames -for the object's life: ROOT's pages stay non-reclaimable, and ROOT is not -served through a pager, until then. +Step 2 measures a cached page's fault and a 4 KiB read through a mapping +against Linux (osbench `fs.*.map_touch_16m`, `fs.*.map_read_4k`). Owed, by +map7b, with step 3's metal: a cold DATA page's miss in stages, one object +mapped by 255 children and withdrawn (a revocation's cost grows with its +mappers), and many threads cold-faulting one large file; each exits as an +osbench row judged against Linux. + +A memory object a space maps keeps its frames for the object's life, since +only a pager's pages are revoked: ROOT's pages stay non-reclaimable until ROOT +is served through a pager. **Paged objects** (memory design §7, `issues/toyos-builds-itself.md` item 13) are split in two. **7a**, delivered by the storage track's file-server work: From 3e3a52cb1a89da9423ef5ac94d72fd5ca0e2b9e8 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:58:52 +0200 Subject: [PATCH 16/34] issues: what step 2 owes on metal while the benches are down The memory track records step 2's metal for one batched boot of main when a bench returns: the whole testcases boot with `mapped_file`, shared-debug and proctreecase on both benches, every job exiting 0; osbench's fs, pipe, ipc, mem and proc groups before and after, no slower; and the two mapping rows beside Linux's `mmap`, unmeasured until then. The rows step 2 did not write stay owed by map7b. Co-Authored-By: Claude Opus 5.5 --- ...b-pages-still-copies-each-programs-text.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index 3eeb4838ba..f8b6ffbeb3 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -139,6 +139,25 @@ mappings and write-back; (6) the #1077 retry against main after, four boots a side. Main after must be no slower than before; a spawn no longer copies a `/system` program's text, and whether that makes it faster is not measured on metal. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: the whole + `testcases` boot, whose `mapped_file` member maps a paged object from a + second process (faults that wait, revocation, refusals, kills), and the + `shared-debug` and `proctreecase` boots. Main must show every job of each + boot exit 0. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's + `fs`, `pipe`, `ipc`, `mem` and `proc` groups, main before #1269 against + main after, four boots a side. Main after must be no slower, since the + fault and copy paths changed under all of them. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's + `fs.*.map_touch_16m` and `fs.*.map_read_4k` (a cached file's pages faulted + through a mapping, and 4 KiB read through one) on main, beside Linux's + `mmap` of the same file. Main must show both rows on every file target; + how a mapping compares with Linux's is not measured on metal. + +Rows step 2 did not write, owed by map7b: a cold DATA page's miss in stages, +one object mapped by 255 children and withdrawn (a revocation's cost grows +with its mappers), and many threads cold-faulting one large file; each exits +as an osbench row judged against Linux on metal. Step 2 measures a cached page's fault and a 4 KiB read through a mapping against Linux (osbench `fs.*.map_touch_16m`, `fs.*.map_read_4k`). Owed, by From acff199b5d9f005b61bcbbcc8f68eabb76a12665 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 22:29:00 +0200 Subject: [PATCH 17/34] Review r1 of #1269: a revocation visit holds the pager's lock only to read, and a touch past a mapped file's end is SIGBUS B3. A visit no longer runs its shootdown and futex revocation inside the pager's lock. It reads the object's revoking pages, lowest first, 64 at a time (`Pages::revoking_from`, unit-tested), under the pager's lock alone; then, under its space's lock alone, clears every leaf of its ranges that maps one of those frames, shoots the span down, and only then ends the futex waits on the frames, as `revoke_frames`' contract says. It touches only the revoked pages, not every leaf of every range of the object. Each batch still walks the object's resident pages under the pager's lock; the memory track records that cost with the row that prices it. The orchestrator's ruling on S1: a touch of a mapped page wholly past the object's end ends the program with `Fault::Mapping` (POSIX's `SIGBUS`) and its record (`Why::PastEnd`); a copy from one stays `BadAddress`, as POSIX's `EFAULT`. The manual's memory page says so, and `mapprobe`'s `shrunk` case expects it. S3: a fault's request run is `vm::AROUND` in pages, not a second 16. Co-Authored-By: Claude Opus 5.5 --- docs/src/architecture/memory.md | 4 +- kernel/pure/paged.rs | 48 ++++++++++++++++++++ kernel/src/mm/space.rs | 75 +++++++++++++++++++------------- kernel/src/object/paged.rs | 17 ++++++-- kernel/src/process.rs | 15 +++++-- tests/guest/mapprobe/src/main.rs | 7 +-- 6 files changed, 123 insertions(+), 43 deletions(-) diff --git a/docs/src/architecture/memory.md b/docs/src/architecture/memory.md index a466152f19..45aa029de6 100644 --- a/docs/src/architecture/memory.md +++ b/docs/src/architecture/memory.md @@ -64,7 +64,9 @@ waits' own timeouts, bound it. - A touch of a mapped page its file server failed, or of a file whose server ended, ends the program with a diagnostic naming the file's page and the - cause; a copying system call answers an I/O error instead. A touched + cause; a copying system call answers an I/O error instead. A touch of a + mapped page wholly past the file's end ends the program the same way, as + POSIX's `SIGBUS`, and a copy from one is refused as a bad address. A touched unwritten page of a file held in RAM becomes resident within its server's capacity and is refused once that is full. - Clean pages are reclaimed; dirty pages are written back first, without losing diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 129f4468ad..b17c1e9239 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -748,6 +748,31 @@ impl Pages { None } + /// `o`'s revoking pages at or past `from`, the lowest first, with the + /// frame each keeps until its walk ends: as many as fit in `out`, whose + /// count this answers. A walk's visit clears a mapper's leaves of them a + /// batch at a time, asking again from past the last. + pub fn revoking_from(&self, o: &Object, from: u64, out: &mut [(u64, F)]) -> usize { + let mut n = 0; + let mut at = o.head; + while at != NONE { + let s = self.slot(at); + at = s.next; + let (Some(_), Some(frame)) = (s.revoking, s.frame) else { continue }; + if s.page < from || (n == out.len() && s.page >= out[n - 1].0) { + continue; + } + let mut k = n.min(out.len() - 1); + n = (n + 1).min(out.len()); + while k > 0 && out[k - 1].0 > s.page { + out[k] = out[k - 1]; + k -= 1; + } + out[k] = (s.page, frame); + } + n + } + pub fn lookup(&self, o: &Object, page: u64) -> Lookup { if o.gone { return Lookup::Gone; @@ -1087,6 +1112,29 @@ mod tests { /// A removed page reads as absent, takes no second frame and no write, /// and keeps its frame until every mapper of its object was visited; /// then its frame goes out once and the page is supplied afresh. + #[test] + fn revoking_pages_come_lowest_first_a_batch_at_a_time() { + let mut p = Pages::new(16, 4).unwrap(); + let mut o = p.object(1, 16 * PAGE); + for page in [9, 2, 14, 5, 0, 11, 7] { + p.supply(&mut o, page, 100 + page, false, (page + 1) * PAGE).unwrap(); + } + let (removed, _) = p.withdraw(&mut o, 2, 12, &mut Nobody); + assert_eq!(removed, 5, "pages 2, 5, 7, 9 and 11 are withdrawn"); + let mut batch = [(0, 0); 2]; + let mut seen = Vec::new(); + let mut from = 0; + loop { + let n = p.revoking_from(&o, from, &mut batch); + seen.extend_from_slice(&batch[..n]); + if n < batch.len() { + break; + } + from = batch[n - 1].0 + 1; + } + assert_eq!(seen, [(2, 102), (5, 105), (7, 107), (9, 109), (11, 111)]); + } + #[test] fn a_revoking_page_keeps_its_one_frame_until_every_mapper_is_visited() { let mut p = Pages::new(8, 8).unwrap(); diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 186cc8655e..3f37c95340 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -556,42 +556,55 @@ impl AddressSpace { } /// The visit of a walk of `node`'s object: every leaf of this space's - /// ranges of it that the index no longer holds present as that frame is - /// cleared, every futex wait on its frame ends, and every CPU drops what - /// it cached of them. A fault meanwhile waits on this space's lock and - /// then finds the page as the index has it. + /// ranges of it that maps a revoking page's frame is cleared and shot + /// down, and then every futex wait on those frames ends. The pager's lock + /// is held only to read a batch of the revoking pages, never across a + /// shootdown; a fault meanwhile waits on this space's lock and then finds + /// the page as the index has it. pub fn revoke_paged(&mut self, node: &MapNode) { - let tables = &mut self.tables; - let mut cleared = [0u64; 64]; - let mut n = 0; - node.with_index(|object| { + const BATCH: usize = 64; + let mut from = 0; + loop { + let mut batch = [(0u64, 0u64); BATCH]; + let n = node.with_index(|index| index.revoking_from(from, &mut batch)); + let mut hit = [false; BATCH]; + let (mut lo, mut hi) = (u64::MAX, 0); + let tables = &mut self.tables; for (start, entry) in self.inventory.meeting(FLOOR, TOP) { let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; if !core::ptr::eq(range.node(), node) { continue; } - let mut stale = false; - tables.edit(start, entry.end, |va, leaf| match leaf { - Leaf::Mapped(m) if object.page((offset + (va - start)) / PAGE_SIZE) != Ok(m.phys) => { - stale = true; - if n == cleared.len() { - cleared.sort_unstable(); - crate::sched::futex::revoke_sorted(&cleared, |&phys| phys); - n = 0; + for (k, &(page, phys)) in batch[..n].iter().enumerate() { + let Some(va) = (page * PAGE_SIZE).checked_sub(*offset).map(|into| start + into).filter(|&va| va < entry.end) else { + continue; + }; + tables.edit(va, va + PAGE_SIZE, |_, leaf| match leaf { + Leaf::Mapped(m) if m.phys == phys => { + hit[k] = true; + (lo, hi) = (lo.min(va), hi.max(va + PAGE_SIZE)); + Some(Leaf::Empty) } - cleared[n] = m.phys; - n += 1; - Some(Leaf::Empty) - } - _ => None, - }); - if stale { - let _reached = tables.shoot_down(start, entry.end, &Unlinked::none()); + _ => None, + }); } } - }); - cleared[..n].sort_unstable(); - crate::sched::futex::revoke_sorted(&cleared[..n], |&phys| phys); + if lo < hi { + let _reached = tables.shoot_down(lo, hi, &Unlinked::none()); + let mut frames = [0u64; BATCH]; + let mut m = 0; + for (&(_, phys), _) in batch[..n].iter().zip(hit).filter(|&(_, hit)| hit) { + frames[m] = phys; + m += 1; + } + frames[..m].sort_unstable(); + crate::sched::futex::revoke_sorted(&frames[..m], |&phys| phys); + } + if n < BATCH { + return; + } + from = batch[n - 1].0 + 1; + } } /// Take every page of `[start, start + len)` away, whatever holds it: the @@ -1008,7 +1021,7 @@ impl AddressSpace { match found { Ok(phys) => phys, Err(Missing::Absent(since)) => return Populated::Wait(PageWait { range: range.clone(), page: of(page), since }), - Err(Missing::PastEnd | Missing::Hole) => return Populated::Unreachable, + Err(Missing::Hole) => unreachable!("space: a fault's touch of a hole is filled"), Err(Missing::Lost(lost)) => return Populated::Lost(lost), } } @@ -1046,10 +1059,10 @@ impl AddressSpace { pub enum Populated { /// The byte's physical address. At(u64), - /// No range allows the access there, or its paged page lies past the - /// object's end. + /// No range allows the access there. Unreachable, - /// A paged page its pager failed or can no longer supply. + /// A paged page its pager failed or can no longer supply, or one wholly + /// past its object's end. Lost(Lost), /// A paged page that is absent: the thread waits for it, holding nothing, /// and touches it again. diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 0a315f4b5b..24255a650a 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -1300,8 +1300,6 @@ unsafe impl Sync for MapNode {} pub enum Missing { /// Absent, or revoking, at this version of the object: the fault waits. Absent(Version), - /// At or past the object's end: nothing is there. - PastEnd, /// A hole of an anonymous object, which only [`Index::fault`] fills. Hole, /// Its pager cannot let a mapping have it. @@ -1328,6 +1326,9 @@ pub enum Why { Ended, /// A hole of an anonymous object, and the pager has no room for its page. NoRoom, + /// The page lies wholly past the object's end, which POSIX answers a + /// touch of with `SIGBUS` and a copy with `EFAULT`. + PastEnd, } impl Why { @@ -1337,6 +1338,7 @@ impl Why { Why::Released => "released by its pager", Why::Ended => "its pager ended", Why::NoRoom => "no room in its pager", + Why::PastEnd => "past the object's end", } } } @@ -1356,13 +1358,20 @@ impl Index<'_> { _ if self.state.gone => Err(self.lost(page, Why::Ended)), Lookup::Present(phys) => Ok(phys), Lookup::Absent => Err(Missing::Absent(object.version())), - Lookup::PastEnd => Err(Missing::PastEnd), + Lookup::PastEnd => Err(self.lost(page, Why::PastEnd)), Lookup::Zero => Err(Missing::Hole), Lookup::Failed => Err(self.lost(page, Why::Failed)), Lookup::Gone => Err(self.lost(page, Why::Released)), } } + /// The object's revoking pages at or past `from`, lowest first, each + /// with the frame it keeps until its walk ends + /// ([`Pages::revoking_from`]). + pub fn revoking_from(&self, from: u64, out: &mut [(u64, u64)]) -> usize { + self.state.pages.revoking_from(&self.state.entry(self.slot).object, from, out) + } + /// As [`Self::page`], for the page a fault touched: a hole of an /// anonymous object, a MEMORY volume's, gets a zeroed frame of its /// pager's, installed clean and charged as a supply's is, and is refused @@ -1416,7 +1425,7 @@ impl Index<'_> { } /// The pages a fault's request names: the aligned run around its page. -const FAULT_RUN: u64 = 16; +const FAULT_RUN: u64 = kernel::vm::AROUND / PAGE; impl MapNode { /// `f` over the object's pages, under the pager's lock. diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 1c54c791c9..9a9e7fdcfb 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -989,6 +989,8 @@ impl Faulted { /// What a copy this fault stopped short answers. pub fn stop(self) -> crate::user_ptr::Stop { match self { + // Nothing is there, as for a copy POSIX answers `EFAULT`. + Faulted::Lost(lost) if lost.why == crate::object::paged::Why::PastEnd => crate::user_ptr::Stop::Unreachable, Faulted::Lost(_) => crate::user_ptr::Stop::Lost, Faulted::Absent => crate::user_ptr::Stop::Absent, Faulted::Served | Faulted::Unreachable | Faulted::Killed => crate::user_ptr::Stop::Unreachable, @@ -1048,8 +1050,12 @@ pub fn wait_in(start: u64, end: u64, access: kernel::vm::Access) -> Result<(), c let touched = touch(page, access, |page| if page.wait() { Faulted::Served } else { Faulted::Killed }); match touched { Faulted::Served => {} - Faulted::Lost(_) => return Err(crate::user_ptr::Faulted::Lost), - Faulted::Unreachable | Faulted::Absent | Faulted::Killed => return Err(crate::user_ptr::Faulted::Unreachable), + stopped => { + return Err(match stopped.stop() { + crate::user_ptr::Stop::Lost => crate::user_ptr::Faulted::Lost, + _ => crate::user_ptr::Faulted::Unreachable, + }) + } } page += crate::mm::PAGE_SIZE; } @@ -1057,8 +1063,9 @@ pub fn wait_in(start: u64, end: u64, access: kernel::vm::Access) -> Result<(), c } /// A Ring 3 touch at `fault_addr` of a mapped file's page its pager cannot -/// let the process have ends the process as `SIGBUS`, with one record of what -/// was lost. The caller has put the CPU's fault mark down. +/// let the process have, or of one wholly past the file's end, ends the +/// process as `SIGBUS`, with one record of what was lost. The caller has put +/// the CPU's fault mark down. pub fn lost(fault_addr: u64, lost: crate::object::paged::Lost) -> ! { log!( "fault: pid {} touched {fault_addr:#x}, page {} of pid {}'s object {}: {}", diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs index 87c0fddcbb..5896ef2d88 100644 --- a/tests/guest/mapprobe/src/main.rs +++ b/tests/guest/mapprobe/src/main.rs @@ -16,7 +16,8 @@ //! parked in a write's copy from the mapping, which writes nothing. //! - `hole`: an unwritten page of an anonymous object reads zero with no //! request, and one the pager has no room for ends the mapper. -//! - `shrunk`: a page the object shrank past ends its mapper as nothing. +//! - `shrunk`: a touch of a page the object shrank past ends its mapper with +//! `SIGBUS`, as POSIX says of a page wholly past a mapped file's end. //! - `ends`, last: the pager's end answers a write from the mapping `Io` //! and ends the mapper at its next touch. @@ -232,9 +233,9 @@ fn shrunk(served: &mut Served) { assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never read its last page"); served.pager.resize(object, PAGE).expect("mapprobe: a shrink"); assert_eq!(go.write(&[1]), Ok(1)); - assert_eq!(ended(&mut child), End::Fault(Fault::Memory), "mapprobe: a touch past a shrunk object"); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a touch past a shrunk object"); served.pager.release(object).expect("mapprobe: release"); - println!("mapprobe: a page the object shrank past is nothing to its mapper"); + println!("mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS"); } fn ends(mut served: Served) { From 5d73370d73dced81853526e8cbe1db845055e777 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 22:29:01 +0200 Subject: [PATCH 18/34] Review r1 of #1269: every copy a syscall makes answers its own cause B5. The manual says a copying system call answers an I/O error for a mapped file's lost page. Fifteen sites flattened a failed copy to `BadAddress`: the argument records of spawn, message send and receive, `vm_map` and the paged op, a port's badge, the affinity mask read and written, a thread's name, the three PCI grants' zeroing, a device acknowledgement and the log cursor's two reads. Each now answers the copy's own `SyscallError` (`BadAddress` or `Io`), as `user_ptr` makes it. A window refused for its bounds stays `BadAddress`. Co-Authored-By: Claude Opus 5.5 --- kernel/src/object/ops.rs | 6 ++++-- kernel/src/syscall/device.rs | 12 ++++++------ kernel/src/syscall/dispatch.rs | 20 ++++++++++---------- kernel/src/syscall/machine.rs | 9 ++++++--- kernel/src/syscall/paged.rs | 2 +- kernel/src/syscall/vm.rs | 2 +- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index dc1ffedad6..67edd27644 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -541,8 +541,10 @@ fn write_device(claim: &DeviceClaim, buf: &UserBytes) -> u64 { return SyscallError::PermissionDenied.to_u64(); } let mut word = [0u8; 4]; - if buf.len() == word.len() && buf.read_at(0, &mut word).is_err() { - return SyscallError::BadAddress.to_u64(); + if buf.len() == word.len() { + if let Err(faulted) = buf.read_at(0, &mut word) { + return SyscallError::from(faulted).to_u64(); + } } let acknowledges = buf.len() == word.len() && u32::from_ne_bytes(word) == ack; let pid = crate::process::current_process(); diff --git a/kernel/src/syscall/device.rs b/kernel/src/syscall/device.rs index a40ade251f..e8770ea069 100644 --- a/kernel/src/syscall/device.rs +++ b/kernel/src/syscall/device.rs @@ -263,8 +263,8 @@ pub(super) fn sys_device_firmware_window(ctx: &SyscallContext, handle: RawHandle // Written first, before anything is opened, so a window the caller cannot // write as the call begins leaves no grant it was never told of. One a // sibling unmaps meanwhile can; the claim's release takes it back. - if out.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = out.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let mut grant = toyos_abi::pci::WindowGrant { shm: toyos_abi::HANDLE_INVALID, refused: 0, offset: 0, len: 0 }; match on_function(&claim, |function| crate::pcidev::firmware_window(function, window)) { @@ -304,8 +304,8 @@ pub(super) fn sys_device_dma_alloc( let Some(mut window) = ctx.user_bytes_mut(out, len) else { return SyscallError::BadAddress.to_u64(); }; - if window.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = window.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let granted = on_function(&claim, |function| crate::pcidev::dma_alloc(function, bytes)); let (memory, device_addr, granted) = match granted { @@ -363,8 +363,8 @@ pub(super) fn sys_device_dma_map( return SyscallError::BadAddress.to_u64(); }; // Written first, before anything is mapped, for `sys_device_dma_alloc`'s reason. - if window.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = window.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let mapped = on_function(&claim, |function| crate::pcidev::dma_map(function, &memory)); let (device_addr, bytes) = match mapped { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 070fcc4209..613716478e 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -108,7 +108,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_MACHINE_STOPS => sys_machine_stops(RawHandle(a1 as u32), a2), SYS_PIPE => sys_pipe(), SYS_SPAWN => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a1)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a1)) { Ok(args) => args, Err(e) => return e.to_u64() }; // First: each range's object is the caller's, and an endowment // below may move the caller's own handle to it. let program = match spawn_program(&ctx, &args) { @@ -304,13 +304,13 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } SYS_ACCEPT => sys_accept(RawHandle(a1 as u32), a2), SYS_MESSAGE_SEND => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a2)) { Ok(args) => args, Err(e) => return e.to_u64() }; let Ok(out) = outgoing(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; let Some(out) = out else { return bad_addr }; sys_message_send(RawHandle(a1 as u32), &out, args.flags & MESSAGE_NONBLOCK != 0) } SYS_MESSAGE_RECV => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a2)) { Ok(args) => args, Err(e) => return e.to_u64() }; let Ok(into) = incoming(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; let Some(mut into) = into else { return bad_addr }; sys_message_recv(RawHandle(a1 as u32), &mut into, args.flags & MESSAGE_NONBLOCK != 0) @@ -334,8 +334,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } let Some(bytes) = ctx.user_bytes(UserAddr::new(a2), len) else { return bad_addr }; let mut badge = [0u8; MAX_BADGE]; - if bytes.read_at(0, &mut badge[..len as usize]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut badge[..len as usize]) { + return SyscallError::from(faulted).to_u64(); } sys_port_mint(RawHandle(a1 as u32), &badge[..len as usize]) } @@ -481,8 +481,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> return bad_addr; }; let mut raw = [0u8; CPU_MASK_WORDS * 8]; - if bytes.read_at(0, &mut raw[..a3 as usize * 8]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut raw[..a3 as usize * 8]) { + return SyscallError::from(faulted).to_u64(); } let words: [u64; CPU_MASK_WORDS] = core::array::from_fn(|i| u64::from_le_bytes(raw[i * 8..][..8].try_into().unwrap())); @@ -498,7 +498,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> match sys_thread_get_affinity(a1) { Ok(words) => { let wrote = words.iter().enumerate().try_for_each(|(i, word)| out.write_at(i * 8, &word.to_le_bytes())); - wrote.map_or(bad_addr, |()| 0) + wrote.map_or_else(|faulted| SyscallError::from(faulted).to_u64(), |()| 0) } Err(e) => e.to_u64(), } @@ -515,8 +515,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> return bad_addr; }; let mut name = [0u8; process::THREAD_NAME_LEN]; - if bytes.read_at(0, &mut name[..len]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut name[..len]) { + return SyscallError::from(faulted).to_u64(); } process::set_current_thread_name(&name[..len]); 0 diff --git a/kernel/src/syscall/machine.rs b/kernel/src/syscall/machine.rs index 5d3e330e31..317c7b35ff 100644 --- a/kernel/src/syscall/machine.rs +++ b/kernel/src/syscall/machine.rs @@ -99,7 +99,7 @@ fn read_on_cursor( let words = |bytes: &[u8]| -> Vec { bytes.as_chunks::<8>().0.iter().map(|word| u64::from_le_bytes(*word)).collect() }; - let Ok(head) = ctx.user_vec(at, HEAD as u64) else { return SyscallError::BadAddress.to_u64() }; + let head = match ctx.user_vec(at, HEAD as u64) { Ok(head) => head, Err(e) => return e.to_u64() }; let mut head = CursorHead::from_words(words(&head).try_into().expect("the head's words")); head.shards = u32::try_from(rings).expect("`smp::size` bounds the CPUs below a u32"); let put = |head: CursorHead, positions: &[u64]| -> Result<(), SyscallError> { @@ -112,10 +112,13 @@ fn read_on_cursor( if head.slots as usize != rings { return put(head, &[]).err().unwrap_or(SyscallError::InvalidArgument).to_u64(); } - let Some(positions) = at.raw().checked_add(HEAD as u64).and_then(|p| ctx.user_vec(UserAddr::new(p), (8 * rings) as u64).ok()) - else { + let Some(after) = at.raw().checked_add(HEAD as u64) else { return SyscallError::BadAddress.to_u64(); }; + let positions = match ctx.user_vec(UserAddr::new(after), (8 * rings) as u64) { + Ok(positions) => positions, + Err(e) => return e.to_u64(), + }; let (count, walk) = match read(&words(&positions)) { Ok(read) => read, Err(e) => return put(head, &[]).err().unwrap_or(e).to_u64(), diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index dbbe960f22..802dda17c4 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -104,7 +104,7 @@ fn readable_holder( } pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args: UserAddr) -> u64 { - let Ok(args) = ctx.copy_in::(args) else { return SyscallError::BadAddress.to_u64() }; + let args = match ctx.copy_in::(args) { Ok(args) => args, Err(e) => return e.to_u64() }; let pager = match pager(h, Rights::WRITE) { Ok(pager) => pager, Err(refused) => return refused, diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index 1c24114b0f..e388e131c9 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -81,7 +81,7 @@ pub(crate) fn map_rights(prot: Prot) -> Rights { /// `toyos_abi::syscall::vm_map`: a memory object's own pages. pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) -> u64 { - let Ok(args) = ctx.copy_in::(args) else { return SyscallError::BadAddress.to_u64() }; + let args = match ctx.copy_in::(args) { Ok(args) => args, Err(e) => return e.to_u64() }; let asked = (|| { let prot = vm_prot(args.prot)?; let len = whole_pages(args.len).ok_or(SyscallError::InvalidArgument)?; From 93bfcc29b0960b8db1dd4331ba04aa0253b6efd1 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 22:29:01 +0200 Subject: [PATCH 19/34] Review r1 of #1269: the ABI says what a paged holder's mapping is, and the range count's control runs B2. `vm_map`'s contract gains a paged holder's: `Read` only, needing `READ`; a pager's process refused; a range past the object's end; a mapping counts as a holder; a touch of an unsupplied page waits and is reported overdue; a lost page or one wholly past the end ends the process with `Fault::Mapping`, and a copy from a lost one answers `Io`; `Gone` for an object already gone. `NOTICE_NO_HOLDERS` counts a mapping as a holder, the paged section says its objects are mapped too, and a withdraw, fail or shrink returns once no mapping reaches what it removed. B1. `ranges-less-relaxed` is registered in `build/ci.rs`'s `CONTROLS`, so `--ci host` runs it red and `every_model_control_is_run` holds. Co-Authored-By: Claude Opus 5.5 --- build/ci.rs | 3 +++ sdk/abi/src/syscall.rs | 23 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/build/ci.rs b/build/ci.rs index 468a0905d4..591cdb2632 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -587,6 +587,9 @@ pub(crate) const CONTROLS: &[Control] = &[ red(KERNEL_LOOM, "tlb-tags-resume-relaxed", Some("tlb_tags"), &[ Fails("a_change_passing_an_idle_holder_by_is_flushed_when_its_thread_resumes"), ]), + red(KERNEL_LOOM, "ranges-less-relaxed", Some("paged_ranges"), &[ + Fails("the_last_to_let_go_sees_every_leaf_cleared"), + ]), red(KERNEL_LOOM, "shootdown-served-relaxed", Some("counters_round"), &[ Fails("a_cpu_read_as_answered_is_read_with_its_answer"), ]), diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 31bd2316bc..edf7eaeeb1 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2126,7 +2126,23 @@ crate::user_safe! { /// last handle. [`SyscallError::Gone`] for such a window whose last handle was /// closed under the call: nothing is mapped. /// +/// **A paged holder** ([`paged_holder`]) maps its object's pages as its pager's +/// index holds them, [`VmProt::Read`] only, which needs [`Rights::READ`]; +/// any other protection is `InvalidArgument`, and a pager's own process is +/// refused `PermissionDenied`, as its holder reads are. The range may reach +/// past the object's end, where the object may grow. The mapping counts as a +/// holder of the object until it goes ([`NOTICE_NO_HOLDERS`]). A touch of a +/// page the pager has not supplied waits for it, as [`paged_read`] does, and +/// is reported when overdue, never ended by a timeout; a touch of a page the +/// pager failed, of any page once the object is released or the pager ends, +/// or of a page wholly past the object's end ends the process with +/// [`Fault::Mapping`], and a copy a call makes from such a page answers +/// [`SyscallError::Io`] (past the end, `BadAddress`). `Gone` for an object +/// released or a pager ended before the call. +/// /// [`Rights::MAP`]: crate::handle::Rights::MAP +/// [`Rights::READ`]: crate::handle::Rights::READ +/// [`Fault::Mapping`]: crate::end::Fault::Mapping /// [`Rights::WRITE`]: crate::handle::Rights::WRITE /// /// # Safety @@ -2893,7 +2909,7 @@ mod tests { // --------------------------------------------------------------------------- -// Paged objects: memory a pager supplies, read by its holders through the kernel. +// Paged objects: memory a pager supplies, read and mapped by its holders through the kernel. // --------------------------------------------------------------------------- /// The most pages one write-back range covers. @@ -2901,6 +2917,8 @@ pub const PAGED_RANGE_PAGES: u64 = 512; /// What a pager asks of one of its objects through [`SYS_PAGED_OP`], each /// with a [`PagedArgs`] naming the object by the id [`paged_create`] answered. +/// A withdraw, a failure and a shrink return only once no mapping reaches a +/// page they removed: every mapper's leaves of it are cleared. #[repr(u64)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PagedOp { @@ -2990,7 +3008,8 @@ crate::user_safe! { /// A reader waits for pages of the object. pub const NOTICE_REQUESTED: u64 = 1; -/// The object's last holder handle went: it may be released. +/// The object's last holder went, a handle or a mapping ([`vm_map`]): it may be +/// released. pub const NOTICE_NO_HOLDERS: u64 = 2; /// Make this process's one pager, whose objects' pages are at most From b78b476ccab5389345cf3357886822873a7bcaea Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 22:29:01 +0200 Subject: [PATCH 20/34] Review r1 of #1269: osbench's map group, with the threads and 255-mapper rows, and a guest row for it The orchestrator's ruling: the owed S4 and S5 rows are written now, so the batched main boot can run them. The mapping rows move from `files` into a `map` group of their own (in `all`'s order after `files`, a job of the benchcase image): `touch_16m`, `read_4k`, `threads_touch_16m` (eight threads faulting one mapping, each its eighth) and `truncate_255` (a 1 MiB file 255 `osbench map-hold` helpers map and touch whole, truncated to nothing, the truncate alone timed). `osbench_map_group_exits_0` runs the group in a guest and requires each row measured; what they measure is owed on metal. B6: `map_file` sits above `unmap`'s safety contract again, which names it. S5: the QEMU row matches the overdue report of the probe's own stalled run, pages 16+16, not any pager's. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/files.rs | 143 ++++++++++++++++++++++++--- tests/guest/osbench/src/main.rs | 8 +- tests/guest/osbench/src/sys/linux.rs | 6 +- tests/guest/osbench/src/sys/toyos.rs | 6 +- tests/images/benchcase.toml | 2 + tests/toyos.rs | 45 ++++++++- 6 files changed, 183 insertions(+), 27 deletions(-) diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index bc2e507613..3deb6461b4 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -117,31 +117,73 @@ fn parts( each(&name("pread_8"), 100, 2000, fields, |i| io("pread", f.read_at(&mut buf[..8], page_at(i, pages))).map(drop)); } -/// The 16 MiB file `big`, its pages cached, mapped read-only and shared. -/// Mapped, each page touched once and unmapped, so each touch is a fault -/// unless fault-around mapped its page already: 20 after 2. Then 4 KiB copied -/// out of a page of one mapping every page of which was touched: 2,000 after -/// 100. +/// The `map` group under `target`: files written there, so their pages are +/// cached, mapped read-only and shared, each line naming what served them. +/// The 16 MiB file is mapped, each page touched once and unmapped, so each +/// touch is a fault unless fault-around mapped its page already +/// (`touch_16m`, 20 after 2); the same by eight threads, each its eighth of +/// the pages (`threads_touch_16m`, 10 after 1); and 4 KiB copied out of a +/// page of one mapping every page of which was touched (`read_4k`, 2,000 +/// after 100). A 1 MiB file mapped and touched whole by 255 other processes +/// is truncated to nothing, which takes its pages from every one +/// (`truncate_255`, 5, the truncate alone timed). +pub fn map_suite(target: &Target) { + condition(&format!("map.{}.served_by", target.name), &target.served_by); + let scratch = match Scratch::new(&target.dir) { + Ok(s) => s, + Err(e) => return refuse(&format!("map.{}", target.name), format!("no scratch directory under {}: {e}", target.dir)), + }; + let name = |op: &str| format!("map.{}.{op}", target.name); + let big = scratch.at("big"); + if let Err(e) = fill(&big, 16 * MIB) { + return refuse(&name("setup"), e); + } + let fields: Vec<(&'static str, String)> = + [("file_bytes", (16 * MIB).to_string())].into_iter().chain(target.fields()).collect(); + mapped(&name, &fields, &big, 16 * MIB / 4096); + let _ = fs::remove_file(&big); + let held = scratch.at("held"); + let fields: Vec<(&'static str, String)> = + [("file_bytes", MIB.to_string()), ("mappers", MAPPERS.to_string())].into_iter().chain(target.fields()).collect(); + held_by_many(&name("truncate_255"), &fields, &held); + let _ = fs::remove_file(&held); +} + +/// Every page of `len` bytes at `at`, read once. +fn touch(at: usize, pages: std::ops::Range) -> u8 { + pages.fold(0u8, |sum, page| { + // SAFETY: a page of a read-only mapping the caller holds. + sum.wrapping_add(unsafe { (at as *const u8).add(page * 4096).read_volatile() }) + }) +} + fn mapped(name: &dyn Fn(&str) -> String, fields: &[(&'static str, String)], big: &Path, pages: usize) { let len = pages * 4096; - let touch = |at: *mut u8| (0..pages).fold(0u8, |sum, page| { - // SAFETY: a page of the mapping, read-only. - sum.wrapping_add(unsafe { at.add(page * 4096).read_volatile() }) - }); - each(&name("map_touch_16m"), 2, 20, fields, |_| { + each(&name("touch_16m"), 2, 20, fields, |_| { let at = crate::sys::map_file(big, len)?; - std::hint::black_box(touch(at)); + std::hint::black_box(touch(at as usize, 0..pages)); // SAFETY: the mapping just made, which nothing else names. unsafe { crate::sys::unmap(at, len) }; Ok(()) }); + each(&name("threads_touch_16m"), 1, 10, fields, |_| { + let at = crate::sys::map_file(big, len)? as usize; + std::thread::scope(|threads| { + for t in 0..THREADS { + threads.spawn(move || std::hint::black_box(touch(at, t * pages / THREADS..(t + 1) * pages / THREADS))); + } + }); + // SAFETY: the mapping just made, whose threads have all ended. + unsafe { crate::sys::unmap(at as *mut u8, len) }; + Ok(()) + }); let at = match crate::sys::map_file(big, len) { Ok(at) => at, - Err(why) => return refuse(&name("map_read_4k"), why), + Err(why) => return refuse(&name("read_4k"), why), }; - std::hint::black_box(touch(at)); + std::hint::black_box(touch(at as usize, 0..pages)); let mut buf = [0u8; 4096]; - each(&name("map_read_4k"), 100, 2000, fields, |i| { + each(&name("read_4k"), 100, 2000, fields, |i| { // SAFETY: a page of the mapping, every one of which is mapped. unsafe { core::ptr::copy_nonoverlapping(at.add(page_at(i, pages) as usize), buf.as_mut_ptr(), buf.len()) }; std::hint::black_box(&buf); @@ -151,6 +193,78 @@ fn mapped(name: &dyn Fn(&str) -> String, fields: &[(&'static str, String)], big: unsafe { crate::sys::unmap(at, len) }; } +/// The threads `threads_touch_16m` faults one mapping with. +const THREADS: usize = 8; +/// The processes holding `truncate_255`'s file mapped. +const MAPPERS: usize = 255; + +/// `held`, 1 MiB, mapped and touched whole by [`MAPPERS`] helpers of this +/// program (`osbench map-hold`), then truncated to nothing: the truncate's +/// time, five times, the file written again and the helpers started afresh +/// for each. +fn held_by_many(bench: &str, fields: &[(&'static str, String)], held: &Path) { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(e) => return refuse(bench, format!("no path to this program: {e}")), + }; + let mut ns = Vec::new(); + for _ in 0..5 { + let sample = (|| -> Result { + io("write", fill(held, MIB))?; + let mut helpers = Vec::with_capacity(MAPPERS); + for _ in 0..MAPPERS { + let mut helper = io( + "spawn", + std::process::Command::new(&exe) + .arg("map-hold") + .arg(held) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn(), + )?; + let mut said = [0u8; 5]; + io("read", helper.stdout.take().expect("piped").read_exact(&mut said))?; + if &said != b"held\n" { + return Err(format!("a helper said {said:?}")); + } + helpers.push(helper); + } + let file = io("open", OpenOptions::new().write(true).open(held))?; + let t = std::time::Instant::now(); + io("truncate", file.set_len(0))?; + let took = t.elapsed().as_nanos() as u64; + for mut helper in helpers { + drop(helper.stdin.take()); + let status = io("wait", helper.wait())?; + if !status.success() { + return Err(format!("a helper ended {status}")); + } + } + Ok(took) + })(); + match sample { + Ok(took) => ns.push(took), + Err(why) => return refuse(bench, why), + } + } + crate::report::result(bench, "ns", &mut ns, fields); +} + +/// `osbench map-hold `: the file at `path` mapped whole and every page +/// touched, said by "held", and held until stdin ends. +pub fn hold(args: &[String]) { + let path = Path::new(args.first().map(String::as_str).expect("osbench map-hold: a path")); + let len = fs::metadata(path).expect("osbench map-hold: the file").len() as usize; + let at = crate::sys::map_file(path, len).expect("osbench map-hold: the file mapped"); + std::hint::black_box(touch(at as usize, 0..len / 4096)); + let mut out = std::io::stdout(); + out.write_all(b"held\n").and_then(|()| out.flush()).expect("osbench map-hold: say held"); + let _ = std::io::stdin().read_to_end(&mut Vec::new()); + // SAFETY: the mapping made above; the file may have shrunk under it, and + // nothing touches it again. + unsafe { crate::sys::unmap(at, len) }; +} + /// Every benchmark under `target`, each line naming what served it and its /// medium. Each one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { @@ -237,7 +351,6 @@ pub fn suite(target: &Target) { } Err(e) => refuse(&name("read_4k_at"), e), } - mapped(&name, &with(&[("file_bytes", (16 * MIB).to_string())]), &big, pages); let _ = fs::remove_file(&big); // Listing a directory of 1,000 files, every name: 50 after 5. diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index cf184ad1b3..6886e6ddb6 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -130,18 +130,23 @@ fn files() { files::suite(&files::memory()); } +fn map() { + files::map_suite(&files::memory()); +} + /// Every group the machine alone runs, in the order `all` and the ToyOS job /// list run them: `boot` first, since it reads how long the boot took to /// reach it; and `crunch`, which heats it, last: ToyOS reads no temperature /// to cool down by, so a group after it would start at a clock its Linux run /// does not. -const GROUPS: [(&str, fn()); 16] = [ +const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), ("system", system), ("sched", sched), ("fair", fair), ("files", files), + ("map", map), ("ipc", ipc::suite), ("net", net::suite), ("procs", procs::suite), @@ -205,6 +210,7 @@ fn main() { "fair-job" => return fair::job(rest), "fair-helper" => return fair::helper(rest), "ipc-serve" => return ipc::serve(), + "map-hold" => return files::hold(rest), "cc" => return work::cc(rest), named => match GROUPS.iter().chain(&BY_NAME).find(|(name, _)| *name == named) { Some((name, run)) => group(name, *run), diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index 73c528b014..264875eaaa 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -58,9 +58,6 @@ pub fn map_anon(len: usize) -> *mut u8 { ptr.cast() } -/// # Safety -/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing -/// reaches it afterwards. /// `len` bytes of the file at `path` from its start, mapped read-only and /// shared. pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { @@ -74,6 +71,9 @@ pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { Ok(ptr.cast()) } +/// # Safety +/// `ptr` and `len` are one whole mapping from [`map_anon`] or +/// [`map_file`], and nothing reaches it afterwards. pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. assert_eq!(unsafe { libc::munmap(ptr.cast(), len) }, 0); diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 71374012e0..861bf24e04 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -102,9 +102,6 @@ pub fn huge_backing() -> String { "0KiB".into() } -/// # Safety -/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing -/// reaches it afterwards. /// `len` bytes of the file at `path` from its start, mapped read-only and /// shared: its file server's own cached pages. pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { @@ -120,6 +117,9 @@ pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { mapped.map_err(|e| format!("map: {e:?}")) } +/// # Safety +/// `ptr` and `len` are one whole mapping from [`map_anon`] or +/// [`map_file`], and nothing reaches it afterwards. pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. unsafe { syscall::vm_release(ptr, len) }.expect("vm_release of a whole mapping"); diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 1972b82cb3..09d19d4a9c 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -78,6 +78,7 @@ args = [ "osbench_sched", "osbench_fair", "osbench_files", + "osbench_map", "osbench_ipc", "osbench_net", "osbench_procs", @@ -103,6 +104,7 @@ args = [ "bin/osbench_sched" = "/system/bin/osbench" "bin/osbench_fair" = "/system/bin/osbench" "bin/osbench_files" = "/system/bin/osbench" +"bin/osbench_map" = "/system/bin/osbench" "bin/osbench_ipc" = "/system/bin/osbench" "bin/osbench_net" = "/system/bin/osbench" "bin/osbench_procs" = "/system/bin/osbench" diff --git a/tests/toyos.rs b/tests/toyos.rs index 3e3ab30d8c..99c703a68f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -665,6 +665,12 @@ const MACHINE_TESTS: &[MachineTest] = &[ // endowments a job gets is the image's and the supervisor's, which no // host build has, and the benches run osbench only nightly. ("osbench_capability_groups_exit_0", |_| osbench_capability_groups_exit_0()), + // `osbench map` on tests/images/benchcase.toml with only that job: its + // rows over a mapped file, eight threads faulting one mapping and 255 + // processes holding one while it is truncated, each measured or refused + // by name. Whether they measure correctly is a guest's to say; what they + // measure is owed on metal (the memory track record). + ("osbench_map_group_exits_0", |_| osbench_map_group_exits_0()), // What the kernel reads and writes for the `acpi` claim's holder and what // it refuses. A red here is a write the kernel made — to RAM, to the // firmware's tables, to COM1, to `PM1a_CNT`, to a function's configuration @@ -3460,6 +3466,32 @@ fn osbench_capability_groups_exit_0() -> Result<(), String> { Ok(()) } +/// `osbench map` on tests/images/benchcase.toml alone: it exits 0, which it +/// does only once every row it ran measured, and says each row's name. +fn osbench_map_group_exits_0() -> Result<(), String> { + const JOB: &str = "osbench_map"; + let committed = compile::repo_root().join("tests/images/benchcase.toml"); + let text = fs::read_to_string(&committed).map_err(|e| format!("{}: {e}", committed.display()))?; + let derived = toyos_metal::image::derive(&text, 600_000, &[JOB], &[]).map_err(|why| why.to_string())?; + let config = common::lane::dir().join("osbench-map-group.toml"); + fs::write(&config, derived).map_err(|e| format!("{}: {e}", config.display()))?; + let mut qemu = QemuInstance::boot_with_options( + &config, + &[], + &[], + BootOptions { ready_marker: bootlog::COMPLETE, ..Default::default() }, + ); + let mut console = format!("{}\n", qemu.boot_log()); + let end = format!("===TEST_END {JOB} "); + await_guest(&mut qemu, &mut console, "osbench's map group to end", |said| said.contains(&end))?; + let said = serial::Serial::named("osbench map", console); + said.must_say(&format!("===TEST_END {JOB} exit=0==="))?; + for row in ["touch_16m", "threads_touch_16m", "read_4k", "truncate_255"] { + said.must_say(&format!(".{row} unit="))?; + } + Ok(()) +} + /// Boot the image `--console-boot` builds, on the machine shape with a panel /// for `/system/bin/console` to claim, and wait for what that image owes at /// boot: the console up on its panel, having read the log and the keyboard, @@ -3600,7 +3632,7 @@ const MAPPED_FILE_SAID: &[&str] = &[ "mapprobe: a range taken away under a parked thread is nothing when its page comes", "mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing", "mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room", - "mapprobe: a page the object shrank past is nothing to its mapper", + "mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS", "mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch", "mapped_file: a mapped paged object follows its pager", ]; @@ -3623,10 +3655,13 @@ fn a_mapped_file_follows_its_pager(test_config: &Path) -> Result<(), String> { if let Some(claim) = MAPPED_FILE_SAID.iter().find(|claim| !result.stdout.contains(*claim)) { return Err(format!("the job never said {claim:?}:\n{}", result.stdout)); } - for report in ["'s pager has a read overdue: object ", "'s pager has no read overdue"] { - if !result.serial.contains(report) { - return Err(format!("the kernel never said {report:?}:\n{}", result.serial)); - } + // The probe's own stalled run, pages 16 to 32, not any pager's slow read. + let overdue = |line: &str| line.contains("'s pager has a read overdue: object ") && line.contains(" pages 16+16, "); + if !result.serial.lines().any(overdue) { + return Err(format!("the kernel never said the probe's run was overdue:\n{}", result.serial)); + } + if !result.serial.contains("'s pager has no read overdue") { + return Err(format!("the kernel never said the overdue wait ended:\n{}", result.serial)); } Ok(()) } From 112eca8d972c750e0d47a0ac962ad5ed7366102c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 22:29:01 +0200 Subject: [PATCH 21/34] Review r1 of #1269: the records say what step 2 owes and departs from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B4. The memory track's step 2 bullets name every boot and group owed on metal and what main must show: the shared boots with `mapped_file` and the pager-end test, the A/B of the groups the fault and copy paths run under with the process-exit cost every end now pays, and the `map` group beside Linux with each row's ratio. It records the pager-wide wakes (a departure from §3/S4) and the revocation visit's cost with the rows that price them, names step 3 as R4's, keeps the cold DATA miss as a row still to write, and records the pinned-copy test owed with #1258. libc's `msync` reason is what is true: a file is mapped read-only only. Co-Authored-By: Claude Opus 5.5 --- ...bc-refuses-what-toyos-cannot-yet-answer.md | 3 +- ...b-pages-still-copies-each-programs-text.md | 59 ++++++++++++------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index 63fcf431a4..ea5e7585cb 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -17,7 +17,8 @@ These answer failure in their POSIX form and do nothing LLVM's `getDefaultStackSize` (`llvm/lib/Support/ProgramStack.cpp`) ignores the failure and reads the `rlimit` it did not get. - `gethostname`, `ENOSYS`: no host name is published to a process. -- `msync`, `ENOSYS`: no mapping is of a file, so there is nothing to write back. +- `msync`, `ENOSYS`: a file is mapped read-only only, so no mapping holds + anything to write back; it answers once 7b's step 5 maps files writable. - `mmap` of a file other than `PROT_READ`, `ENODEV`: the kernel maps a file's pages read-only and shared, and a private one shares them until it is written, which waits for 7b's write faults (step 5); a private copy range of a file diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index f8b6ffbeb3..e4aa5cada2 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -97,10 +97,11 @@ live split of a user leaf. A shared segment's mapping is never widened, so a program can no longer make its own read-only segment writable (`mprotect`). Exit for that: a protect that adds write to such a range commits its pages and makes it a - copy range of the same object (7b's R4, which step 2 did not build: it - maps a paged object read-only only, and its range refuses a protect past - that); only a page shared until its first write waits for step 5's write - faults. Exit: + copy range of the same object (7b's R4, which step 3 builds, since a + program's writable segment mapped from its image is a copy range with a + paged source; step 2 maps a paged object read-only only, and its range + refuses a protect past that); only a page shared until its first write + waits for step 5's write faults. Exit: two processes of one binary hold its text once, wherever it is served from. 6. **`/apps` image sharing.** Decided by the orchestrator under the owner's @@ -121,7 +122,7 @@ live split of a user leaf. mapped where the caller says, an immutable one shared and executable, which is stage 5 for `/system`: built; (2) a paged object mapped read-only and shared: faults that park on the pager, reverse records, revocation: built, -but for R4's copy ranges of a paged object, which item 5 owes; (3) a +but for R4's copy ranges of a paged object, which step 3 builds (item 5); (3) a spawn maps its image from any pager-backed volume (stage 6), sealed against writes in place while it runs; (4) the storage track's attributes and names without a request; (5) writable shared @@ -141,23 +142,41 @@ mappings and write-back; (6) the #1077 retry that makes it faster is not measured on metal. - **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: the whole `testcases` boot, whose `mapped_file` member maps a paged object from a - second process (faults that wait, revocation, refusals, kills), and the - `shared-debug` and `proctreecase` boots. Main must show every job of each - boot exit 0. + second process (faults that wait, revocation, refusals, kills) and whose + `a_pager_s_end_answers_its_waiting_holders_gone` runs only there and + exercises the rewritten pager end, and the `shared-debug` and + `proctreecase` boots. Main must show every job of each boot exit 0. - **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's - `fs`, `pipe`, `ipc`, `mem` and `proc` groups, main before #1269 against - main after, four boots a side. Main after must be no slower, since the - fault and copy paths changed under all of them. + `system` (its pipe rows), `files`, `ipc`, `procs` and `mem` groups, main + before #1269 against main after, four boots a side. Main after must show no row + worse than noise: every fault and pinned copy changed, and every process + exit now empties its space through the full clear (futex bucket walks, + shootdowns, table unlinks) where its drop skipped them. - **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's - `fs.*.map_touch_16m` and `fs.*.map_read_4k` (a cached file's pages faulted - through a mapping, and 4 KiB read through one) on main, beside Linux's - `mmap` of the same file. Main must show both rows on every file target; - how a mapping compares with Linux's is not measured on metal. - -Rows step 2 did not write, owed by map7b: a cold DATA page's miss in stages, -one object mapped by 255 children and withdrawn (a revocation's cost grows -with its mappers), and many threads cold-faulting one large file; each exits -as an osbench row judged against Linux on metal. + `map` group on main beside Linux's `mmap` of the same files, recorded with + each row's ratio to Linux's: `touch_16m` and `threads_touch_16m` (a cached + file's pages faulted through a mapping, by one thread and by eight), + `read_4k` (4 KiB read through one) and `truncate_255` (a file 255 + processes map, truncated). Their correctness runs in a guest + (`osbench_map_group_exits_0`); what they measure is unmeasured until then. + +What these rows price: fault waiters wake on their pager's watch, so a +supply, a write's end or a removal wakes every fault waiter of that pager and +not only those its pages meet, a departure from the design's per-request +wakes (§3, S4) that `threads_touch_16m` on a file past its server's capacity +prices; and a revocation visits each mapper once per batch of 64 revoking +pages, each batch read by walking the object's resident pages under its +pager's lock, which `truncate_255` prices. Exit for both: per-request wakes +or a cheaper visit where those rows show the cost. A cold DATA page's miss in +stages is not yet an osbench row (owner map7b; exit: such a row on DATA, +judged against Linux on metal). + +The pinned-copy mode (a copy holding a page pin stops at an absent page, lets +its pins go, waits holding nothing and copies again) is reached by no process +until #1258's holder write, which copies from user memory under a write pin. +Its guest test, a holder write sourced from a mapped file's absent page that +completes, lands with whichever of #1258 and #1269 lands second (owner +map7b). Step 2 measures a cached page's fault and a 4 KiB read through a mapping against Linux (osbench `fs.*.map_touch_16m`, `fs.*.map_read_4k`). Owed, by From 5eadbcd0a4eb220043896ee7001ce13beff93764 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 23:00:19 +0200 Subject: [PATCH 22/34] Review r1 of #1269: the source gate counts MapNode's two Send and Sync, and the loom model clones once The full host suite found two defects step 2 brought in. `MapNode`'s hand-written `Send` and `Sync` (its link is reached only under its pager's lock) were not declared in `build/sourcegate.rs`'s table of them, so `every_hand_written_auto_trait_impl_is_declared` was red. And `paged_ranges.rs` cloned its counts and leaf for a second thread that could take the originals, which the workspace's clippy denies as a redundant clone. Co-Authored-By: Claude Opus 5.5 --- build/sourcegate.rs | 2 +- kernel/loom/tests/paged_ranges.rs | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/build/sourcegate.rs b/build/sourcegate.rs index a705b0d78f..6cf1643777 100644 --- a/build/sourcegate.rs +++ b/build/sourcegate.rs @@ -137,7 +137,7 @@ const AUTO_TRAIT_IMPLS: &[(&str, usize)] = &[ ("kernel/src/drivers/panic_console/mod.rs", 3), ("kernel/src/arch/x86_64/hw.rs", 1), ("kernel/src/mm/mmio.rs", 2), - ("kernel/src/object/paged.rs", 1), + ("kernel/src/object/paged.rs", 3), ("kernel/src/sched/driver.rs", 2), ]; diff --git a/kernel/loom/tests/paged_ranges.rs b/kernel/loom/tests/paged_ranges.rs index d97cbcd3cb..a0f9922d4b 100644 --- a/kernel/loom/tests/paged_ranges.rs +++ b/kernel/loom/tests/paged_ranges.rs @@ -55,14 +55,11 @@ fn the_last_to_let_go_sees_every_leaf_cleared() { } }) }; - let parked = { - let (ranges, leaf) = (ranges.clone(), leaf.clone()); - loom::thread::spawn(move || { - if ranges.less() { - unlink(&leaf); - } - }) - }; + let parked = loom::thread::spawn(move || { + if ranges.less() { + unlink(&leaf); + } + }); unmap.join().unwrap(); parked.join().unwrap(); }); From 5513d279c536b5609ec5de98a0472afc39e7c8f9 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 00:00:46 +0200 Subject: [PATCH 23/34] Review r2 of #1269: the walk reads its own set of revoking pages, one lock at a time, and a page past the end is a variant of its own B3. Each object keeps the walk's set: its revoking slots in a list, appended when a page is marked and emptied only by the walk's end, which now gives out the set's frames and nothing else (O(R), not O(S)). The walk splits into `next_visit` and `finish`. A visit reads the set a batch of 64 at a time by a cursor, under the pager's lock alone, drops it, and has the space clear and shoot down that batch's leaves under the space's lock alone (`Space::clear_paged`): one lock at a time, and a one-page truncate of a large file costs one page's work. B8, in part. After every mapper's visit and before any frame goes back, the walk ends every futex wait on a frame of its set, read under the pager's lock and ended outside it; the visit's own per-leaf revocation is gone. A page marked during that sweep is not yet covered here; the next commits' rule is. B7. The page a pager names is placed by `vm::page_at`, whose every step is checked: a page near the last a pager may name, offset 0, is no place rather than an overflow. Its unit test caught the first form, whose `then_some` added before the guard. A touch past a mapped object's end is `Missing::PastEnd`, `Populated::PastEnd` and `Faulted::PastEnd`, not a `Why` two consumers had to test: a touch ends the process with its record, and a copy, and now a futex word, answers `BadAddress`, since nothing is there. The futex word's comment says how a mapped file's frame keeps the futex contract. The pure model's visit is the kernel's rule with batches of one. `mapprobe`'s `shrunk` case shrinks an object of 80 pages, every one touched by two mappers, to one: the 79 revoked pages take two batches, and the mappers touch the first and the last of them, each ended `Fault::Mapping`, whichever batch cleared it. The harness names the map row's QEMU machine and says each of its rows is required measured. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 181 ++++++++++++++--------- kernel/pure/vm.rs | 22 +++ kernel/src/arch/aarch64/trap.rs | 2 +- kernel/src/arch/x86_64/idt/exceptions.rs | 2 +- kernel/src/mm/space.rs | 81 ++++------ kernel/src/object/paged.rs | 119 +++++++++++---- kernel/src/process.rs | 8 +- kernel/src/user_ptr.rs | 2 +- tests/guest/mapprobe/src/main.rs | 44 ++++-- tests/toyos.rs | 10 +- 10 files changed, 300 insertions(+), 171 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index b17c1e9239..d329743920 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -262,6 +262,14 @@ pub enum Supplied { Present, } +/// Where a reader of an object's revoking pages has read to +/// ([`Pages::revoking`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Cursor { + Start, + After(u32), +} + /// What a revoking page becomes once no mapping reaches its frame, by the /// object as the walk's end finds it. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -334,6 +342,8 @@ struct Slot { revoking: Option, prev: u32, next: u32, + /// The next of its object's revoking slots, in the order they were marked. + revoke_next: u32, } /// One object's part of the table: its size, whether its pager ended, the @@ -351,8 +361,11 @@ pub struct Object { writing: u32, /// A call that did not take all it was given left its write under way. held: bool, - /// Its slots that are revoking. + /// Its slots that are revoking, and the walk's set of them: a list in + /// the order they were marked, which only the walk's end empties. revoking: usize, + revoke_head: u32, + revoke_tail: u32, /// The pass of the walk over its mappers, moved at each pass's start. epoch: u64, walk: Walk, @@ -548,6 +561,8 @@ impl Pages { writing: 0, held: false, revoking: 0, + revoke_head: NONE, + revoke_tail: NONE, epoch: 0, walk: Walk::Idle, finished: 0, @@ -600,7 +615,7 @@ impl Pages { if next != NONE { self.slot_mut(next).prev = i; } - self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, revoking: None, prev: NONE, next }); + self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, revoking: None, prev: NONE, next, revoke_next: NONE }); o.head = i; o.pages += 1; let mut at = self.hash(o.id, page); @@ -690,18 +705,22 @@ impl Pages { let counted = s.state.counts_dirty(); s.state = State::CLEAN; s.revoking = Some(becomes); + s.revoke_next = NONE; o.revoking += 1; if counted { self.dirty_held -= 1; } + match o.revoke_tail { + NONE => o.revoke_head = i, + tail => self.slot_mut(tail).revoke_next = i, + } + o.revoke_tail = i; } /// Visit `o`'s next mapper after pages were marked revoking /// ([`Object::revoke`]), or, once every mapper has been visited since the /// last marking, give out each revoking page's frame, its page and what it - /// became, and answer `None`. A page that ends absent was withdrawn, - /// perhaps after a supply for a waiter's notice, without moving the - /// version: every want of `waiters` it meets is named again. + /// became, and answer `None`: [`Self::next_visit`], then [`Self::finish`]. pub fn walk( &mut self, o: &mut Object, @@ -709,6 +728,17 @@ impl Pages { waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked), ) -> Option { + let next = self.next_visit(o, mappers); + if next.is_none() { + self.finish(o, waiters, out); + } + next + } + + /// `o`'s next mapper to visit, a new pass begun where a removal queued + /// more pages meanwhile; `None` once every mapper has been visited since + /// the last marking, when [`Self::finish`] is owed under the same hold. + pub fn next_visit(&mut self, o: &mut Object, mappers: &mut M) -> Option { loop { if let Some(node) = mappers.head().filter(|&n| mappers.seen(n) != o.epoch) { mappers.visited(node, o.epoch); @@ -719,56 +749,59 @@ impl Pages { o.epoch += 1; o.walk = Walk::Running { again: false }; } - Walk::Running { again: false } => break, + Walk::Running { again: false } => return None, Walk::Idle => panic!("paged: a walk of object {} that never began", o.id), } } + } + + /// The walk's end, once [`Self::next_visit`] answered `None`: each + /// revoking page leaves, its frame given out with its page and what it + /// became. A page that ends absent was withdrawn, perhaps after a supply + /// for a waiter's notice, without moving the version: every want of + /// `waiters` it meets is named again. + pub fn finish(&mut self, o: &mut Object, waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked)) { + assert_eq!(o.walk, Walk::Running { again: false }, "paged: object {}'s walk ended with a pass owed", o.id); let held = !o.gone; - let mut at = o.head; + let mut at = core::mem::replace(&mut o.revoke_head, NONE); + o.revoke_tail = NONE; while at != NONE { let s = self.slot(at); - let (next, page, revoking) = (s.next, s.page, s.revoking); - if let Some(becomes) = revoking { - let frame = self.remove(o, at).expect("paged: a revoking page holds its frame"); - let becomes = if held && page < o.page_end() { becomes } else { Revoked::Removed }; - match becomes { - Revoked::Failed => self.insert(o, page, None, State::CLEAN).expect("paged: the slot just freed"), - Revoked::Absent => waiters.each(&mut |want| { - want.withdrawn(page, 1); - }), - Revoked::Removed => {} - } - out(frame, page, becomes); + let (next, page, becomes) = (s.revoke_next, s.page, s.revoking.expect("paged: the walk's set holds revoking slots")); + let frame = self.remove(o, at).expect("paged: a revoking page holds its frame"); + let becomes = if held && page < o.page_end() { becomes } else { Revoked::Removed }; + match becomes { + Revoked::Failed => self.insert(o, page, None, State::CLEAN).expect("paged: the slot just freed"), + Revoked::Absent => waiters.each(&mut |want| { + want.withdrawn(page, 1); + }), + Revoked::Removed => {} } + out(frame, page, becomes); at = next; } debug_assert_eq!(o.revoking, 0); o.walk = Walk::Idle; o.finished = o.epoch; - None } - /// `o`'s revoking pages at or past `from`, the lowest first, with the - /// frame each keeps until its walk ends: as many as fit in `out`, whose - /// count this answers. A walk's visit clears a mapper's leaves of them a - /// batch at a time, asking again from past the last. - pub fn revoking_from(&self, o: &Object, from: u64, out: &mut [(u64, F)]) -> usize { + /// `o`'s revoking pages from `at` on, in the order they were marked, each + /// with the frame it keeps until the walk's end: as many as fit in `out`, + /// whose count this answers, `at` moved past them. Only the walk's end + /// takes a slot off the set, so a cursor stays good across the drops of + /// the pager's lock between batches, and sees pages marked since. + pub fn revoking(&self, o: &Object, at: &mut Cursor, out: &mut [(u64, F)]) -> usize { + let mut next = match *at { + Cursor::Start => o.revoke_head, + Cursor::After(i) => self.slot(i).revoke_next, + }; let mut n = 0; - let mut at = o.head; - while at != NONE { - let s = self.slot(at); - at = s.next; - let (Some(_), Some(frame)) = (s.revoking, s.frame) else { continue }; - if s.page < from || (n == out.len() && s.page >= out[n - 1].0) { - continue; - } - let mut k = n.min(out.len() - 1); - n = (n + 1).min(out.len()); - while k > 0 && out[k - 1].0 > s.page { - out[k] = out[k - 1]; - k -= 1; - } - out[k] = (s.page, frame); + while n < out.len() && next != NONE { + let s = self.slot(next); + out[n] = (s.page, s.frame.expect("paged: a revoking page holds its frame")); + *at = Cursor::After(next); + next = s.revoke_next; + n += 1; } n } @@ -1109,32 +1142,38 @@ mod tests { } } - /// A removed page reads as absent, takes no second frame and no write, - /// and keeps its frame until every mapper of its object was visited; - /// then its frame goes out once and the page is supplied afresh. + /// The walk's set holds each revoking page once, in the order they were + /// marked; a cursor reads it a batch at a time and sees pages marked + /// between batches; the walk's end empties it. #[test] - fn revoking_pages_come_lowest_first_a_batch_at_a_time() { + fn the_walks_set_is_read_a_batch_at_a_time_in_marking_order() { let mut p = Pages::new(16, 4).unwrap(); let mut o = p.object(1, 16 * PAGE); for page in [9, 2, 14, 5, 0, 11, 7] { p.supply(&mut o, page, 100 + page, false, (page + 1) * PAGE).unwrap(); } - let (removed, _) = p.withdraw(&mut o, 2, 12, &mut Nobody); - assert_eq!(removed, 5, "pages 2, 5, 7, 9 and 11 are withdrawn"); - let mut batch = [(0, 0); 2]; - let mut seen = Vec::new(); - let mut from = 0; - loop { - let n = p.revoking_from(&o, from, &mut batch); - seen.extend_from_slice(&batch[..n]); - if n < batch.len() { - break; - } - from = batch[n - 1].0 + 1; + for page in [7, 2, 11] { + assert_eq!(p.withdraw(&mut o, page, 1, &mut Nobody).0, 1); } - assert_eq!(seen, [(2, 102), (5, 105), (7, 107), (9, 109), (11, 111)]); + let mut at = Cursor::Start; + let mut batch = [(0, 0); 2]; + assert_eq!(p.revoking(&o, &mut at, &mut batch), 2); + assert_eq!(batch, [(7, 107), (2, 102)]); + assert_eq!(p.withdraw(&mut o, 2, 1, &mut Nobody).0, 0, "a revoking page is marked once"); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(p.revoking(&o, &mut at, &mut batch), 2, "a page marked between batches is read"); + assert_eq!(batch, [(11, 111), (0, 100)]); + assert_eq!(p.revoking(&o, &mut at, &mut batch), 0); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), None); + assert_eq!(freed, [107, 102, 111, 100]); + assert_eq!(p.revoking(&o, &mut Cursor::Start, &mut batch), 0, "the walk's end empties its set"); } + /// A removed page reads as absent, takes no second frame and no write, + /// and keeps its frame until every mapper of its object was visited; + /// then its frame goes out once and the page is supplied afresh. #[test] fn a_revoking_page_keeps_its_one_frame_until_every_mapper_is_visited() { let mut p = Pages::new(8, 8).unwrap(); @@ -1185,7 +1224,7 @@ mod tests { let visits: Vec<_> = core::iter::from_fn(|| p.walk(&mut o, &mut ring, &mut Nobody, &mut out)).collect(); assert_eq!(visits, [11, 10, 12, 11], "the first pass ends at 11, the second visits all three"); assert!(o.finished(second)); - assert_eq!(freed, [(8, 1, Revoked::Failed), (7, 0, Revoked::Absent)]); + assert_eq!(freed, [(7, 0, Revoked::Absent), (8, 1, Revoked::Failed)], "in marking order"); assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed)); } @@ -1220,7 +1259,7 @@ mod tests { assert_eq!(o.revoke(), Revoke::Walk); let mut ended = Vec::new(); assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), None); - assert_eq!(ended, [(8, 1, Revoked::Removed), (7, 0, Revoked::Failed)]); + assert_eq!(ended, [(7, 0, Revoked::Failed), (8, 1, Revoked::Removed)]); assert_eq!((p.lookup(&o, 0), p.used()), (Lookup::Failed, 1)); } @@ -1874,7 +1913,9 @@ mod tests { next: u64, freed: BTreeSet, walking: bool, - visiting: Option, + /// The space a walk visits, and where its visit has read the + /// object's revoking pages to. + visiting: Option<(u32, Cursor)>, /// Faults parked on an absent page: the space, the page, and the /// request as the pager hears of it. parked: std::vec::Vec<(u32, u64, Want, Version)>, @@ -1965,7 +2006,7 @@ mod tests { assert!(freed.insert(f), "frame {f} given back twice"); }); match walked { - Some(node) => self.visiting = Some(node), + Some(node) => self.visiting = Some((node, Cursor::Start)), None => self.walking = false, } } @@ -2010,14 +2051,18 @@ mod tests { } } } + // The kernel's visit, a batch of one: the next revoking + // page read, and the space's leaf of it cleared where it + // maps that frame; anything may run between batches. Step::Visit if self.visiting.is_some() => { - let s = self.visiting.take().unwrap() as usize; - for page in 0..2 { - if let Some(f) = self.leaves[s][page] { - if self.p.lookup(&self.o, page as u64) != Lookup::Present(f) { - self.leaves[s][page] = None; - } + let (s, mut at) = self.visiting.take().unwrap(); + let mut batch = [(0, 0)]; + if self.p.revoking(&self.o, &mut at, &mut batch) == 1 { + let (page, f) = batch[0]; + if self.leaves[s as usize][page as usize] == Some(f) { + self.leaves[s as usize][page as usize] = None; } + self.visiting = Some((s, at)); } } _ => return false, @@ -2127,6 +2172,8 @@ mod tests { Step::Answer, Step::Shrink, Step::Walk, + // A batch of one: the revoked page, then the read that finds none. + Step::Visit, Step::Visit, Step::Walk, Step::Grow, diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index a462a7ca04..024fcf668d 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -21,6 +21,14 @@ pub const PAGE: u64 = 4096; /// run of 64 KiB. pub const AROUND: u64 = 16 * PAGE; +/// Where a range at `[start, end)` that maps its object from byte `offset` +/// maps the object's `page`, if it does. A pager names the page, so every +/// step is checked and none overflows. +pub fn page_at(start: u64, end: u64, offset: u64, page: u64) -> Option { + let into = page.checked_mul(PAGE)?.checked_sub(offset)?; + (into < end - start).then(|| start + into) +} + /// The pages `[from, to)` a touch of the empty `page` of the range `[start, /// end)` populates: its own, and where a page beside it in the range is /// `populated`, every page of its aligned run of [`AROUND`] in the range. So a @@ -390,6 +398,20 @@ impl Inventory { #[cfg(test)] mod tests { use super::*; + + /// A page a range maps lands at its place; one before the range's + /// offset, at or past its end, or so high its byte offset overflows is + /// no place, never an overflow: a pager names the page. + #[test] + fn a_page_is_placed_only_inside_its_range() { + let (start, end) = (0x10_0000, 0x10_0000 + 4 * PAGE); + assert_eq!(page_at(start, end, 2 * PAGE, 2), Some(start)); + assert_eq!(page_at(start, end, 2 * PAGE, 5), Some(start + 3 * PAGE)); + assert_eq!(page_at(start, end, 2 * PAGE, 6), None, "the range's end"); + assert_eq!(page_at(start, end, 2 * PAGE, 1), None, "before the range's offset"); + assert_eq!(page_at(start, end, 0, (1 << 52) - 2), None, "a page near the last a pager may name"); + assert_eq!(page_at(start, end, 0, u64::MAX), None, "a byte offset past u64"); + } use std::vec; use std::vec::Vec; diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 2cd9a45f01..cd7aef57af 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -360,7 +360,7 @@ fn user_abort(frame: &mut Frame) { cpu::disable_interrupts(); match faulted { crate::process::Faulted::Unreachable => {} - crate::process::Faulted::Lost(lost) => { + crate::process::Faulted::Lost(lost) | crate::process::Faulted::PastEnd(lost) => { percpu::set_fault_state(CpuFaultState::Normal); crate::process::lost(frame.far, lost); } diff --git a/kernel/src/arch/x86_64/idt/exceptions.rs b/kernel/src/arch/x86_64/idt/exceptions.rs index 4c37252e69..423117689b 100644 --- a/kernel/src/arch/x86_64/idt/exceptions.rs +++ b/kernel/src/arch/x86_64/idt/exceptions.rs @@ -535,7 +535,7 @@ pub(super) fn page_fault_handler(frame: &mut TrapFrame) { if is_user { match process::handle_page_fault(fault_addr, access) { process::Faulted::Unreachable => {} - process::Faulted::Lost(lost) => { + process::Faulted::Lost(lost) | process::Faulted::PastEnd(lost) => { percpu::set_fault_state(percpu::CpuFaultState::Normal); process::lost(fault_addr, lost); } diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 3f37c95340..bc9a407dc7 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -254,7 +254,7 @@ impl FutexWait { let mut held = space.lock(); let phys = match held.reach(addr.raw(), Access::Read) { Populated::At(phys) => phys, - Populated::Unreachable => return Err(Untranslated::Unreachable), + Populated::Unreachable | Populated::PastEnd(_) => return Err(Untranslated::Unreachable), Populated::Lost(_) | Populated::Wait(_) => return Err(Untranslated::Away), }; // Counted under this hold: a change made before it is in `at`, @@ -555,55 +555,33 @@ impl AddressSpace { Ok(UserAddr::new(start)) } - /// The visit of a walk of `node`'s object: every leaf of this space's - /// ranges of it that maps a revoking page's frame is cleared and shot - /// down, and then every futex wait on those frames ends. The pager's lock - /// is held only to read a batch of the revoking pages, never across a - /// shootdown; a fault meanwhile waits on this space's lock and then finds - /// the page as the index has it. - pub fn revoke_paged(&mut self, node: &MapNode) { - const BATCH: usize = 64; - let mut from = 0; - loop { - let mut batch = [(0u64, 0u64); BATCH]; - let n = node.with_index(|index| index.revoking_from(from, &mut batch)); - let mut hit = [false; BATCH]; - let (mut lo, mut hi) = (u64::MAX, 0); - let tables = &mut self.tables; - for (start, entry) in self.inventory.meeting(FLOOR, TOP) { - let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; - if !core::ptr::eq(range.node(), node) { - continue; - } - for (k, &(page, phys)) in batch[..n].iter().enumerate() { - let Some(va) = (page * PAGE_SIZE).checked_sub(*offset).map(|into| start + into).filter(|&va| va < entry.end) else { - continue; - }; - tables.edit(va, va + PAGE_SIZE, |_, leaf| match leaf { - Leaf::Mapped(m) if m.phys == phys => { - hit[k] = true; - (lo, hi) = (lo.min(va), hi.max(va + PAGE_SIZE)); - Some(Leaf::Empty) - } - _ => None, - }); - } + /// One batch of a walk's visit of `node`'s object ([`MapNode`]): every + /// leaf of this space's ranges of it that maps one of `revoking`'s + /// `(page, frame)` is cleared, and the span shot down, under this space's + /// lock alone. The walk ends the futex waits on those frames once every + /// mapper is visited, before any goes back. A fault meanwhile waits on + /// this space's lock and then finds the page as the index has it. + pub fn clear_paged(&mut self, node: &MapNode, revoking: &[(u64, u64)]) { + let (mut lo, mut hi) = (u64::MAX, 0); + let tables = &mut self.tables; + for (start, entry) in self.inventory.meeting(FLOOR, TOP) { + let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; + if !core::ptr::eq(range.node(), node) { + continue; } - if lo < hi { - let _reached = tables.shoot_down(lo, hi, &Unlinked::none()); - let mut frames = [0u64; BATCH]; - let mut m = 0; - for (&(_, phys), _) in batch[..n].iter().zip(hit).filter(|&(_, hit)| hit) { - frames[m] = phys; - m += 1; - } - frames[..m].sort_unstable(); - crate::sched::futex::revoke_sorted(&frames[..m], |&phys| phys); - } - if n < BATCH { - return; + for &(page, phys) in revoking { + let Some(va) = vm::page_at(start, entry.end, *offset, page) else { continue }; + tables.edit(va, va + PAGE_SIZE, |_, leaf| match leaf { + Leaf::Mapped(m) if m.phys == phys => { + (lo, hi) = (lo.min(va), hi.max(va + PAGE_SIZE)); + Some(Leaf::Empty) + } + _ => None, + }); } - from = batch[n - 1].0 + 1; + } + if lo < hi { + let _reached = tables.shoot_down(lo, hi, &Unlinked::none()); } } @@ -1023,6 +1001,7 @@ impl AddressSpace { Err(Missing::Absent(since)) => return Populated::Wait(PageWait { range: range.clone(), page: of(page), since }), Err(Missing::Hole) => unreachable!("space: a fault's touch of a hole is filled"), Err(Missing::Lost(lost)) => return Populated::Lost(lost), + Err(Missing::PastEnd(record)) => return Populated::PastEnd(record), } } Kind::Private { committed: true } | Kind::Copy { .. } => { @@ -1061,9 +1040,11 @@ pub enum Populated { At(u64), /// No range allows the access there. Unreachable, - /// A paged page its pager failed or can no longer supply, or one wholly - /// past its object's end. + /// A paged page its pager failed or can no longer supply. Lost(Lost), + /// A paged page wholly past its object's end, with the record a touch + /// of it ends the process with. + PastEnd(Lost), /// A paged page that is absent: the thread waits for it, holding nothing, /// and touches it again. Wait(PageWait), diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 24255a650a..d4bb45bd7c 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -58,7 +58,7 @@ use core::ptr; use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::Stack; -use kernel::paged::{Core, Draw, Leave, Lookup, Mappers, Object, Pages, Refused, Revoke, Revoked, Supplied, Version, Waiters, Want, PAGE}; +use kernel::paged::{Core, Cursor, Draw, Leave, Lookup, Mappers, Object, Pages, Refused, Revoke, Revoked, Supplied, Version, Waiters, Want, PAGE}; use kernel::sched::task::WaitClass; use kernel::slots::Slots; use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; @@ -379,26 +379,38 @@ impl PagerObject { } let job = self.core.job(); state.entry_mut(slot).refs += 1; - let mut absent = false; + // Whether every futex wait on a revoking frame was ended since the + // last pass began: owed before any frame goes back. + let mut swept = false; loop { - let mut waiters = Linked { head: state.requests, slot }; - let State { pages, entries, held, gone, .. } = &mut *state; + let State { pages, entries, .. } = &mut *state; let Entry { object, mappers, .. } = entries.get_mut(slot).expect("paged: a walked slot is held"); - let next = pages.walk(object, &mut Ring(mappers), &mut waiters, &mut |phys, _, becomes| { - held.give_back(job, *gone, phys, true); - absent |= becomes == Revoked::Absent; - }); - let Some(node) = next else { break }; - // SAFETY: linked, so live, under the lock; a range holds it, so - // it upgrades, and the count kept holds it past its unlinking - // while the lock is dropped. - let node = unsafe { (*node).this.upgrade() }.expect("paged: a linked mapper is held by a range"); - drop(state); - if let Some(space) = node.space.upgrade() { - space.lock().revoke_paged(&node); + match pages.next_visit(object, &mut Ring(mappers)) { + Some(node) => { + swept = false; + // SAFETY: linked, so live, under the lock; a range holds + // it, so it upgrades, and the count kept holds it past + // its unlinking while the lock is dropped. + let node = unsafe { (*node).this.upgrade() }.expect("paged: a linked mapper is held by a range"); + state = self.visit(state, slot, &node); + } + None if !swept => { + state = self.end_futex_waits(state, slot); + swept = true; + } + None => break, } - state = self.state.lock(); } + // Every mapper visited and every wait ended, in this hold since + // `next_visit` said so: the frames go back. + let mut absent = false; + let mut waiters = Linked { head: state.requests, slot }; + let State { pages, entries, held, gone, .. } = &mut *state; + let object = &mut entries.get_mut(slot).expect("paged: a walked slot is held").object; + pages.finish(object, &mut waiters, &mut |phys, _, becomes| { + held.give_back(job, *gone, phys, true); + absent |= becomes == Revoked::Absent; + }); self.unref_locked(&mut state, slot); if absent { self.watch.post(); @@ -406,6 +418,54 @@ impl PagerObject { (state, None) } + /// A walk's visit of `node`'s space: the object's revoking pages read a + /// batch at a time under the pager's lock alone, each batch's leaves then + /// cleared and shot down under the space's lock alone + /// ([`Space::clear_paged`]). One lock at a time, and O(revoking pages). + fn visit<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32, node: &MapNode) -> LockGuard<'a, State> { + let mut at = Cursor::Start; + loop { + let mut batch = [(0u64, 0u64); REVOKE_BATCH]; + let n = state.pages.revoking(&state.entry(slot).object, &mut at, &mut batch); + drop(state); + if n > 0 { + let Some(space) = node.space.upgrade() else { return self.state.lock() }; + space.lock().clear_paged(node, &batch[..n]); + } + state = self.state.lock(); + if n < REVOKE_BATCH { + return state; + } + } + } + + /// Every futex wait on a revoking frame of the object at `slot` ends, + /// read a batch at a time under the pager's lock and ended outside it: + /// after every mapper's visit, so no leaf maps one any more and none is + /// waited on afresh, and before any frame goes back, so no waiter keyed + /// on one meets the frame reissued (`sched::futex::revoke_frames`). This + /// covers a leaf an unmap, `discard`, `protect` to none or a space's end + /// cleared before the frame was removed, whose clearing revokes no + /// frame it does not own. + fn end_futex_waits<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { + let mut at = Cursor::Start; + loop { + let mut batch = [(0u64, 0u64); REVOKE_BATCH]; + let n = state.pages.revoking(&state.entry(slot).object, &mut at, &mut batch); + drop(state); + let mut frames = [0u64; REVOKE_BATCH]; + for (frame, &(_, phys)) in frames.iter_mut().zip(&batch[..n]) { + *frame = phys; + } + frames[..n].sort_unstable(); + crate::sched::futex::revoke_sorted(&frames[..n], |&phys| phys); + state = self.state.lock(); + if n < REVOKE_BATCH { + return state; + } + } + } + /// The pager lets object `id` go, in one hold of the lock: it answers /// `Gone` from now on, its pages go, and its slot with its last holder. A /// second release, or one racing the pager's end, finds it released. @@ -1304,6 +1364,8 @@ pub enum Missing { Hole, /// Its pager cannot let a mapping have it. Lost(Lost), + /// Wholly past the object's end: nothing is there. + PastEnd(Lost), } /// A page a fault was refused, as the record of its end names it. @@ -1326,8 +1388,9 @@ pub enum Why { Ended, /// A hole of an anonymous object, and the pager has no room for its page. NoRoom, - /// The page lies wholly past the object's end, which POSIX answers a - /// touch of with `SIGBUS` and a copy with `EFAULT`. + /// The page lies wholly past the object's end: nothing is there, so a + /// touch of it ends the process and a copy from it is refused as a bad + /// address. PastEnd, } @@ -1358,20 +1421,13 @@ impl Index<'_> { _ if self.state.gone => Err(self.lost(page, Why::Ended)), Lookup::Present(phys) => Ok(phys), Lookup::Absent => Err(Missing::Absent(object.version())), - Lookup::PastEnd => Err(self.lost(page, Why::PastEnd)), + Lookup::PastEnd => Err(Missing::PastEnd(self.record(page, Why::PastEnd))), Lookup::Zero => Err(Missing::Hole), Lookup::Failed => Err(self.lost(page, Why::Failed)), Lookup::Gone => Err(self.lost(page, Why::Released)), } } - /// The object's revoking pages at or past `from`, lowest first, each - /// with the frame it keeps until its walk ends - /// ([`Pages::revoking_from`]). - pub fn revoking_from(&self, from: u64, out: &mut [(u64, u64)]) -> usize { - self.state.pages.revoking_from(&self.state.entry(self.slot).object, from, out) - } - /// As [`Self::page`], for the page a fault touched: a hole of an /// anonymous object, a MEMORY volume's, gets a zeroed frame of its /// pager's, installed clean and charged as a supply's is, and is refused @@ -1420,10 +1476,17 @@ impl Index<'_> { } fn lost(&self, page: u64, why: Why) -> Missing { - Missing::Lost(Lost { pager: self.pager.pid, object: self.slot, page, why }) + Missing::Lost(self.record(page, why)) + } + + fn record(&self, page: u64, why: Why) -> Lost { + Lost { pager: self.pager.pid, object: self.slot, page, why } } } +/// The revoking pages a walk reads under its pager's lock at once. +const REVOKE_BATCH: usize = 64; + /// The pages a fault's request names: the aligned run around its page. const FAULT_RUN: u64 = kernel::vm::AROUND / PAGE; diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 9a9e7fdcfb..81b3b1e364 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -977,6 +977,8 @@ pub enum Faulted { Unreachable, /// A mapped file's page its pager cannot let the process have. Lost(crate::object::paged::Lost), + /// A mapped file's page wholly past its end. + PastEnd(crate::object::paged::Lost), /// A mapped file's page that is absent, met by a copy holding page pins /// (`user_ptr::Pinned`), which does not wait for it. Absent, @@ -989,11 +991,10 @@ impl Faulted { /// What a copy this fault stopped short answers. pub fn stop(self) -> crate::user_ptr::Stop { match self { - // Nothing is there, as for a copy POSIX answers `EFAULT`. - Faulted::Lost(lost) if lost.why == crate::object::paged::Why::PastEnd => crate::user_ptr::Stop::Unreachable, Faulted::Lost(_) => crate::user_ptr::Stop::Lost, Faulted::Absent => crate::user_ptr::Stop::Absent, - Faulted::Served | Faulted::Unreachable | Faulted::Killed => crate::user_ptr::Stop::Unreachable, + // Nothing is there, so a copy answers `BadAddress`. + Faulted::Served | Faulted::Unreachable | Faulted::PastEnd(_) | Faulted::Killed => crate::user_ptr::Stop::Unreachable, } } } @@ -1031,6 +1032,7 @@ fn touch(va: u64, access: kernel::vm::Access, mut wait: impl FnMut(crate::mm::sp Populated::At(_) => break, Populated::Unreachable => return Faulted::Unreachable, Populated::Lost(lost) => return Faulted::Lost(lost), + Populated::PastEnd(record) => return Faulted::PastEnd(record), Populated::Wait(page) => match wait(page) { Faulted::Served => {} stopped => return stopped, diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index ff470f951c..85be05b8ef 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -60,7 +60,7 @@ pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Result Ok(phys), - Populated::Unreachable => Err(Untranslated::Unreachable), + Populated::Unreachable | Populated::PastEnd(_) => Err(Untranslated::Unreachable), Populated::Lost(_) | Populated::Wait(_) => Err(Untranslated::Away), } } diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs index 5896ef2d88..8412d354b2 100644 --- a/tests/guest/mapprobe/src/main.rs +++ b/tests/guest/mapprobe/src/main.rs @@ -16,8 +16,11 @@ //! parked in a write's copy from the mapping, which writes nothing. //! - `hole`: an unwritten page of an anonymous object reads zero with no //! request, and one the pager has no room for ends the mapper. -//! - `shrunk`: a touch of a page the object shrank past ends its mapper with -//! `SIGBUS`, as POSIX says of a page wholly past a mapped file's end. +//! - `shrunk`: an object of 80 pages, every one touched by two mappers, is +//! shrunk to one; a touch of the first page past the new end ends one +//! mapper and of the last the other, each with `SIGBUS`. The shrink revokes +//! 79 pages, more than a walk reads at once, so one of the two is cleared +//! by a later batch. //! - `ends`, last: the pager's end answers a write from the mapping `Io` //! and ends the mapper at its next touch. @@ -224,16 +227,26 @@ fn hole(served: &mut Served) { println!("mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room"); } +/// The pages `shrunk`'s object holds before its shrink: more than a walk's +/// batch, so its revocation takes two. +const SHRUNK: u64 = 80; + fn shrunk(served: &mut Served) { - let object = served.pager.object(4 * PAGE).expect("mapprobe: an object"); - served.supply(object, 0, 4, 0x66); - let (ready, theirs) = pipe(); - let (go, mine) = go_pipe(); - let mut child = served.mapper(object, "shrunk", vec![(READY, theirs), (GO, mine)]); - assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never read its last page"); + let object = served.pager.object(SHRUNK * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, SHRUNK, 0x66); + let mut mappers = Vec::new(); + for page in [1, SHRUNK - 1] { + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let child = served.mapper(object, &format!("shrunk:{page}"), vec![(READY, theirs), (GO, mine)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: a mapper never read every page"); + mappers.push((page, child, go)); + } served.pager.resize(object, PAGE).expect("mapprobe: a shrink"); - assert_eq!(go.write(&[1]), Ok(1)); - assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a touch past a shrunk object"); + for (page, mut child, go) in mappers { + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a touch of page {page} past a shrunk object"); + } served.pager.release(object).expect("mapprobe: release"); println!("mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS"); } @@ -316,13 +329,14 @@ fn map(case: &str) { let read = byte(at, 1); panic!("mapprobe --map: a hole the pager had no room for read {read}"); } - "shrunk" => { - let at = mapped(4); - assert_eq!(byte(at, 3), 0x66); + shrunk if shrunk.starts_with("shrunk:") => { + let page: u64 = shrunk["shrunk:".len()..].parse().expect("mapprobe --map: a page"); + let at = mapped(SHRUNK); + assert!((0..SHRUNK).all(|page| byte(at, page) == 0x66), "mapprobe --map: the supplied pages"); assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); - let read = byte(at, 3); - panic!("mapprobe --map: a page past the object read {read}"); + let read = byte(at, page); + panic!("mapprobe --map: page {page}, past the object's end, read {read}"); } "ends" => { let at = mapped(2); diff --git a/tests/toyos.rs b/tests/toyos.rs index 99c703a68f..100ebe0747 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -665,11 +665,11 @@ const MACHINE_TESTS: &[MachineTest] = &[ // endowments a job gets is the image's and the supervisor's, which no // host build has, and the benches run osbench only nightly. ("osbench_capability_groups_exit_0", |_| osbench_capability_groups_exit_0()), - // `osbench map` on tests/images/benchcase.toml with only that job: its - // rows over a mapped file, eight threads faulting one mapping and 255 - // processes holding one while it is truncated, each measured or refused - // by name. Whether they measure correctly is a guest's to say; what they - // measure is owed on metal (the memory track record). + // `osbench map` on tests/images/benchcase.toml with only that job, on + // QEMU x86-64's q35: its rows over a mapped file, eight threads faulting + // one mapping and 255 processes holding one while it is truncated, each + // required measured. Whether they measure correctly is a guest's to say; + // what they measure is owed on metal (the memory track record). ("osbench_map_group_exits_0", |_| osbench_map_group_exits_0()), // What the kernel reads and writes for the `acpi` claim's holder and what // it refuses. A red here is a write the kernel made — to RAM, to the From 6455b27bd4c6044cc72ea4a22404c70eda79a8f5 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 23:28:46 +0200 Subject: [PATCH 24/34] Review r2 of #1269: the records say what no row prices, and msync's true reason B4a: the pager-wide wakes are recorded as unpriced, since `threads_touch_16m` faults pages that are never absent; owner map7b with step 3, exit a many-threads cold-fault row on a DATA file past its pager's capacity. The index-read departure is gone with the O(R) visit. B4b: `msync` answers `ENOSYS` because no call answers which pages of a range are mapped, which POSIX's `ENOMEM` needs; the record and `refused.rs` both say so. Co-Authored-By: Claude Opus 5.5 --- ...bc-refuses-what-toyos-cannot-yet-answer.md | 4 ++-- ...b-pages-still-copies-each-programs-text.md | 19 +++++++++---------- sdk/libc/src/refused.rs | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index ea5e7585cb..09ae7e281f 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -17,8 +17,8 @@ These answer failure in their POSIX form and do nothing LLVM's `getDefaultStackSize` (`llvm/lib/Support/ProgramStack.cpp`) ignores the failure and reads the `rlimit` it did not get. - `gethostname`, `ENOSYS`: no host name is published to a process. -- `msync`, `ENOSYS`: a file is mapped read-only only, so no mapping holds - anything to write back; it answers once 7b's step 5 maps files writable. +- `msync`, `ENOSYS`: no call answers which pages of a range are mapped, which + POSIX's `ENOMEM` for an unmapped one needs. - `mmap` of a file other than `PROT_READ`, `ENODEV`: the kernel maps a file's pages read-only and shared, and a private one shares them until it is written, which waits for 7b's write faults (step 5); a private copy range of a file diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index e4aa5cada2..5e767029bf 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -160,16 +160,15 @@ mappings and write-back; (6) the #1077 retry processes map, truncated). Their correctness runs in a guest (`osbench_map_group_exits_0`); what they measure is unmeasured until then. -What these rows price: fault waiters wake on their pager's watch, so a -supply, a write's end or a removal wakes every fault waiter of that pager and -not only those its pages meet, a departure from the design's per-request -wakes (§3, S4) that `threads_touch_16m` on a file past its server's capacity -prices; and a revocation visits each mapper once per batch of 64 revoking -pages, each batch read by walking the object's resident pages under its -pager's lock, which `truncate_255` prices. Exit for both: per-request wakes -or a cheaper visit where those rows show the cost. A cold DATA page's miss in -stages is not yet an osbench row (owner map7b; exit: such a row on DATA, -judged against Linux on metal). +Fault waiters wake on their pager's watch, so a supply, a write's end or a +removal wakes every fault waiter of that pager and not only those its pages +meet: a departure from the design's per-request wakes (§3, S4). No row +prices it, since `threads_touch_16m` faults a RAM file's pages, which are +never absent. Owner: map7b, with step 3, whose spawns fault a DATA image's +pages cold. Exit: a many-threads cold-fault row on a DATA file larger than its +pager's capacity, judged against Linux on metal, and per-request wakes where +it shows the cost. A cold DATA page's miss in stages is the same kind of row +and waits with it. The pinned-copy mode (a copy holding a page pin stops at an absent page, lets its pins go, waits holding nothing and copies again) is reached by no process diff --git a/sdk/libc/src/refused.rs b/sdk/libc/src/refused.rs index 3ec65d6dc4..33eed109c3 100644 --- a/sdk/libc/src/refused.rs +++ b/sdk/libc/src/refused.rs @@ -110,8 +110,8 @@ pub unsafe extern "C" fn getpwuid_r(_uid: u32, _pwd: *mut u8, _buf: *mut u8, _bu ENOSYS } -/// No call answers which pages of a range are mapped, and no mapping is of a -/// file (`mmap`). +/// No call answers which pages of a range are mapped, which POSIX's `ENOMEM` +/// for an unmapped one needs. #[no_mangle] pub unsafe extern "C" fn msync(_addr: *mut u8, _len: usize, _flags: i32) -> i32 { refuse() From d594827783a49da760bfdfd77f5d2541518548d4 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 00:34:30 +0200 Subject: [PATCH 25/34] Review r3 of #1269: the walk sweeps again whenever a pass began since its last sweep B10. The sweep could miss a page marked while it had the pager's lock dropped: the removal set `again`, the new pass found no mapper, and the walk gave the page's frame back unswept, with a waiter keyed on it through a leaf an unmap had already cleared. The rule is now the pure crate's: `next_visit` answers `Visit(node)`, `Sweep` or `Finish`, the object records the epoch it was swept at, and it answers `Sweep` again whenever a pass has begun since; `finish` asserts the sweep is current. So every frame the walk gives back was swept after the last pass's visits, and no wait can arm on it after: no leaf maps a revoking frame once its pass's visits are done, and no fault installs one. An unmap, `discard`, `protect` to none or a space's end that cleared a paged leaf, revoking only frames it owns, leaves the frame in the index until a removal's walk sweeps it. The kernel loop is a plain match on that answer. The model takes the sweep as a step of its own, a frame at a time with every other step free to run between, and refuses a frame given back that the current sweep did not reach. `a_page_marked_during_the_sweep_is_swept_before_its_frame_goes_back` drives the pure API through r3's sequence: it passes, and under a once-per-walk rule (a checked one-line patch; the rule before r3 swept again after any visit) it is red. Both model runs red too, but only through `finish`'s assert, not their own oracle. `Pages::walk` has no production caller left and moves into the test module. Suggestions: the `vm` test sits after the test module's imports, and `REVOKE_BATCH` names `mapprobe`'s `shrunk` case, which needs two batches. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 165 +++++++++++++++++++++++++++++-------- kernel/pure/vm.rs | 4 +- kernel/src/object/paged.rs | 51 ++++++------ 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index d329743920..54a167081d 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -262,6 +262,17 @@ pub enum Supplied { Present, } +/// What a walk does next ([`Pages::next_visit`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Next { + /// Visit this mapper: clear its leaves of the walk's set. + Visit(N), + /// End every futex wait on a frame of the walk's set. + Sweep, + /// Give the set's frames back ([`Pages::finish`]). + Finish, +} + /// Where a reader of an object's revoking pages has read to /// ([`Pages::revoking`]). #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -371,6 +382,9 @@ pub struct Object { walk: Walk, /// The epoch whose pass last ended the walk. finished: u64, + /// The epoch the running walk last ended every futex wait on its set at + /// ([`Next::Sweep`]). + swept: Option, } /// What a holder's read began at ([`Object::read_begin`]). @@ -455,6 +469,7 @@ impl Object { Walk::Idle => { self.epoch += 1; self.walk = Walk::Running { again: false }; + self.swept = None; Revoke::Walk } Walk::Running { .. } => { @@ -563,6 +578,7 @@ impl Pages { revoking: 0, revoke_head: NONE, revoke_tail: NONE, + swept: None, epoch: 0, walk: Walk::Idle, finished: 0, @@ -717,51 +733,42 @@ impl Pages { o.revoke_tail = i; } - /// Visit `o`'s next mapper after pages were marked revoking - /// ([`Object::revoke`]), or, once every mapper has been visited since the - /// last marking, give out each revoking page's frame, its page and what it - /// became, and answer `None`: [`Self::next_visit`], then [`Self::finish`]. - pub fn walk( - &mut self, - o: &mut Object, - mappers: &mut M, - waiters: &mut impl Waiters, - out: &mut impl FnMut(F, u64, Revoked), - ) -> Option { - let next = self.next_visit(o, mappers); - if next.is_none() { - self.finish(o, waiters, out); - } - next - } - - /// `o`'s next mapper to visit, a new pass begun where a removal queued - /// more pages meanwhile; `None` once every mapper has been visited since - /// the last marking, when [`Self::finish`] is owed under the same hold. - pub fn next_visit(&mut self, o: &mut Object, mappers: &mut M) -> Option { + /// What `o`'s walk does next, after pages were marked revoking + /// ([`Object::revoke`]): visit a mapper not visited in this pass, a new + /// pass begun where a removal queued more pages meanwhile; once every + /// mapper has been visited, end every futex wait on the walk's set, + /// again whenever a pass has begun since the last time, since a page + /// marked during a sweep may be waited on through a leaf an unmap + /// already cleared; and once both are done, [`Self::finish`], owed under + /// the same hold. + pub fn next_visit(&mut self, o: &mut Object, mappers: &mut M) -> Next { loop { if let Some(node) = mappers.head().filter(|&n| mappers.seen(n) != o.epoch) { mappers.visited(node, o.epoch); - return Some(node); + return Next::Visit(node); } match o.walk { Walk::Running { again: true } => { o.epoch += 1; o.walk = Walk::Running { again: false }; } - Walk::Running { again: false } => return None, + Walk::Running { again: false } if o.swept != Some(o.epoch) => { + o.swept = Some(o.epoch); + return Next::Sweep; + } + Walk::Running { again: false } => return Next::Finish, Walk::Idle => panic!("paged: a walk of object {} that never began", o.id), } } } - /// The walk's end, once [`Self::next_visit`] answered `None`: each + /// The walk's end, once [`Self::next_visit`] answered [`Next::Finish`]: each /// revoking page leaves, its frame given out with its page and what it /// became. A page that ends absent was withdrawn, perhaps after a supply /// for a waiter's notice, without moving the version: every want of /// `waiters` it meets is named again. pub fn finish(&mut self, o: &mut Object, waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked)) { - assert_eq!(o.walk, Walk::Running { again: false }, "paged: object {}'s walk ended with a pass owed", o.id); + assert!(o.walk == Walk::Running { again: false } && o.swept == Some(o.epoch), "paged: object {}'s walk ended with a pass or a sweep owed", o.id); let held = !o.gone; let mut at = core::mem::replace(&mut o.revoke_head, NONE); o.revoke_tail = NONE; @@ -1091,6 +1098,30 @@ mod tests { use super::*; + impl Pages { + /// The kernel's walk where no futex wait is to end: the next mapper + /// to visit, or, once every one is visited and the sweep passed, the + /// walk's end and `None`. + fn walk( + &mut self, + o: &mut Object, + mappers: &mut M, + waiters: &mut impl Waiters, + out: &mut impl FnMut(F, u64, Revoked), + ) -> Option { + loop { + match self.next_visit(o, mappers) { + Next::Visit(node) => return Some(node), + Next::Sweep => {} + Next::Finish => { + self.finish(o, waiters, out); + return None; + } + } + } + } + } + /// No space maps anything: a walk ends at its first step. struct Unmapped; @@ -1142,6 +1173,38 @@ mod tests { } } + /// A page marked while the sweep has its lock dropped is swept before the + /// walk gives its frame back, though the pass its marking begins finds no + /// mapper: an unmap may have cleared the leaf a waiter keyed on it went + /// through, and only the sweep ends that wait. + #[test] + fn a_page_marked_during_the_sweep_is_swept_before_its_frame_goes_back() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + p.supply(&mut o, 0, 100, false, PAGE).unwrap(); + p.supply(&mut o, 1, 101, false, 2 * PAGE).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(o.revoke(), Revoke::Walk); + let mut swept = Vec::new(); + let mut sweep = |p: &Pages, o: &Object| { + let mut batch = [(0, 0); 4]; + let n = p.revoking(o, &mut Cursor::Start, &mut batch); + swept.extend(batch[..n].iter().map(|&(_, frame)| frame)); + }; + assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Sweep); + sweep(&p, &o); + // Between the sweep's last read and its relock, another removal. + assert_eq!(p.withdraw(&mut o, 1, 1, &mut Nobody).0, 1); + assert!(matches!(o.revoke(), Revoke::Queued(_))); + assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Sweep, "a pass began since the sweep"); + sweep(&p, &o); + assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Finish); + let mut freed = Vec::new(); + p.finish(&mut o, &mut Nobody, &mut |frame, _, _| freed.push(frame)); + assert_eq!(freed, [100, 101]); + assert!(freed.iter().all(|frame| swept.contains(frame)), "a frame went back unswept: swept {swept:?}"); + } + /// The walk's set holds each revoking page once, in the order they were /// marked; a cursor reads it a batch at a time and sees pages marked /// between batches; the walk's end empties it. @@ -1872,6 +1935,9 @@ mod tests { Walk, /// The walker's visit of the node it took, under that space's lock. Visit, + /// The walker's sweep of the futex waits on the walk's set, a + /// frame at a time. + Sweep, /// The server reads its notices: each request not yet named. Report, /// The server answers the oldest request it read, by a supply. @@ -1880,7 +1946,7 @@ mod tests { Retry, } - pub(super) const STEPS: [Step; 21] = [ + pub(super) const STEPS: [Step; 22] = [ Step::Map(0), Step::Map(1), Step::Unmap(0), @@ -1899,6 +1965,7 @@ mod tests { Step::End, Step::Walk, Step::Visit, + Step::Sweep, Step::Report, Step::Answer, Step::Retry, @@ -1916,6 +1983,10 @@ mod tests { /// The space a walk visits, and where its visit has read the /// object's revoking pages to. visiting: Option<(u32, Cursor)>, + /// Where the walker's sweep has read the walk's set to, and the + /// frames that sweep reached. + sweeping: Option, + swept: BTreeSet, /// Faults parked on an absent page: the space, the page, and the /// request as the pager hears of it. parked: std::vec::Vec<(u32, u64, Want, Version)>, @@ -1948,6 +2019,8 @@ mod tests { freed: BTreeSet::new(), walking: false, visiting: None, + sweeping: None, + swept: BTreeSet::new(), parked: std::vec::Vec::new(), asked: std::collections::VecDeque::new(), } @@ -2000,14 +2073,30 @@ mod tests { self.p.invalidate(&mut self.o); self.revoke(); } - Step::Walk if self.walking && self.visiting.is_none() => { - let freed = &mut self.freed; - let walked = self.p.walk(&mut self.o, &mut self.ring, &mut Parked(&mut self.parked), &mut |f, _, _| { - assert!(freed.insert(f), "frame {f} given back twice"); - }); - match walked { - Some(node) => self.visiting = Some((node, Cursor::Start)), - None => self.walking = false, + Step::Walk if self.walking && self.visiting.is_none() && self.sweeping.is_none() => { + match self.p.next_visit(&mut self.o, &mut self.ring) { + Next::Visit(node) => self.visiting = Some((node, Cursor::Start)), + Next::Sweep => { + self.sweeping = Some(Cursor::Start); + self.swept.clear(); + } + Next::Finish => { + let (freed, swept) = (&mut self.freed, &self.swept); + self.p.finish(&mut self.o, &mut Parked(&mut self.parked), &mut |f, _, _| { + assert!(swept.contains(&f), "frame {f} given back unswept"); + assert!(freed.insert(f), "frame {f} given back twice"); + }); + self.walking = false; + } + } + } + // The sweep, a frame at a time: anything may run between. + Step::Sweep if self.sweeping.is_some() => { + let mut at = self.sweeping.take().unwrap(); + let mut batch = [(0, 0)]; + if self.p.revoking(&self.o, &mut at, &mut batch) == 1 { + self.swept.insert(batch[0].1); + self.sweeping = Some(at); } } Step::Report => { @@ -2086,7 +2175,7 @@ mod tests { /// answers every request it hears of, and every parked fault /// retries: none is left parked. pub(super) fn drain(&mut self, steps: &[Step]) { - let fair = [Step::Walk, Step::Visit, Step::Report, Step::Answer, Step::Retry]; + let fair = [Step::Walk, Step::Visit, Step::Sweep, Step::Report, Step::Answer, Step::Retry]; for _ in 0..64 { if !fair.iter().fold(false, |moved, &step| self.step(step) | moved) { break; @@ -2176,6 +2265,10 @@ mod tests { Step::Visit, Step::Visit, Step::Walk, + // The sweep: the revoked page's frame, then the read that finds none. + Step::Sweep, + Step::Sweep, + Step::Walk, Step::Grow, Step::Retry, ]; diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index 024fcf668d..091d7d2d83 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -398,6 +398,8 @@ impl Inventory { #[cfg(test)] mod tests { use super::*; + use std::vec; + use std::vec::Vec; /// A page a range maps lands at its place; one before the range's /// offset, at or past its end, or so high its byte offset overflows is @@ -412,8 +414,6 @@ mod tests { assert_eq!(page_at(start, end, 0, (1 << 52) - 2), None, "a page near the last a pager may name"); assert_eq!(page_at(start, end, 0, u64::MAX), None, "a byte offset past u64"); } - use std::vec; - use std::vec::Vec; /// Shared memory is executed only where nobody can write it and /// written only where somebody can; a range is whole pages inside the diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index d4bb45bd7c..f8d28a05d8 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -58,7 +58,7 @@ use core::ptr; use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::Stack; -use kernel::paged::{Core, Cursor, Draw, Leave, Lookup, Mappers, Object, Pages, Refused, Revoke, Revoked, Supplied, Version, Waiters, Want, PAGE}; +use kernel::paged::{Core, Cursor, Draw, Leave, Lookup, Mappers, Next, Object, Pages, Refused, Revoke, Revoked, Supplied, Version, Waiters, Want, PAGE}; use kernel::sched::task::WaitClass; use kernel::slots::Slots; use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; @@ -366,11 +366,13 @@ impl PagerObject { } /// The pages of the object at `slot` just marked revoking leave once no - /// mapping reaches them: this walks the object's mappers, the lock dropped - /// for each space's visit and the slot held meanwhile, gives each frame - /// back, and names again every waiter's want a page that ends absent meets - /// (`Pages::walk`). Answers the lock and, where another's walk runs and - /// takes the pages in its next pass, that pass's epoch. + /// mapping reaches them and no futex wait is keyed on their frames: this + /// walks the object's mappers, sweeps the waits on the walk's set after + /// every pass ([`Pages::next_visit`]), the lock dropped for each visit and + /// sweep and the slot held meanwhile, then gives each frame back and names + /// again every waiter's want a page that ends absent meets + /// ([`Pages::finish`]). Answers the lock and, where another's walk runs + /// and takes the pages in its next pass, that pass's epoch. fn revoked<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> (LockGuard<'a, State>, Option) { match state.entry_mut(slot).object.revoke() { Revoke::Done => return (state, None), @@ -379,30 +381,23 @@ impl PagerObject { } let job = self.core.job(); state.entry_mut(slot).refs += 1; - // Whether every futex wait on a revoking frame was ended since the - // last pass began: owed before any frame goes back. - let mut swept = false; loop { let State { pages, entries, .. } = &mut *state; let Entry { object, mappers, .. } = entries.get_mut(slot).expect("paged: a walked slot is held"); match pages.next_visit(object, &mut Ring(mappers)) { - Some(node) => { - swept = false; + Next::Visit(node) => { // SAFETY: linked, so live, under the lock; a range holds // it, so it upgrades, and the count kept holds it past // its unlinking while the lock is dropped. let node = unsafe { (*node).this.upgrade() }.expect("paged: a linked mapper is held by a range"); state = self.visit(state, slot, &node); } - None if !swept => { - state = self.end_futex_waits(state, slot); - swept = true; - } - None => break, + Next::Sweep => state = self.end_futex_waits(state, slot), + Next::Finish => break, } } - // Every mapper visited and every wait ended, in this hold since - // `next_visit` said so: the frames go back. + // Every mapper visited and every wait swept since the last pass + // began, in this hold since `next_visit` said so: the frames go back. let mut absent = false; let mut waiters = Linked { head: state.requests, slot }; let State { pages, entries, held, gone, .. } = &mut *state; @@ -439,14 +434,15 @@ impl PagerObject { } } - /// Every futex wait on a revoking frame of the object at `slot` ends, - /// read a batch at a time under the pager's lock and ended outside it: - /// after every mapper's visit, so no leaf maps one any more and none is - /// waited on afresh, and before any frame goes back, so no waiter keyed - /// on one meets the frame reissued (`sched::futex::revoke_frames`). This - /// covers a leaf an unmap, `discard`, `protect` to none or a space's end - /// cleared before the frame was removed, whose clearing revokes no - /// frame it does not own. + /// Every futex wait on a frame of the walk's set at `slot` ends, read a + /// batch at a time under the pager's lock and ended outside it + /// ([`Next::Sweep`]): after a pass's visits, so no leaf of a page marked + /// before it maps its frame and none is waited on afresh, and again after + /// any pass a page marked meanwhile began, before any frame goes back, so + /// no waiter keyed on one meets the frame reissued + /// (`sched::futex::revoke_frames`). This covers a leaf an unmap, + /// `discard`, `protect` to none or a space's end cleared before the frame + /// was removed, whose clearing revokes no frame it does not own. fn end_futex_waits<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { let mut at = Cursor::Start; loop { @@ -1485,6 +1481,9 @@ impl Index<'_> { } /// The revoking pages a walk reads under its pager's lock at once. +/// `tests/guest/mapprobe`'s `shrunk` case revokes 79 pages so that its +/// revocation takes two batches; at 79 or more it would take one, and that +/// case would no longer reach a second. const REVOKE_BATCH: usize = 64; /// The pages a fault's request names: the aligned run around its page. From 76bcaa5dadbbcc9d467b0b9fbb1b54f39fecae0a Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 00:34:30 +0200 Subject: [PATCH 26/34] Review r4 of #1269: the walk ends itself, and the model's futex waiter is its oracle S1, required. `Pages::walk` is the one step of a walk: it answers `Visit(node)` or `Sweep`, or ends the walk itself and answers `Finished`, the set's frames given out to its caller's closure. `finish` is private, so no caller can give frames back off the rule, and its assert goes. The kernel's loop builds the waiters and the give-back closure from disjoint fields of the same `State` each step and breaks on `Finished`; the test-only helper is gone. S2, required. The model's oracle is the property itself: a `Wait(s, page)` step keys a futex waiter on the frame the space's leaf maps, only a sweep's read ends it (a visit or an unmap revokes no frame it does not own), and the walk's end refuses a frame a waiter is keyed on. The `swept` set it replaces was a proxy. `a_waiter_an_unmap_left_is_never_on_a_frame_given_back` explores five steps from a waiter an unmap left and a walk begun with no mapper, which reaches r3's sequence. Controls, each a checked patch, the source restored byte-identical: - sweeping before the pass's visits (B8's defect): both model runs red on the oracle, "frame N given back with a futex waiter keyed on it", and four scripted tests on their expected order; - the rule before r3 (a sweep's flag reset by a visit, sweep while unset): the no-mapper exploration red on the oracle, and `a_page_marked_during_the_sweep_is_swept_before_its_frame_goes_back` red ("a pass began since the sweep"). B12: `swept` is the epoch the walk was last told to sweep at; `revoked` sweeps as `Pages::walk` directs; `end_futex_waits` says what it does and why, not when, which is the walk's rule alone; the module header names the sweep before the walk's end, and its `Pages::walk` links name a production function again. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 188 +++++++++++++++++++++---------------- kernel/src/object/paged.rs | 50 +++++----- 2 files changed, 129 insertions(+), 109 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 54a167081d..bca8e4865a 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -14,8 +14,9 @@ //! reads as absent, takes no supply or write, and keeps its slot and frame, //! so no page is ever two frames at once. One walk per object visits each of //! its mappers ([`Mappers`]) after the marking, and a removal that finds a -//! walk running is taken by that walk's next pass; the last pass's end gives -//! every revoking page's frame out ([`Pages::walk`]). +//! walk running is taken by that walk's next pass; after the last pass and a +//! sweep of the futex waits on them, the walk's end gives every revoking +//! page's frame out ([`Pages::walk`]). //! //! **A dirty page holds a slot until a write-back cleans it.** Slots are //! bounded by `dirty_ceiling`. A write-back begin clears the dirty bit and @@ -262,15 +263,15 @@ pub enum Supplied { Present, } -/// What a walk does next ([`Pages::next_visit`]). +/// What a walk's step asks of its caller, or that it ended ([`Pages::walk`]). #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Next { /// Visit this mapper: clear its leaves of the walk's set. Visit(N), /// End every futex wait on a frame of the walk's set. Sweep, - /// Give the set's frames back ([`Pages::finish`]). - Finish, + /// The walk ended: the set's frames were given out. + Finished, } /// Where a reader of an object's revoking pages has read to @@ -382,7 +383,7 @@ pub struct Object { walk: Walk, /// The epoch whose pass last ended the walk. finished: u64, - /// The epoch the running walk last ended every futex wait on its set at + /// The epoch at which the running walk was last told to sweep /// ([`Next::Sweep`]). swept: Option, } @@ -733,15 +734,22 @@ impl Pages { o.revoke_tail = i; } - /// What `o`'s walk does next, after pages were marked revoking + /// The next step of `o`'s walk, after pages were marked revoking /// ([`Object::revoke`]): visit a mapper not visited in this pass, a new /// pass begun where a removal queued more pages meanwhile; once every - /// mapper has been visited, end every futex wait on the walk's set, - /// again whenever a pass has begun since the last time, since a page - /// marked during a sweep may be waited on through a leaf an unmap - /// already cleared; and once both are done, [`Self::finish`], owed under - /// the same hold. - pub fn next_visit(&mut self, o: &mut Object, mappers: &mut M) -> Next { + /// mapper has been visited, have every futex wait on the walk's set + /// ended, again whenever a pass has begun since it was last asked, since + /// a page marked during a sweep may be waited on through a leaf an unmap + /// already cleared; and once both are done, end the walk here + /// ([`Next::Finished`]): each revoking page leaves, its frame given out + /// to `out` with its page and what it became. + pub fn walk( + &mut self, + o: &mut Object, + mappers: &mut M, + waiters: &mut impl Waiters, + out: &mut impl FnMut(F, u64, Revoked), + ) -> Next { loop { if let Some(node) = mappers.head().filter(|&n| mappers.seen(n) != o.epoch) { mappers.visited(node, o.epoch); @@ -756,19 +764,21 @@ impl Pages { o.swept = Some(o.epoch); return Next::Sweep; } - Walk::Running { again: false } => return Next::Finish, + Walk::Running { again: false } => { + self.finish(o, waiters, out); + return Next::Finished; + } Walk::Idle => panic!("paged: a walk of object {} that never began", o.id), } } } - /// The walk's end, once [`Self::next_visit`] answered [`Next::Finish`]: each - /// revoking page leaves, its frame given out with its page and what it - /// became. A page that ends absent was withdrawn, perhaps after a supply - /// for a waiter's notice, without moving the version: every want of - /// `waiters` it meets is named again. - pub fn finish(&mut self, o: &mut Object, waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked)) { - assert!(o.walk == Walk::Running { again: false } && o.swept == Some(o.epoch), "paged: object {}'s walk ended with a pass or a sweep owed", o.id); + /// The walk's end, which only [`Self::walk`] reaches: each revoking page + /// leaves, its frame given out with its page and what it became. A page + /// that ends absent was withdrawn, perhaps after a supply for a waiter's + /// notice, without moving the version: every want of `waiters` it meets + /// is named again. + fn finish(&mut self, o: &mut Object, waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked)) { let held = !o.gone; let mut at = core::mem::replace(&mut o.revoke_head, NONE); o.revoke_tail = NONE; @@ -1098,29 +1108,6 @@ mod tests { use super::*; - impl Pages { - /// The kernel's walk where no futex wait is to end: the next mapper - /// to visit, or, once every one is visited and the sweep passed, the - /// walk's end and `None`. - fn walk( - &mut self, - o: &mut Object, - mappers: &mut M, - waiters: &mut impl Waiters, - out: &mut impl FnMut(F, u64, Revoked), - ) -> Option { - loop { - match self.next_visit(o, mappers) { - Next::Visit(node) => return Some(node), - Next::Sweep => {} - Next::Finish => { - self.finish(o, waiters, out); - return None; - } - } - } - } - } /// No space maps anything: a walk ends at its first step. struct Unmapped; @@ -1141,7 +1128,10 @@ mod tests { let mut freed = Vec::new(); match o.revoke() { Revoke::Done => {} - Revoke::Walk => assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), None), + Revoke::Walk => { + assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Sweep); + assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); + } Revoke::Queued(_) => panic!("a walk ran with no walker"), } freed @@ -1191,16 +1181,15 @@ mod tests { let n = p.revoking(o, &mut Cursor::Start, &mut batch); swept.extend(batch[..n].iter().map(|&(_, frame)| frame)); }; - assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Sweep); + let mut freed = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Sweep); sweep(&p, &o); // Between the sweep's last read and its relock, another removal. assert_eq!(p.withdraw(&mut o, 1, 1, &mut Nobody).0, 1); assert!(matches!(o.revoke(), Revoke::Queued(_))); - assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Sweep, "a pass began since the sweep"); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Sweep, "a pass began since the sweep"); sweep(&p, &o); - assert_eq!(p.next_visit(&mut o, &mut Unmapped), Next::Finish); - let mut freed = Vec::new(); - p.finish(&mut o, &mut Nobody, &mut |frame, _, _| freed.push(frame)); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Finished); assert_eq!(freed, [100, 101]); assert!(freed.iter().all(|frame| swept.contains(frame)), "a frame went back unswept: swept {swept:?}"); } @@ -1229,7 +1218,8 @@ mod tests { assert_eq!(p.revoking(&o, &mut at, &mut batch), 0); assert_eq!(o.revoke(), Revoke::Walk); let mut freed = Vec::new(); - assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), None); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); assert_eq!(freed, [107, 102, 111, 100]); assert_eq!(p.revoking(&o, &mut Cursor::Start, &mut batch), 0, "the walk's end empties its set"); } @@ -1253,10 +1243,11 @@ mod tests { assert_eq!(o.revoke(), Revoke::Walk); let mut freed = Vec::new(); let mut out = |f, page, becomes| freed.push((f, page, becomes)); - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(11)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(11)); assert!(!o.empty()); - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), None); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); assert_eq!(freed, [(7, 0, Revoked::Absent)]); assert!(o.finished(o.epoch())); assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Installed)); @@ -1279,13 +1270,18 @@ mod tests { assert_eq!(o.revoke(), Revoke::Walk); let mut freed = Vec::new(); let mut out = |f, page, becomes| freed.push((f, page, becomes)); - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); ring.link(12, &o); p.fail(&mut o, 1, 1).unwrap(); let Revoke::Queued(second) = o.revoke() else { panic!("a walk runs") }; assert!(!o.finished(second)); - let visits: Vec<_> = core::iter::from_fn(|| p.walk(&mut o, &mut ring, &mut Nobody, &mut out)).collect(); + let visits: Vec<_> = core::iter::from_fn(|| match p.walk(&mut o, &mut ring, &mut Nobody, &mut out) { + Next::Visit(node) => Some(node), + Next::Sweep | Next::Finished => None, + }) + .collect(); assert_eq!(visits, [11, 10, 12, 11], "the first pass ends at 11, the second visits all three"); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished, "the sweep, then the end"); assert!(o.finished(second)); assert_eq!(freed, [(7, 0, Revoked::Absent), (8, 1, Revoked::Failed)], "in marking order"); assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed)); @@ -1303,9 +1299,10 @@ mod tests { p.withdraw(&mut o, 0, 1, &mut Nobody); assert_eq!(o.revoke(), Revoke::Walk); let mut out = |_, _, _| {}; - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Some(10)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); ring.link(11, &o); - assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), None); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); } /// A page failed and then shrunk away while revoking ends with no slot, @@ -1321,7 +1318,8 @@ mod tests { p.resize(&mut o, PAGE); assert_eq!(o.revoke(), Revoke::Walk); let mut ended = Vec::new(); - assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), None); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), Next::Finished); assert_eq!(ended, [(7, 0, Revoked::Failed), (8, 1, Revoked::Removed)]); assert_eq!((p.lookup(&o, 0), p.used()), (Lookup::Failed, 1)); } @@ -1753,7 +1751,8 @@ mod tests { assert_eq!(p.withdraw(&mut taken, 0, 1, &mut Nobody).0, 1); assert_eq!(taken.revoke(), Revoke::Walk); assert!(!want.unreported(), "named only once no mapping reaches the page"); - assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), None); + assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), Next::Sweep); + assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), Next::Finished); assert!(!p.wait_over(&taken, 0, since), "a withdraw leaves the read waiting"); assert!(want.unreported(), "and its walk's end names its range again"); } @@ -1869,8 +1868,9 @@ mod tests { /// A removal's walk, over no mapper, to its end. fn settle(&mut self) { if self.o.revoke() == Revoke::Walk { - let walked = self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()); - assert_eq!(walked, None); + for end in [Next::Sweep, Next::Finished] { + assert_eq!(self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()), end); + } } } @@ -1944,9 +1944,14 @@ mod tests { Answer, /// Every parked fault looks at its page again. Retry, + /// A thread of space `s` waits on a futex word in `page`, keyed on + /// the frame its leaf maps; only a sweep ends the wait, since a + /// visit or an unmap clears a leaf and revokes no frame it does + /// not own. + Wait(u32, u64), } - pub(super) const STEPS: [Step; 22] = [ + pub(super) const STEPS: [Step; 24] = [ Step::Map(0), Step::Map(1), Step::Unmap(0), @@ -1969,6 +1974,8 @@ mod tests { Step::Report, Step::Answer, Step::Retry, + Step::Wait(0, 0), + Step::Wait(1, 1), ]; pub(super) struct World { @@ -1983,10 +1990,10 @@ mod tests { /// The space a walk visits, and where its visit has read the /// object's revoking pages to. visiting: Option<(u32, Cursor)>, - /// Where the walker's sweep has read the walk's set to, and the - /// frames that sweep reached. + /// Where the walker's sweep has read the walk's set to. sweeping: Option, - swept: BTreeSet, + /// The frames a futex waiter is keyed on. + futex: BTreeSet, /// Faults parked on an absent page: the space, the page, and the /// request as the pager hears of it. parked: std::vec::Vec<(u32, u64, Want, Version)>, @@ -2020,7 +2027,7 @@ mod tests { walking: false, visiting: None, sweeping: None, - swept: BTreeSet::new(), + futex: BTreeSet::new(), parked: std::vec::Vec::new(), asked: std::collections::VecDeque::new(), } @@ -2074,20 +2081,15 @@ mod tests { self.revoke(); } Step::Walk if self.walking && self.visiting.is_none() && self.sweeping.is_none() => { - match self.p.next_visit(&mut self.o, &mut self.ring) { + let (freed, futex) = (&mut self.freed, &self.futex); + let walked = self.p.walk(&mut self.o, &mut self.ring, &mut Parked(&mut self.parked), &mut |f, _, _| { + assert!(!futex.contains(&f), "frame {f} given back with a futex waiter keyed on it"); + assert!(freed.insert(f), "frame {f} given back twice"); + }); + match walked { Next::Visit(node) => self.visiting = Some((node, Cursor::Start)), - Next::Sweep => { - self.sweeping = Some(Cursor::Start); - self.swept.clear(); - } - Next::Finish => { - let (freed, swept) = (&mut self.freed, &self.swept); - self.p.finish(&mut self.o, &mut Parked(&mut self.parked), &mut |f, _, _| { - assert!(swept.contains(&f), "frame {f} given back unswept"); - assert!(freed.insert(f), "frame {f} given back twice"); - }); - self.walking = false; - } + Next::Sweep => self.sweeping = Some(Cursor::Start), + Next::Finished => self.walking = false, } } // The sweep, a frame at a time: anything may run between. @@ -2095,10 +2097,13 @@ mod tests { let mut at = self.sweeping.take().unwrap(); let mut batch = [(0, 0)]; if self.p.revoking(&self.o, &mut at, &mut batch) == 1 { - self.swept.insert(batch[0].1); + self.futex.remove(&batch[0].1); self.sweeping = Some(at); } } + Step::Wait(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_some() => { + self.futex.insert(self.leaves[s as usize][page as usize].unwrap()); + } Step::Report => { let before = self.asked.len(); for (_, _, want, _) in &mut self.parked { @@ -2248,6 +2253,27 @@ mod tests { assert!(played > 20_000, "only {played} sequences played"); } + /// From a futex waiter an unmap left keyed on a frame and a walk begun + /// with no mapper left, every sequence of five steps gives back no frame + /// a waiter is keyed on: a page marked while the sweep has its lock + /// dropped is swept before the walk ends. + #[test] + fn a_waiter_an_unmap_left_is_never_on_a_frame_given_back() { + use mapped::Step; + let prefix = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Wait(0, 0), + Step::Unmap(0), + Step::Withdraw(1), + Step::Walk, + ]; + let played = explore(&prefix, 5); + assert!(played > 1_000, "only {played} sequences played"); + } + /// A fault whose request the pager read and answered, before the fault /// looked again, meets a shrink and a grow: the version moved, so it /// asks afresh and is heard. diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index f8d28a05d8..8dc19c4fbb 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -367,12 +367,12 @@ impl PagerObject { /// The pages of the object at `slot` just marked revoking leave once no /// mapping reaches them and no futex wait is keyed on their frames: this - /// walks the object's mappers, sweeps the waits on the walk's set after - /// every pass ([`Pages::next_visit`]), the lock dropped for each visit and - /// sweep and the slot held meanwhile, then gives each frame back and names - /// again every waiter's want a page that ends absent meets - /// ([`Pages::finish`]). Answers the lock and, where another's walk runs - /// and takes the pages in its next pass, that pass's epoch. + /// visits the object's mappers and sweeps the waits on the walk's set as + /// [`Pages::walk`] directs, the lock dropped for each visit and sweep and + /// the slot held meanwhile, until the walk's end gives each frame back + /// and names again every waiter's want a page that ends absent meets. + /// Answers the lock and, where another's walk runs and takes the pages in + /// its next pass, that pass's epoch. fn revoked<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> (LockGuard<'a, State>, Option) { match state.entry_mut(slot).object.revoke() { Revoke::Done => return (state, None), @@ -381,10 +381,16 @@ impl PagerObject { } let job = self.core.job(); state.entry_mut(slot).refs += 1; + let mut absent = false; loop { - let State { pages, entries, .. } = &mut *state; + let mut waiters = Linked { head: state.requests, slot }; + let State { pages, entries, held, gone, .. } = &mut *state; let Entry { object, mappers, .. } = entries.get_mut(slot).expect("paged: a walked slot is held"); - match pages.next_visit(object, &mut Ring(mappers)) { + let walked = pages.walk(object, &mut Ring(mappers), &mut waiters, &mut |phys, _, becomes| { + held.give_back(job, *gone, phys, true); + absent |= becomes == Revoked::Absent; + }); + match walked { Next::Visit(node) => { // SAFETY: linked, so live, under the lock; a range holds // it, so it upgrades, and the count kept holds it past @@ -393,19 +399,9 @@ impl PagerObject { state = self.visit(state, slot, &node); } Next::Sweep => state = self.end_futex_waits(state, slot), - Next::Finish => break, + Next::Finished => break, } } - // Every mapper visited and every wait swept since the last pass - // began, in this hold since `next_visit` said so: the frames go back. - let mut absent = false; - let mut waiters = Linked { head: state.requests, slot }; - let State { pages, entries, held, gone, .. } = &mut *state; - let object = &mut entries.get_mut(slot).expect("paged: a walked slot is held").object; - pages.finish(object, &mut waiters, &mut |phys, _, becomes| { - held.give_back(job, *gone, phys, true); - absent |= becomes == Revoked::Absent; - }); self.unref_locked(&mut state, slot); if absent { self.watch.post(); @@ -434,15 +430,13 @@ impl PagerObject { } } - /// Every futex wait on a frame of the walk's set at `slot` ends, read a - /// batch at a time under the pager's lock and ended outside it - /// ([`Next::Sweep`]): after a pass's visits, so no leaf of a page marked - /// before it maps its frame and none is waited on afresh, and again after - /// any pass a page marked meanwhile began, before any frame goes back, so - /// no waiter keyed on one meets the frame reissued - /// (`sched::futex::revoke_frames`). This covers a leaf an unmap, - /// `discard`, `protect` to none or a space's end cleared before the frame - /// was removed, whose clearing revokes no frame it does not own. + /// Every futex wait on a frame of the walk's set at `slot` ends + /// ([`Next::Sweep`]): the frames read a batch at a time under the pager's + /// lock, and the waits ended outside it (`sched::futex::revoke_frames`). + /// An unmap, `discard`, `protect` to none or a space's end clears a leaf + /// without ending the waits on a frame it does not own, so a waiter keyed + /// on a file's frame through such a leaf is ended here, before the frame + /// goes back. fn end_futex_waits<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { let mut at = Cursor::Start; loop { From eb7deec71361478704f92c021474f75654c71852 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 01:11:59 +0200 Subject: [PATCH 27/34] Review r5 of #1269: a revocation visit's shootdown is counted, and the model's futex wait translates and arms apart B13. `clear_paged` shot its span down through the tables directly, which leaves the space's count of changes where it was; #1253's `Translation::holds` trusts an unmoved count, so a futex wait that translated before a visit cleared its leaf could arm after the walk's sweep, keyed on a frame the walk then gave back. The visit now shoots down through the space's own `shoot_down`, which counts the change under the space's lock before the sweep can run. `clear_paged`'s and `futex_word`'s docs say so. The model's futex wait is two steps, as the kernel's is: `Translate(s, page)` notes the frame and the space's count; `Arm` keys the waiter on the frame where the count has not moved and otherwise looks at the leaf again. A visit's clear and an unmap move the count. The reviewer's sequence is `a_wait_translated_before_a_visit_does_not_arm_on_a_frame_given_back`; with the visit's clear uncounted (a checked patch, the source restored byte-identical) it is red on the model's oracle, "frame 101 given back with a futex waiter keyed on it". B14: `Unmapped`'s doc says only that no space maps anything, and the double blank line above it is gone. Teardown takes the space out of the process data, as the handles line above it does, before ending it, so the process data's lock is not held across the space's shootdowns and pager calls. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 80 ++++++++++++++++++++++++++++++++++-------- kernel/src/mm/space.rs | 16 ++++++--- kernel/src/process.rs | 3 +- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index bca8e4865a..82dbf732b6 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -1108,8 +1108,7 @@ mod tests { use super::*; - - /// No space maps anything: a walk ends at its first step. + /// No space maps anything. struct Unmapped; impl Mappers for Unmapped { @@ -1944,14 +1943,18 @@ mod tests { Answer, /// Every parked fault looks at its page again. Retry, - /// A thread of space `s` waits on a futex word in `page`, keyed on - /// the frame its leaf maps; only a sweep ends the wait, since a - /// visit or an unmap clears a leaf and revokes no frame it does - /// not own. - Wait(u32, u64), - } - - pub(super) const STEPS: [Step; 24] = [ + /// A thread of space `s` begins a futex wait on a word in `page`: + /// it translates the word to the frame its leaf maps, under the + /// space's lock, and notes the space's count of changes. + Translate(u32, u64), + /// That thread arms its wait, keyed on the frame: where the count + /// has not moved it trusts the translation, and otherwise looks + /// at the leaf again. Only a sweep ends the wait, since a visit or + /// an unmap clears a leaf and revokes no frame it does not own. + Arm, + } + + pub(super) const STEPS: [Step; 25] = [ Step::Map(0), Step::Map(1), Step::Unmap(0), @@ -1974,8 +1977,9 @@ mod tests { Step::Report, Step::Answer, Step::Retry, - Step::Wait(0, 0), - Step::Wait(1, 1), + Step::Translate(0, 0), + Step::Translate(1, 1), + Step::Arm, ]; pub(super) struct World { @@ -1994,6 +1998,11 @@ mod tests { sweeping: Option, /// The frames a futex waiter is keyed on. futex: BTreeSet, + /// Each space's count of the leaf changes it shot down. + counted: [u64; 2], + /// A futex wait translated and not yet armed: its space, page, + /// frame and the count it read. + translated: Option<(u32, u64, u64, u64)>, /// Faults parked on an absent page: the space, the page, and the /// request as the pager hears of it. parked: std::vec::Vec<(u32, u64, Want, Version)>, @@ -2028,6 +2037,8 @@ mod tests { visiting: None, sweeping: None, futex: BTreeSet::new(), + counted: [0; 2], + translated: None, parked: std::vec::Vec::new(), asked: std::collections::VecDeque::new(), } @@ -2042,6 +2053,7 @@ mod tests { } Step::Unmap(s) if self.mapped[s as usize] => { self.leaves[s as usize] = [None; 2]; + self.counted[s as usize] += 1; let at = self.ring.0.iter().position(|&(n, _)| n == s).expect("linked while mapped"); self.ring.0.remove(at); self.mapped[s as usize] = false; @@ -2101,8 +2113,16 @@ mod tests { self.sweeping = Some(at); } } - Step::Wait(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_some() => { - self.futex.insert(self.leaves[s as usize][page as usize].unwrap()); + Step::Translate(s, page) if self.translated.is_none() && self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_some() => { + let f = self.leaves[s as usize][page as usize].unwrap(); + self.translated = Some((s, page, f, self.counted[s as usize])); + } + Step::Arm if self.translated.is_some() => { + let (s, page, f, at) = self.translated.take().unwrap(); + let holds = self.counted[s as usize] == at || self.leaves[s as usize][page as usize] == Some(f); + if holds { + self.futex.insert(f); + } } Step::Report => { let before = self.asked.len(); @@ -2155,6 +2175,9 @@ mod tests { let (page, f) = batch[0]; if self.leaves[s as usize][page as usize] == Some(f) { self.leaves[s as usize][page as usize] = None; + // The visit's shootdown is counted, as an + // unmap's is. + self.counted[s as usize] += 1; } self.visiting = Some((s, at)); } @@ -2253,6 +2276,32 @@ mod tests { assert!(played > 20_000, "only {played} sequences played"); } + /// A futex wait that translated its word before a walk's visit cleared + /// the leaf, and arms only after the walk's sweep, does not arm on the + /// frame the walk then gives back: the visit's shootdown is counted, so + /// the arm looks at the leaf again. + #[test] + fn a_wait_translated_before_a_visit_does_not_arm_on_a_frame_given_back() { + use mapped::Step; + let steps = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Translate(0, 0), + Step::Withdraw(0), + Step::Walk, + Step::Visit, + Step::Visit, + Step::Walk, + Step::Sweep, + Step::Sweep, + Step::Arm, + Step::Walk, + ]; + play(&steps).expect("every step is taken").drain(&steps); + } + /// From a futex waiter an unmap left keyed on a frame and a walk begun /// with no mapper left, every sequence of five steps gives back no frame /// a waiter is keyed on: a page marked while the sweep has its lock @@ -2265,7 +2314,8 @@ mod tests { Step::Supply(0), Step::Supply(1), Step::Fault(0, 0), - Step::Wait(0, 0), + Step::Translate(0, 0), + Step::Arm, Step::Unmap(0), Step::Withdraw(1), Step::Walk, diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index bc9a407dc7..93a607745a 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -273,7 +273,10 @@ impl Drop for FutexWait { /// A futex word's frame as a hold of its space's lock found it, borrowed from /// the wait that counts it. Not a lease: nothing pins the frame, and a -/// sibling's `munmap` can free it under a parked wait. +/// sibling's `munmap` can free it under a parked wait. A mapped file's frame +/// goes back only after its pager's walk: each visit counts its clear, so +/// [`holds`](Self::holds) looks at the leaf again, and the walk's sweep ends +/// the waits keyed on the frame before it is reissued. pub struct Translation<'w> { space: &'w Space, va: u64, @@ -558,9 +561,12 @@ impl AddressSpace { /// One batch of a walk's visit of `node`'s object ([`MapNode`]): every /// leaf of this space's ranges of it that maps one of `revoking`'s /// `(page, frame)` is cleared, and the span shot down, under this space's - /// lock alone. The walk ends the futex waits on those frames once every - /// mapper is visited, before any goes back. A fault meanwhile waits on - /// this space's lock and then finds the page as the index has it. + /// lock alone. The shootdown is counted ([`Translation::holds`]), so a + /// futex wait that translated before the clear looks at its leaf again + /// and does not arm on a frame the walk goes on to give back; the walk + /// ends the waits already keyed on those frames once every mapper is + /// visited. A fault meanwhile waits on this space's lock and then finds + /// the page as the index has it. pub fn clear_paged(&mut self, node: &MapNode, revoking: &[(u64, u64)]) { let (mut lo, mut hi) = (u64::MAX, 0); let tables = &mut self.tables; @@ -581,7 +587,7 @@ impl AddressSpace { } } if lo < hi { - let _reached = tables.shoot_down(lo, hi, &Unlinked::none()); + self.shoot_down(lo, hi, &Unlinked::none()); } } diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 81b3b1e364..b7e7c57f99 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -663,7 +663,8 @@ fn teardown_resources( let handles = Arc::clone(&process_data_arc.lock().handles); ops::close_all(&mut handles.lock()); - process_data_arc.lock().space.lock().end(); + let space = Arc::clone(&process_data_arc.lock().space); + space.lock().end(); // Every thread has left, and none returns to Ring 3 to use them. crate::isa::process_ends(pid); let space = Arc::clone(&process_data_arc.lock().space); From d1ad82e61c615ddde1c77dde340966ed47f3e07f Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 01:41:52 +0200 Subject: [PATCH 28/34] Review r6 of #1269: the model refuses a waiter keyed on a given-back frame after any step The model's oracle refused a futex waiter on a given-back frame only inside the walk's end, so a wait that armed after the walk ended went unseen. Its `check`, run after every step, now asserts no waiter is keyed on a given-back frame. `a_wait_translated_before_a_removal_never_arms_on_a_frame_given_back` explores six steps from a wait translated before its page was withdrawn and a walk begun, holding B13's class rather than one sequence. With the visit's clear uncounted (a checked patch, the source restored byte-identical), that exploration and the scripted sequence are both red on the oracle, "frame 101 given back with a futex waiter keyed on it". Test code only. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 82dbf732b6..b72c7ee91b 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -2235,6 +2235,7 @@ mod tests { } } assert!(self.walking || self.o.revoking == 0, "after {step:?}: pages revoking with no walk"); + assert!(self.futex.is_disjoint(&self.freed), "after {step:?}: a futex waiter keyed on a given-back frame"); } } } @@ -2302,6 +2303,26 @@ mod tests { play(&steps).expect("every step is taken").drain(&steps); } + /// From a futex wait translated before its page was withdrawn and a walk + /// begun, every sequence of six steps arms no waiter on a frame the walk + /// gives back, whenever it arms: a visit's clear is counted, so the arm + /// looks at the leaf again. + #[test] + fn a_wait_translated_before_a_removal_never_arms_on_a_frame_given_back() { + use mapped::Step; + let prefix = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Translate(0, 0), + Step::Withdraw(0), + Step::Walk, + ]; + let played = explore(&prefix, 6); + assert!(played > 100_000, "only {played} sequences played"); + } + /// From a futex waiter an unmap left keyed on a frame and a walk begun /// with no mapper left, every sequence of five steps gives back no frame /// a waiter is keyed on: a page marked while the sweep has its lock From 54c90cc90f85dbabbff2190203ff24bf99ae5573 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 05:15:45 +0200 Subject: [PATCH 29/34] Review r8 of #1269: a walk sweeps the futex waits only of an object a mapper ever linked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's #1223 took the walk of all 64 futex buckets off unmaps. Step 2's revocation walk put it back on every removal of present pages from any paged object: once per 64 frames on each withdraw, fail, shrink and release. That covered `/tmp` deletes and the cache's withdraws, nearly all of whose objects were never mapped. Only a mapping's leaf can key a futex wait on a frame of a paged object. A pager may not map its objects, and a supply copies into fresh frames. An `Object` now records whether a mapper ever linked. `Object::link` sets the bit under the pager's lock and answers the epoch the new node is seen at, in place of the `epoch` getter, so no node links without setting it. The bit never resets, because an unmap clears its leaves without ending a wait keyed through them (B8's case). `Pages::walk` asks for `Next::Sweep` only where the bit is set, by the same rule as before: again whenever a pass began since the last sweep. The last sweep's epoch is now a `u64`, 0 before the first, in place of an `Option` reset at each walk's start. Every walk's epoch is at least 1, so the reset was redundant. This keeps `Object` at 88 bytes with the new bit, so the gate costs no slot space (`Entry` 120 bytes, slot 136, on both kernel targets). The model's `a_walk_begun_before_any_mapping_gives_back_no_frame_a_waiter_is_keyed_on` explores five steps from a walk begun on an object no mapper has linked. A mapper links during that walk, maps, waits, and queues another pass. The futex oracle stays green. Each control is a checked patch with the source restored byte-identical (https://github.com/ToyOSOrg/evidence/tree/a5fe696e711c2715f4a1bdc0dfa952c1759993fe/r9/controls): - Gating on a mapper linked now rather than ever is red on the oracle in four tests, among them `a_waiter_an_unmap_left_is_never_on_a_frame_given_back` and `every_interleaving_…`. - A bit never set is red in eleven tests. - A link during a walk left uncounted is red in the new exploration and in `long_runs_…`. - The faithful pre-r3 rule under the `u64` epoch is red in the same two tests as r4's control. B16: `end_futex_waits` names `revoke_sorted`, which walks every bucket, rather than the gated per-space `revoke_frames`. `holders` says it counts the spaces mapping the object. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/paged.rs | 94 ++++++++++++++++++++++---------------- kernel/src/object/paged.rs | 11 +++-- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index b72c7ee91b..05d9f782b9 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -14,9 +14,9 @@ //! reads as absent, takes no supply or write, and keeps its slot and frame, //! so no page is ever two frames at once. One walk per object visits each of //! its mappers ([`Mappers`]) after the marking, and a removal that finds a -//! walk running is taken by that walk's next pass; after the last pass and a -//! sweep of the futex waits on them, the walk's end gives every revoking -//! page's frame out ([`Pages::walk`]). +//! walk running is taken by that walk's next pass; after the last pass and, +//! for an object a mapper ever linked, a sweep of the futex waits on them, +//! the walk's end gives every revoking page's frame out ([`Pages::walk`]). //! //! **A dirty page holds a slot until a write-back cleans it.** Slots are //! bounded by `dirty_ceiling`. A write-back begin clears the dirty bit and @@ -268,7 +268,8 @@ pub enum Supplied { pub enum Next { /// Visit this mapper: clear its leaves of the walk's set. Visit(N), - /// End every futex wait on a frame of the walk's set. + /// End every futex wait on a frame of the walk's set: only of an object + /// a mapper ever linked ([`Object::link`]). Sweep, /// The walk ended: the set's frames were given out. Finished, @@ -383,9 +384,13 @@ pub struct Object { walk: Walk, /// The epoch whose pass last ended the walk. finished: u64, - /// The epoch at which the running walk was last told to sweep - /// ([`Next::Sweep`]). - swept: Option, + /// The epoch at which a walk was last told to sweep ([`Next::Sweep`]): + /// 0, which no walk's pass is, before the first. + swept: u64, + /// A mapper ever linked. Only a mapping's leaf keys a futex wait on one + /// of its frames, and an unmap clears its leaves without ending such a + /// wait, so this never resets. + linked: bool, } /// What a holder's read began at ([`Object::read_begin`]). @@ -454,9 +459,10 @@ impl Object { Ok(()) } - /// The epoch a mapper linking now is seen at: the walk running passes it - /// by, since nothing it maps from now on is revoking. - pub fn epoch(&self) -> u64 { + /// A mapper links: the epoch it is seen at, which the walk running + /// passes by, since nothing it maps from now on is revoking. + pub fn link(&mut self) -> u64 { + self.linked = true; self.epoch } @@ -470,7 +476,6 @@ impl Object { Walk::Idle => { self.epoch += 1; self.walk = Walk::Running { again: false }; - self.swept = None; Revoke::Walk } Walk::Running { .. } => { @@ -579,7 +584,8 @@ impl Pages { revoking: 0, revoke_head: NONE, revoke_tail: NONE, - swept: None, + swept: 0, + linked: false, epoch: 0, walk: Walk::Idle, finished: 0, @@ -737,12 +743,12 @@ impl Pages { /// The next step of `o`'s walk, after pages were marked revoking /// ([`Object::revoke`]): visit a mapper not visited in this pass, a new /// pass begun where a removal queued more pages meanwhile; once every - /// mapper has been visited, have every futex wait on the walk's set - /// ended, again whenever a pass has begun since it was last asked, since - /// a page marked during a sweep may be waited on through a leaf an unmap - /// already cleared; and once both are done, end the walk here - /// ([`Next::Finished`]): each revoking page leaves, its frame given out - /// to `out` with its page and what it became. + /// mapper has been visited, where a mapper ever linked, have every futex + /// wait on the walk's set ended, again whenever a pass has begun since it + /// was last asked, since a page marked during a sweep may be waited on + /// through a leaf an unmap already cleared; and once both are done, end + /// the walk here ([`Next::Finished`]): each revoking page leaves, its + /// frame given out to `out` with its page and what it became. pub fn walk( &mut self, o: &mut Object, @@ -760,8 +766,8 @@ impl Pages { o.epoch += 1; o.walk = Walk::Running { again: false }; } - Walk::Running { again: false } if o.swept != Some(o.epoch) => { - o.swept = Some(o.epoch); + Walk::Running { again: false } if o.linked && o.swept != o.epoch => { + o.swept = o.epoch; return Next::Sweep; } Walk::Running { again: false } => { @@ -1122,13 +1128,13 @@ mod tests { fn visited(&mut self, (): (), _: u64) {} } - /// The frames of `o`'s revoking pages, once a walk over no mappers ends. + /// The frames of `o`'s revoking pages, once a walk over no mappers ends: + /// none ever linked, so it sweeps nothing. fn settle(p: &mut Pages, o: &mut Object) -> Vec { let mut freed = Vec::new(); match o.revoke() { Revoke::Done => {} Revoke::Walk => { - assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Sweep); assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); } Revoke::Queued(_) => panic!("a walk ran with no walker"), @@ -1142,8 +1148,8 @@ mod tests { struct Ring(std::collections::VecDeque<(u32, u64)>); impl Ring { - fn link(&mut self, space: u32, o: &Object) { - self.0.push_back((space, o.epoch())); + fn link(&mut self, space: u32, o: &mut Object) { + self.0.push_back((space, o.link())); } } @@ -1170,6 +1176,8 @@ mod tests { fn a_page_marked_during_the_sweep_is_swept_before_its_frame_goes_back() { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(1, 2 * PAGE); + // A mapper linked, and its unmap since cleared its leaves. + o.link(); p.supply(&mut o, 0, 100, false, PAGE).unwrap(); p.supply(&mut o, 1, 101, false, 2 * PAGE).unwrap(); assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); @@ -1217,7 +1225,6 @@ mod tests { assert_eq!(p.revoking(&o, &mut at, &mut batch), 0); assert_eq!(o.revoke(), Revoke::Walk); let mut freed = Vec::new(); - assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Sweep); assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); assert_eq!(freed, [107, 102, 111, 100]); assert_eq!(p.revoking(&o, &mut Cursor::Start, &mut batch), 0, "the walk's end empties its set"); @@ -1231,8 +1238,8 @@ mod tests { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(1, 2 * PAGE); let mut ring = Ring::default(); - ring.link(10, &o); - ring.link(11, &o); + ring.link(10, &mut o); + ring.link(11, &mut o); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); assert_eq!(p.lookup(&o, 0), Lookup::Absent); @@ -1248,7 +1255,7 @@ mod tests { assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); assert_eq!(freed, [(7, 0, Revoked::Absent)]); - assert!(o.finished(o.epoch())); + assert!(o.empty()); assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Installed)); assert_eq!(o.revoke(), Revoke::Done, "nothing revoking"); } @@ -1261,8 +1268,8 @@ mod tests { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(1, 2 * PAGE); let mut ring = Ring::default(); - ring.link(10, &o); - ring.link(11, &o); + ring.link(10, &mut o); + ring.link(11, &mut o); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.supply(&mut o, 1, 8, false, 2 * PAGE).unwrap(); p.withdraw(&mut o, 0, 1, &mut Nobody); @@ -1270,7 +1277,7 @@ mod tests { let mut freed = Vec::new(); let mut out = |f, page, becomes| freed.push((f, page, becomes)); assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); - ring.link(12, &o); + ring.link(12, &mut o); p.fail(&mut o, 1, 1).unwrap(); let Revoke::Queued(second) = o.revoke() else { panic!("a walk runs") }; assert!(!o.finished(second)); @@ -1293,13 +1300,13 @@ mod tests { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(1, PAGE); let mut ring = Ring::default(); - ring.link(10, &o); + ring.link(10, &mut o); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.withdraw(&mut o, 0, 1, &mut Nobody); assert_eq!(o.revoke(), Revoke::Walk); let mut out = |_, _, _| {}; assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); - ring.link(11, &o); + ring.link(11, &mut o); assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); } @@ -1317,7 +1324,6 @@ mod tests { p.resize(&mut o, PAGE); assert_eq!(o.revoke(), Revoke::Walk); let mut ended = Vec::new(); - assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), Next::Sweep); assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), Next::Finished); assert_eq!(ended, [(7, 0, Revoked::Failed), (8, 1, Revoked::Removed)]); assert_eq!((p.lookup(&o, 0), p.used()), (Lookup::Failed, 1)); @@ -1750,7 +1756,6 @@ mod tests { assert_eq!(p.withdraw(&mut taken, 0, 1, &mut Nobody).0, 1); assert_eq!(taken.revoke(), Revoke::Walk); assert!(!want.unreported(), "named only once no mapping reaches the page"); - assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), Next::Sweep); assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), Next::Finished); assert!(!p.wait_over(&taken, 0, since), "a withdraw leaves the read waiting"); assert!(want.unreported(), "and its walk's end names its range again"); @@ -1864,12 +1869,10 @@ mod tests { true } - /// A removal's walk, over no mapper, to its end. + /// A removal's walk, over no mapper ever linked, to its end. fn settle(&mut self) { if self.o.revoke() == Revoke::Walk { - for end in [Next::Sweep, Next::Finished] { - assert_eq!(self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()), end); - } + assert_eq!(self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()), Next::Finished); } } @@ -2048,7 +2051,7 @@ mod tests { pub(super) fn step(&mut self, step: Step) -> bool { match step { Step::Map(s) if !self.mapped[s as usize] => { - self.ring.link(s, &self.o); + self.ring.link(s, &mut self.o); self.mapped[s as usize] = true; } Step::Unmap(s) if self.mapped[s as usize] => { @@ -2345,6 +2348,17 @@ mod tests { assert!(played > 1_000, "only {played} sequences played"); } + /// From a walk begun on an object no mapper has linked, which ends with + /// no sweep, every sequence of five steps gives back no frame a futex + /// waiter is keyed on: a mapper linked during the walk maps nothing + /// revoking, and the walk sweeps the pages removed once it has linked. + #[test] + fn a_walk_begun_before_any_mapping_gives_back_no_frame_a_waiter_is_keyed_on() { + use mapped::Step; + let played = explore(&[Step::Supply(0), Step::Supply(1), Step::Withdraw(0)], 5); + assert!(played > 10_000, "only {played} sequences played"); + } + /// A fault whose request the pager read and answered, before the fault /// looked again, meets a shrink and a grow: the version moved, so it /// asks afresh and is heard. diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 8dc19c4fbb..cdfdb6f23f 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -112,7 +112,8 @@ struct Entry { /// The pager's own reference, while unreleased, each holder object's, and /// each owner read's, while it runs: the slot is free again with the last. refs: u32, - /// Holder objects whose handles have not all gone. + /// Holder objects whose handles have not all gone, and spaces mapping it: + /// a mapping holds the object as a handle does. holders: u32, /// Not released: owner calls reach it. owned: bool, @@ -431,8 +432,10 @@ impl PagerObject { } /// Every futex wait on a frame of the walk's set at `slot` ends - /// ([`Next::Sweep`]): the frames read a batch at a time under the pager's - /// lock, and the waits ended outside it (`sched::futex::revoke_frames`). + /// ([`Next::Sweep`], asked only of an object a mapper ever linked): the + /// frames read a batch at a time under the pager's lock, and the waits + /// ended outside it in every bucket, since a wait in any space that mapped + /// the object can be keyed on them (`sched::futex::revoke_sorted`). /// An unmap, `discard`, `protect` to none or a space's end clears a leaf /// without ending the waits on a frame it does not own, so a waiter keyed /// on a file's frame through such a leaf is ended here, before the frame @@ -1581,7 +1584,7 @@ impl PagerObject { // SAFETY: a fresh node no list holds yet, and the tail's, linked, // under the lock. unsafe { - *(*at).link.get() = Link { seen: entry.object.epoch(), prev: entry.mappers.tail, next: ptr::null() }; + *(*at).link.get() = Link { seen: entry.object.link(), prev: entry.mappers.tail, next: ptr::null() }; match entry.mappers.tail.is_null() { true => entry.mappers.head = at, false => (*(*entry.mappers.tail).link.get()).next = at, From 54d6a6a07be5dde4225375159470d2cd2ff849ca Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 05:15:45 +0200 Subject: [PATCH 30/34] Review r8 of #1269: the /tmp record's slot is measured again, at 136 bytes B17. A pager's object slot grew with step 2, from 72 bytes to 136. `Entry` is now 120 bytes rather than 56: the walk state and the list of mappers take the extra 64. The record's charge per `/tmp` file, and its million-file figure, said 72. They now say 136 and about 136 MB. The sizes were measured by the compiler for `x86_64-unknown-none` and `aarch64-unknown-none-softfloat` with a compile-time probe, applied as a patch and reverted. The same probe gives 72 at main d1cefbaf9: https://github.com/ToyOSOrg/evidence/tree/a5fe696e711c2715f4a1bdc0dfa952c1759993fe/r9/sizes Co-Authored-By: Claude Opus 5.5 --- ...mp-servers-memory-and-deny-every-other-client.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md index df6cda5acd..49f0edb1fd 100644 --- a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md +++ b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md @@ -25,13 +25,14 @@ names the client that took it (`issues/no-physical-memory-fairness.md`). **What one client leaves charged.** The kernel makes a pager's object slots 256 at a time and keeps every piece for the pager's life (`kernel/pure/slots.rs`). -A freed slot is taken again, but its piece never goes back. A slot is 72 -bytes: a 56-byte entry and a 12-byte mark, measured with a host replica of -the layouts: -https://github.com/ToyOSOrg/evidence/tree/f81e4b2ddb7d89a92f82f9eb8da1fdf58587a563/pr-1273/sizes -So the server's job stays charged 72 bytes for each `/tmp` file of the most +A freed slot is taken again, but its piece never goes back. A slot is 136 +bytes: a 120-byte entry and a 12-byte mark, padded. A mapped object's walk +state and list of mappers take 64 of the entry's bytes. Measured by the +compiler for both kernel targets: +https://github.com/ToyOSOrg/evidence/tree/a5fe696e711c2715f4a1bdc0dfa952c1759993fe/r9/sizes +So the server's job stays charged 136 bytes for each `/tmp` file of the most it held at once, until the server ends. A client that makes and removes a -million files leaves about 72 MB. The manual has unused backing return under +million files leaves about 136 MB. The manual has unused backing return under pressure; this never returns. Owner: the file server and the kernel's paged objects, once the owner has From ecfb2248afcf8f9a07fad10670eae401aed389c2 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 05:45:14 +0200 Subject: [PATCH 31/34] Review r9 of #1269: the index's docs name what is left, and the /tmp record says who measures again B19. The `Mappers` doc pointed at `Object::epoch`, which r8 replaced with `Object::link`. `Pages::object` said a released object's id may be reused because `release` takes its pages, but `release` went in the composition with #1273. The docs now say a node is seen at the epoch `link` answers, and an id is reused once its object is empty: `invalidate` marks its pages and the walk's end takes them. B20. The /tmp record now says that whichever of #1269 and #1258 lands second measures the combined slot again, updates the figures and deletes the sentence. #1258 rewrites the same paragraph for its own slot growth. S1, S2: `Entry.refs` says only what it keeps: the slot, free with the last reference. It also counts mappings, walks, queued removals and writes, which the old list missed. The record says every entry carries the 64 bytes of mapping state, not only a mapped object's. Doc comments and a record only. Co-Authored-By: Claude Opus 5.5 --- ...the-tmp-servers-memory-and-deny-every-other-client.md | 8 +++++--- kernel/pure/paged.rs | 9 +++++---- kernel/src/object/paged.rs | 3 +-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md index 49f0edb1fd..fdec47dfd8 100644 --- a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md +++ b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md @@ -26,13 +26,15 @@ names the client that took it (`issues/no-physical-memory-fairness.md`). **What one client leaves charged.** The kernel makes a pager's object slots 256 at a time and keeps every piece for the pager's life (`kernel/pure/slots.rs`). A freed slot is taken again, but its piece never goes back. A slot is 136 -bytes: a 120-byte entry and a 12-byte mark, padded. A mapped object's walk -state and list of mappers take 64 of the entry's bytes. Measured by the +bytes: a 120-byte entry and a 12-byte mark, padded. The walk state and list +of mappers that mapping needs take 64 bytes of every entry. Measured by the compiler for both kernel targets: https://github.com/ToyOSOrg/evidence/tree/a5fe696e711c2715f4a1bdc0dfa952c1759993fe/r9/sizes So the server's job stays charged 136 bytes for each `/tmp` file of the most it held at once, until the server ends. A client that makes and removes a -million files leaves about 136 MB. The manual has unused backing return under +million files leaves about 136 MB. #1258 grows the entry too: whichever of +#1269 and #1258 lands second measures the combined slot again, updates these +figures and deletes this sentence. The manual has unused backing return under pressure; this never returns. Owner: the file server and the kernel's paged objects, once the owner has diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 05d9f782b9..0b58006663 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -295,8 +295,8 @@ pub enum Revoked { } /// The mappers of one object, one node per space that maps it, in the order -/// a walk visits them. A node links at the tail, seen at -/// [`Object::epoch`] then, and leaves only once its space maps none of the +/// a walk visits them. A node links at the tail, seen at the epoch +/// [`Object::link`] answers, and leaves only once its space maps none of the /// object's frames. pub trait Mappers { type Node: Copy; @@ -567,8 +567,9 @@ impl Pages { } /// An object of `size` bytes, at most [`LAST_END`], named `id`: an id no - /// live object of this table has, which a released one's may be, since - /// [`release`](Self::release) takes every page it held. + /// live object of this table has. A released object's may be, once it is + /// [`empty`](Object::empty): [`invalidate`](Self::invalidate) marks every + /// page it held, and its walk's end takes them. pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); Object { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index cdfdb6f23f..1f83ed39b4 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -109,8 +109,7 @@ struct Request { /// One object of a pager's, at its id's slot. struct Entry { object: Object, - /// The pager's own reference, while unreleased, each holder object's, and - /// each owner read's, while it runs: the slot is free again with the last. + /// References that keep the slot: it is free again with the last. refs: u32, /// Holder objects whose handles have not all gone, and spaces mapping it: /// a mapping holds the object as a handle does. From 9a29a356f35d147c3843de0ffeec8c9a4b847d4a Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 08:08:11 +0200 Subject: [PATCH 32/34] A holder's write from a mapped page not yet supplied gives back its frames while it waits #1258's holder write copies its bytes from user memory into frames it drew from its pager's core, before it takes the write side. The source may be a page of a mapping that is absent. The copy then parked on the page's pager while it still held the frames, and frames are what that pager's supply draws. If holders' writes sourced from a mapping of the same pager drew the core dry, the pager could not supply the page they wait for, and none would end. This is the orchestrator's ruling for pinned copies, that no wait holds what its pager needs, and the staging copy now takes the same mode. It runs under `user_ptr::Pinned`, so an absent page stops it. The write gives every frame back and waits for the source pages holding nothing (`wait_in`). It then draws and copies again, against the object as it stands then. A copy that faults answers its own cause, `BadAddress` or `Io`, as every other copy since B5 does, where #1258 flattened it to `BadAddress`. `Pinned`, `Copies` and the fault handler's docs say what the mode guards: a copy that holds what no wait may hold, page pins or a pager's frames. `mapprobe`'s `written` case is the duty the memory record kept for whichever of #1258 and #1269 landed second. The case: - endows a mapper with a write holder of an empty object granted a page of credit; - the mapper writes it from the first page of its mapping, which no one has supplied; - the pager hears the request and supplies the run; - the mapper's write lands whole, and the pager reads the page back as the supplied bytes. The record's paragraph on the owed test goes. Co-Authored-By: Claude Opus 5.5 --- ...b-pages-still-copies-each-programs-text.md | 7 -- kernel/src/object/paged.rs | 104 +++++++++++------- kernel/src/process.rs | 2 +- kernel/src/user_ptr.rs | 20 ++-- tests/guest/mapprobe/src/main.rs | 33 ++++++ tests/toyos.rs | 1 + 6 files changed, 110 insertions(+), 57 deletions(-) diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index 5e767029bf..5adb4efab7 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -170,13 +170,6 @@ pager's capacity, judged against Linux on metal, and per-request wakes where it shows the cost. A cold DATA page's miss in stages is the same kind of row and waits with it. -The pinned-copy mode (a copy holding a page pin stops at an absent page, lets -its pins go, waits holding nothing and copies again) is reached by no process -until #1258's holder write, which copies from user memory under a write pin. -Its guest test, a holder write sourced from a mapped file's absent page that -completes, lands with whichever of #1258 and #1269 lands second (owner -map7b). - Step 2 measures a cached page's fault and a 4 KiB read through a mapping against Linux (osbench `fs.*.map_touch_16m`, `fs.*.map_read_4k`). Owed, by map7b, with step 3's metal: a cold DATA page's miss in stages, one object diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 0632e6cf8d..5aeed75e48 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -45,9 +45,10 @@ //! holder's calls, so no pager waits on a pager. //! //! **Lock order.** A space's lock, then the pager's, then the allocator's. -//! A copy to or from user memory holds pins, never the lock; no wait holds a -//! pin, so such a copy stops at an absent page (`user_ptr::Pinned`), lets its -//! pins go, and waits for the page holding nothing. +//! A copy to or from user memory holds pins, or a holder's write the frames +//! it drew, never the lock; no wait holds either, so such a copy stops at an +//! absent page (`user_ptr::Pinned`), lets them go, and waits for the page +//! holding nothing. //! //! **What a pager holds** (`issues/a-pager-holds-a-fixed-capacity-until-the-memory-service-reclaims-its-clean-pages.md`): //! its core — `dirty` pages and a read's headroom — committed at its making @@ -1261,8 +1262,10 @@ impl PagerObject { } /// A holder's write of `src` at `at`, or at the size where `None`: whole - /// or refused, and never waiting. Its bytes are copied into frames held - /// aside first, with nothing else held; then, in one hold of the lock, it + /// or refused, and never waiting for the write side. Its bytes are copied + /// into frames drawn first, with nothing else held, and a page of `src` + /// that is absent is waited for with the frames given back + /// (`user_ptr::Pinned`); then, in one hold of the lock, it /// takes the write side, refused while anyone holds or waits for it, and /// is checked and admitted whole ([`Pages::holder_fits`]). Its copy into /// the object's frames is the kernel's own, so whoever waits for the side @@ -1286,50 +1289,73 @@ impl PagerObject { let Some(_writing) = HolderWriting::begin(self, slot) else { return SyscallError::Gone.to_u64(); }; - // Frames for the bytes, and for the pages the write would install, - // as the object stands now: one fewer than it needs then refuses it. - let fresh_count = { - let mut state = self.state.lock(); - if state.gone { - return SyscallError::Gone.to_u64(); - } - let (pages, object) = state.split(slot); - let offset = at.unwrap_or(object.size()); - let span = match holder_span(offset, len as u64) { - Ok(span) => span, - Err(why) => return holder_word(why).to_u64(), + let fresh_count = loop { + // Frames for the bytes, and for the pages the write would install, + // as the object stands now: one fewer than it needs then refuses it. + let fresh_count = { + let mut state = self.state.lock(); + if state.gone { + return SyscallError::Gone.to_u64(); + } + let (pages, object) = state.split(slot); + let offset = at.unwrap_or(object.size()); + let span = match holder_span(offset, len as u64) { + Ok(span) => span, + Err(why) => return holder_word(why).to_u64(), + }; + let fresh_count = match pages.holder_fits(object, offset / PAGE, span) { + Ok(fresh) => fresh as usize, + Err(why) => return holder_word(why).to_u64(), + }; + let wanted = staged_count + fresh_count; + let got = self.draw(&mut state, &mut drawn[..wanted]); + if got < wanted { + self.give_back_drawn(&mut state, &drawn[..got]); + return SyscallError::WouldBlock.to_u64(); + } + fresh_count }; - let fresh_count = match pages.holder_fits(object, offset / PAGE, span) { - Ok(fresh) => fresh as usize, - Err(why) => return holder_word(why).to_u64(), + for (k, frame) in staged.iter_mut().take(staged_count).enumerate() { + *frame = materialize(drawn[k], false); + } + for (k, frame) in fresh.iter_mut().take(fresh_count).enumerate() { + *frame = materialize(drawn[staged_count + k], true); + } + // The bytes come in while the write holds frames of its pager's, + // which no wait may hold: a page of `src` that is absent stops the + // copy, the frames go back, and the write waits for the page + // holding nothing and draws again. + let pinned = Pinned::begin(); + let mut stopped = None; + for (k, &frame) in staged[..staged_count].iter().enumerate() { + let n = (len - k * PAGE as usize).min(PAGE as usize); + // SAFETY: a frame this write drew, which nothing else reaches. + if let Err(why) = unsafe { pinned.read(src, k * PAGE as usize, DirectMap::from_phys(frame).as_mut_ptr::(), n) } { + stopped = Some((k * PAGE as usize, why)); + break; + } + } + drop(pinned); + let Some((from, why)) = stopped else { break fresh_count }; + { + let mut state = self.state.lock(); + self.give_frames(&mut state, &staged[..staged_count]); + self.give_frames(&mut state, &fresh[..fresh_count]); + } + let waited = match why { + Stopped::Absent => src.wait_in(from, len - from), + Stopped::Faulted(faulted) => Err(faulted), }; - let wanted = staged_count + fresh_count; - let got = self.draw(&mut state, &mut drawn[..wanted]); - if got < wanted { - self.give_back_drawn(&mut state, &drawn[..got]); - return SyscallError::WouldBlock.to_u64(); + if let Err(faulted) = waited { + return SyscallError::from(faulted).to_u64(); } - fresh_count }; - for (k, frame) in staged.iter_mut().take(staged_count).enumerate() { - *frame = materialize(drawn[k], false); - } - for (k, frame) in fresh.iter_mut().take(fresh_count).enumerate() { - *frame = materialize(drawn[staged_count + k], true); - } let give_up = |e: SyscallError| { let mut state = self.state.lock(); self.give_frames(&mut state, &staged[..staged_count]); self.give_frames(&mut state, &fresh[..fresh_count]); e.to_u64() }; - for (k, &frame) in staged[..staged_count].iter().enumerate() { - let n = (len - k * PAGE as usize).min(PAGE as usize); - // SAFETY: a frame this write drew, which nothing else reaches. - if unsafe { src.read_raw(k * PAGE as usize, DirectMap::from_phys(frame).as_mut_ptr::(), n) }.is_err() { - return give_up(SyscallError::BadAddress); - } - } let mut pinned = [(0u64, 0u64, 0usize, 0usize); HOLDER]; let (offset, count, used) = { let mut state = self.state.lock(); diff --git a/kernel/src/process.rs b/kernel/src/process.rs index b7e7c57f99..6d9889a1ba 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -1003,7 +1003,7 @@ impl Faulted { /// Serve a fault at `fault_addr` for `access` in the running thread's space, /// at its trap's depth with interrupts on: its page populated under the /// space's lock, or, for a paged page that is absent, waited for holding no -/// lock and touched again, unless the fault is a copy's that holds page pins. +/// lock and touched again, unless the fault is a [`crate::user_ptr::Pinned`] copy's. /// While it waits the CPU's fault mark is down, so another fault on this CPU /// meanwhile is no recursion; it is up again on whichever CPU the thread /// wakes on. diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 85be05b8ef..21381b7665 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -121,9 +121,9 @@ pub enum Stop { Absent, } -/// A thread's user copies: whether it holds page pins across them, and what -/// stopped its last one short. Written only by the thread itself, by its -/// copies and their faults. +/// A thread's user copies: whether they run holding what no wait may hold +/// ([`Pinned`]), and what stopped its last one short. Written only by the +/// thread itself, by its copies and their faults. pub struct Copies { pinned: AtomicBool, stop: AtomicU8, @@ -135,8 +135,8 @@ impl Copies { } } -/// Whether the running thread's copies hold page pins ([`Pinned`]), so its -/// copy's fault must not wait. +/// Whether the running thread's copies hold what no wait may ([`Pinned`]), +/// so its copy's fault must not wait. pub fn pinned() -> bool { crate::sched::driver::with_current_copies(|c| c.pinned.load(Ordering::Relaxed)).unwrap_or(false) } @@ -165,11 +165,11 @@ fn faulted(copied: Result<(), Stopped>) -> Result<(), Faulted> { }) } -/// The running thread's copies while it holds page pins, which no wait may -/// hold: a fault on a mapped file's absent page stops the copy -/// ([`Stopped::Absent`]) instead of waiting. The holder lets its pins go, -/// waits for the window's pages holding nothing (`wait_in`), and copies -/// again. Not `Send`: it marks the thread that made it. +/// The running thread's copies while it holds what no wait may hold, page +/// pins or frames drawn from a pager: a fault on a mapped file's absent page +/// stops the copy ([`Stopped::Absent`]) instead of waiting. The holder lets +/// them go, waits for the window's pages holding nothing (`wait_in`), and +/// copies again. Not `Send`: it marks the thread that made it. pub struct Pinned(PhantomData<*const ()>); impl Pinned { diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs index 879f01ffca..8ee07e4317 100644 --- a/tests/guest/mapprobe/src/main.rs +++ b/tests/guest/mapprobe/src/main.rs @@ -21,6 +21,10 @@ //! mapper and of the last the other, each with `SIGBUS`. The shrink revokes //! 79 pages, more than a walk reads at once, so one of the two is cleared //! by a later batch. +//! - `written`: a holder's write whose bytes come from a page of the +//! mapping that is absent lands whole once the pager supplies the page: +//! its copy holds frames of the pager's, so it lets them go and waits for +//! the page holding nothing. //! - `ends`, last: the pager's end answers a write from the mapping `Io` //! and ends the mapper at its next touch. @@ -40,6 +44,7 @@ const HOLDER: &str = "mapprobe-holder"; const READY: &str = "mapprobe-ready"; const GO: &str = "mapprobe-go"; const OUT: &str = "mapprobe-out"; +const TARGET: &str = "mapprobe-target"; /// Every page the pager holds at once, and dirty at once. const CAPACITY: u64 = 256; @@ -65,6 +70,7 @@ fn main() { killed(&mut served); hole(&mut served); shrunk(&mut served); + written(&mut served); ends(served); println!("mapprobe: ok"); } @@ -261,6 +267,25 @@ fn shrunk(served: &mut Served) { println!("mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS"); } +fn written(served: &mut Served) { + let source = served.pager.object(RUN * PAGE).expect("mapprobe: an object"); + let target = served.pager.object(0).expect("mapprobe: an object to write"); + assert_eq!(served.pager.grant(target, 1), Ok(1), "mapprobe: a page of credit"); + let writer = served.pager.holder(target, true).expect("mapprobe: a write holder"); + let mut child = served.mapper(source, "written", vec![(TARGET, writer)]); + served.requested(source, 0); + served.supply(source, 0, RUN, 0x88); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the mapper writing from its mapping"); + let mut bytes = vec![0; PAGE as usize]; + // SAFETY: `bytes` outlives the read and nothing else reaches it. + let out = unsafe { toyos::shm::Bytes::from_raw(bytes.as_mut_ptr(), bytes.len()) }; + assert_eq!(served.pager.read(target, out, 0), Ok(PAGE as usize)); + assert!(bytes.iter().all(|&b| b == 0x88), "mapprobe: the write from an absent page landed other bytes"); + served.pager.release(target).expect("mapprobe: release"); + served.pager.release(source).expect("mapprobe: release"); + println!("mapprobe: a holder's write from a mapped page not yet supplied lands whole once it is"); +} + fn ends(mut served: Served) { let object = served.pager.object(2 * PAGE).expect("mapprobe: an object"); served.supply(object, 0, 1, 0x77); @@ -348,6 +373,14 @@ fn map(case: &str) { let read = byte(at, page); panic!("mapprobe --map: page {page}, past the object's end, read {read}"); } + "written" => { + let at = mapped(RUN); + let handle: OwnedHandle = Endowments::get().take(TARGET).expect("mapprobe --map: a write holder"); + let target = Paged::received(handle).expect("mapprobe --map: a holder it may write"); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at, PAGE as usize) }; + assert_eq!(target.write(page, 0), Ok(PAGE as usize), "mapprobe --map: a write from a page not yet supplied"); + } "ends" => { let at = mapped(2); assert_eq!(byte(at, 0), 0x77); diff --git a/tests/toyos.rs b/tests/toyos.rs index d2810e0548..8e9b6a94eb 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -3633,6 +3633,7 @@ const MAPPED_FILE_SAID: &[&str] = &[ "mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing", "mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room", "mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS", + "mapprobe: a holder's write from a mapped page not yet supplied lands whole once it is", "mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch", "mapped_file: a mapped paged object follows its pager", ]; From ea33f5ad98bdb8e00979093167d809d6345ace13 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 08:08:18 +0200 Subject: [PATCH 33/34] A fault's wait and a hole's frame use their pager's own wait and draw #1258 gave the pager one wait that reports itself overdue past the report time, `wait_changed`, and one draw of frames, `draw` with `materialize`. A fault's wait and a touched hole's zeroed frame each had a copy of those. They now call the pager's. The wait still requests the fault's run and lasts until the page is no longer absent, the version moved or the pager is gone. The hole's frame still comes from the pager's core, with the same refusal where there is no room. Co-Authored-By: Claude Opus 5.5 --- kernel/src/object/paged.rs | 43 ++++++++------------------------------ 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 5aeed75e48..bb704836ad 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -1890,35 +1890,22 @@ impl Index<'_> { fn zeroed(&mut self, page: u64) -> Result { let no_room = self.lost(page, Why::NoRoom); - let job = self.pager.core.job(); - let State { pages, entries, held, gone, .. } = &mut *self.state; - held.settle(); - let phys = match held.core.draw(!held.spare.is_empty()) { - Draw::Aside => { - let phys = held.spare.pop().expect("paged: a frame drawn aside is held aside"); - // SAFETY: a frame held aside, out of the index and unpinned, - // which nothing else reaches. - unsafe { ptr::write_bytes(DirectMap::from_phys(phys).as_mut_ptr::(), 0, PAGE as usize) }; - phys - } - Draw::Pool => pmm::alloc_promised().into_raw(), - Draw::Beyond if pmm::promise(job, 1) => { - held.core.promised(); - pmm::alloc_promised().into_raw() - } - Draw::Beyond => return Err(no_room), - }; - let object = &mut entries.get_mut(self.slot).expect("paged: a mapped slot is held").object; + let mut drawn = [None]; + if self.pager.draw(self.state, &mut drawn) == 0 { + return Err(no_room); + } + let phys = materialize(drawn[0], true); + let (pages, object) = self.state.split(self.slot); match pages.supply(object, page, phys, false, (page + 1) * PAGE) { Ok(Supplied::Installed) => { - held.core.installed(); + self.state.held.core.installed(); Ok(phys) } Ok(Supplied::Present) => unreachable!("paged: a hole found under the lock held a page"), // Inside the object and the pager alive, as the lookup found it: // only a full table refuses. Err(_) => { - held.give_back(job, *gone, phys, false); + self.pager.give_frames(self.state, &[phys]); Err(no_room) } } @@ -1973,19 +1960,7 @@ impl MapNode { waiting.as_ref().request(&mut state, first, count); (first, count) }; - let ready = || over(&pager.state.lock()); - let parkable = crate::scheduler::Parkable::at_entry(); - let began = crate::clock::now(); - if watch::wait_until(&parkable, &pager.changed, 0, WaitClass::Io, Deadline::at(began + pager.report), ready).is_err() { - return false; - } - if !ready() { - let _overdue = Overdue::begin(pager, slot, first, count, began); - if watch::wait_until(&parkable, &pager.changed, 0, WaitClass::Io, Deadline::never(), ready).is_err() { - return false; - } - } - true + pager.wait_changed(slot, first, count, || over(&pager.state.lock())).is_ok() } } From 63f18b2a6eed53920b2e3bc07e5cb480b69571ec Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 12 Nov 2026 09:13:10 +0200 Subject: [PATCH 34/34] Review r12 of #1269: the /tmp record's mapping share, and three docs say what holds B1. In the combined layout, the walk state mapping needs adds 40 bytes to the object: 152 against main's 112. The record's 48 was r9's figure on the layout before #1258. The record now says 40 to the object, and with the list of mappers 56 to a slot. It also says that main's 216 was measured on x86-64. B2. `draw`'s doc no longer says its frames are completed outside the lock: `Index::zeroed` materializes its frame under it. The module header says a holder's write copies into frames drawn first, not held aside. S1. The comment over the holder write's staging copy repeated its doc and the module header, so it goes. `faulted()`'s doc and panic name a copy outside `Pinned`, which is what they mean now that `Pinned` guards a pager's frames as well as pins. S2. `mapprobe`'s `written` case says what it establishes: the write stops at the absent page, waits, draws and copies again, and lands whole with the supplied bytes. That the frames go back while it waits is the code's to show. Doc comments, a panic message and a record only. Co-Authored-By: Claude Opus 5.5 --- ...-the-tmp-servers-memory-and-deny-every-other-client.md | 7 ++++--- kernel/src/object/paged.rs | 8 ++------ kernel/src/user_ptr.rs | 4 ++-- tests/guest/mapprobe/src/main.rs | 5 ++--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md index ead3d446bd..7704d64d10 100644 --- a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md +++ b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md @@ -28,9 +28,10 @@ names the client that took it (`issues/no-physical-memory-fairness.md`). A freed slot is taken again, but its piece never goes back. A slot is 272 bytes: a 248-byte entry and two 12-byte marks, one for each notice it can owe. Of the entry, the object's own state is 152 bytes, the pager's record 64 -and the list of a mapped object's mappers 16; the walk state mapping needs -is 48 of the object's 152. Every entry carries all of it. Measured by the -compiler for both kernel targets, against 216 on main before mapping: +and the list of a mapped object's mappers 16. The walk state mapping needs +adds 40 bytes to the object, and with the list 56 to a slot. Every entry +carries all of it. Measured by the compiler for both kernel targets; main +before mapping, measured the same way on x86-64, has 216: https://github.com/ToyOSOrg/evidence/tree/94fd400ddab5d9c2ff776ac9df66f1b0bb88b768/r12/sizes So the server's job stays charged 272 bytes for each `/tmp` file of the most it held at once, until the server ends. A client that makes and removes a diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index bb704836ad..e8b03551f7 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -12,7 +12,7 @@ //! may be narrowed and sent. //! //! **One write at a time** ([`kernel::paged::Side`]): a holder's write -//! copies its bytes into frames held aside first, then takes the object's +//! copies its bytes into frames drawn first, then takes the object's //! write side in one hold of the lock, refused at once while the side is held //! or the pager waits for it, and is checked whole and admitted there; its //! copy into the object's frames is the kernel's own. The pager holds the @@ -1321,10 +1321,6 @@ impl PagerObject { for (k, frame) in fresh.iter_mut().take(fresh_count).enumerate() { *frame = materialize(drawn[staged_count + k], true); } - // The bytes come in while the write holds frames of its pager's, - // which no wait may hold: a page of `src` that is absent stops the - // copy, the frames go back, and the write waits for the page - // holding nothing and draws again. let pinned = Pinned::begin(); let mut stopped = None; for (k, &frame) in staged[..staged_count].iter().enumerate() { @@ -1443,7 +1439,7 @@ impl PagerObject { /// Under the lock: draw a frame for each of `out`'s entries from the /// core, as a supply does — one held aside, the core's promise, or a /// promise the allocator makes now: how many were drawn, each `Some` for - /// a frame held aside, which [`materialize`] completes outside the lock. + /// a frame held aside, which [`materialize`] completes. fn draw(&self, state: &mut State, out: &mut [Option]) -> usize { state.held.settle(); let mut drawn = 0; diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 21381b7665..a876dc2177 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -157,11 +157,11 @@ pub fn last_stop() -> Stopped { } } -/// A copy holding no pin: what stopped it is a fault. +/// A copy outside [`Pinned`]: what stopped it is a fault. fn faulted(copied: Result<(), Stopped>) -> Result<(), Faulted> { copied.map_err(|why| match why { Stopped::Faulted(faulted) => faulted, - Stopped::Absent => panic!("user_ptr: a copy holding no pin stopped short of an absent page"), + Stopped::Absent => panic!("user_ptr: a copy outside Pinned stopped short of an absent page"), }) } diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs index 8ee07e4317..c3e0235289 100644 --- a/tests/guest/mapprobe/src/main.rs +++ b/tests/guest/mapprobe/src/main.rs @@ -22,9 +22,8 @@ //! 79 pages, more than a walk reads at once, so one of the two is cleared //! by a later batch. //! - `written`: a holder's write whose bytes come from a page of the -//! mapping that is absent lands whole once the pager supplies the page: -//! its copy holds frames of the pager's, so it lets them go and waits for -//! the page holding nothing. +//! mapping that is absent lands whole, with those bytes, once the pager +//! supplies the page: its copy stops there, waits, draws and copies again. //! - `ends`, last: the pager's end answers a write from the mapping `Io` //! and ends the mapper at its next touch.