From 55fafd6580dd7d98b952c617b1cb57ab233cbf1a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 17:53:07 +0200 Subject: [PATCH 1/3] The inventory keeps its gaps once, by length, and inserts and removes in place The free ranges were held twice: by start in one map and by length in a set, both rewritten on every insert and removal, though the entries already say where every gap is. Now only the set by `(length, start)` stays, the one placement needs; an insert or removal finds the gap or gaps it changes from the neighbouring entries. An insert joins the neighbours it continues directly, from the two lookups it makes anyway, rather than inserting and then walking every entry around the range; a removal takes its entries one at a time without first listing their starts. A map and unmap of one page so makes about half the tree lookups and one allocation where it made two. On the host (load average about 90, min of 300 batches), one page placed, inserted and removed in a space of 16 to 512 ranges: the insert path 116-176 ns to 50-82, the removal 148-177 to 105-145; allocations per pair 2 to 1. Driven with the inventory it replaces through 6,000,000 random changes, every answer and the whole state after each were equal; two seeded defects were caught. The mixed-load test now asks every placement's exact answer of a page map. Co-Authored-By: Claude Opus 5.5 --- kernel/pure/vm.rs | 144 ++++++++++++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 63 deletions(-) diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index d98bab73c37..a48b1bf0d53 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -4,8 +4,8 @@ //! //! Ranges are whole 4 KiB pages inside `[floor, top)`, never overlapping. //! What is not an entry is free, and placement looks at nothing else: the -//! free ranges are kept by address and by length, so finding room, cutting a -//! range out and inserting one are logarithmic in the entries. +//! free ranges, the gaps the entries leave, are kept by length, so finding +//! room, cutting a range out and inserting one are logarithmic in the entries. //! //! **Every change is all-or-nothing.** A caller asks what a change would do //! before making it ([`Inventory::covered`], [`Inventory::is_free`], @@ -157,19 +157,15 @@ pub struct Inventory { floor: u64, top: u64, entries: BTreeMap>, - /// Free ranges by start, each to its end. - free: BTreeMap, - /// The same ranges by `(length, start)`. - by_len: BTreeSet<(u64, u64)>, + /// Every gap in `[floor, top)` between entries, by `(length, start)`. + free: BTreeSet<(u64, u64)>, } impl Inventory { /// Nothing held in `[floor, top)`, both on pages. pub fn new(floor: u64, top: u64) -> Self { assert!(floor.is_multiple_of(PAGE) && top.is_multiple_of(PAGE) && floor < top, "vm: {floor:#x}..{top:#x}"); - let mut inventory = Self { floor, top, entries: BTreeMap::new(), free: BTreeMap::new(), by_len: BTreeSet::new() }; - inventory.add_free(floor, top); - inventory + Self { floor, top, entries: BTreeMap::new(), free: BTreeSet::from([(top - floor, floor)]) } } pub fn floor(&self) -> u64 { @@ -244,25 +240,43 @@ impl Inventory { pub fn place(&self, len: u64, align: u64) -> Option { debug_assert!(len.is_multiple_of(PAGE) && len > 0 && align.is_power_of_two() && align >= PAGE); let need = len.checked_add(align - PAGE)?; - let &(_, start) = self.by_len.range((need, 0)..).next()?; + let &(_, start) = self.free.range((need, 0)..).next()?; Some(start.next_multiple_of(align)) } - /// Hold `[start, entry.end)`, which is free. + /// Hold `[start, entry.end)`, which is free. It joins a neighbour it + /// continues. pub fn insert(&mut self, start: u64, entry: Entry) { - assert!(self.holds(start, entry.end), "vm: {start:#x}..{:#x} is not pages of this space", entry.end); - let (free_start, free_end) = self.free.range(..=start).next_back().map(|(&s, &e)| (s, e)).expect("vm: inserted over held memory"); - assert!(entry.end <= free_end, "vm: {start:#x}..{:#x} inserted over held memory", entry.end); - self.take_free(free_start, free_end); - if free_start < start { - self.add_free(free_start, start); + let end = entry.end; + assert!(self.holds(start, end), "vm: {start:#x}..{end:#x} is not pages of this space"); + let (hi, joins_next) = match self.entries.range(start..).next() { + Some((&next, following)) => (next, next == end && entry.continued_by(start, following)), + None => (self.top, false), + }; + let prev = self.entries.range_mut(..start).next_back(); + let lo = prev.as_ref().map_or(self.floor, |(_, e)| e.end); + assert!(lo <= start && end <= hi, "vm: {start:#x}..{end:#x} inserted over held memory"); + self.free.remove(&(hi - lo, lo)); + if lo < start { + self.free.insert((start - lo, lo)); } - if entry.end < free_end { - self.add_free(entry.end, free_end); + if end < hi { + self.free.insert((hi - end, end)); + } + let joined = match prev { + Some((&s, before)) if lo == start && before.continued_by(s, &entry) => { + before.end = end; + s + } + _ => { + self.entries.insert(start, entry); + start + } + }; + if joins_next { + let following = self.entries.remove(&end).expect("vm: the range just found"); + self.entries.get_mut(&joined).expect("vm: the range just held").end = following.end; } - let end = entry.end; - self.entries.insert(start, entry); - self.join(start, end); } /// Take every entry out of `[start, end)`, cutting any that crosses its @@ -270,11 +284,24 @@ impl Inventory { pub fn remove(&mut self, start: u64, end: u64) -> Vec<(u64, Entry)> { self.split_at(start); self.split_at(end); - let starts: Vec = self.entries.range(start..end).map(|(&s, _)| s).collect(); - let removed: Vec<(u64, Entry)> = - starts.into_iter().map(|s| (s, self.entries.remove(&s).expect("vm: a start just listed"))).collect(); - for (s, entry) in &removed { - self.release(*s, entry.end); + let lo = self.entries.range(..start).next_back().map_or(self.floor, |(_, e)| e.end); + // The gaps from `lo` up to the last entry taken join into one. + let mut gap = lo; + let mut removed = Vec::new(); + while let Some((&s, _)) = self.entries.range(start..end).next() { + let entry = self.entries.remove(&s).expect("vm: a start just found"); + if gap < s { + self.free.remove(&(s - gap, gap)); + } + gap = entry.end; + removed.push((s, entry)); + } + if !removed.is_empty() { + let hi = self.entries.range(gap..).next().map_or(self.top, |(&s, _)| s); + if gap < hi { + self.free.remove(&(hi - gap, gap)); + } + self.free.insert((hi - lo, lo)); } removed } @@ -328,32 +355,6 @@ impl Inventory { } } } - - /// `[start, end)` was held and is free now. - fn release(&mut self, start: u64, end: u64) { - let (mut lo, mut hi) = (start, end); - if let Some((&s, &e)) = self.free.range(..start).next_back() { - if e == start { - self.take_free(s, e); - lo = s; - } - } - if let Some(&e) = self.free.get(&end) { - self.take_free(end, e); - hi = e; - } - self.add_free(lo, hi); - } - - fn add_free(&mut self, start: u64, end: u64) { - self.free.insert(start, end); - self.by_len.insert((end - start, start)); - } - - fn take_free(&mut self, start: u64, end: u64) { - self.free.remove(&start); - self.by_len.remove(&(end - start, start)); - } } #[cfg(test)] @@ -473,6 +474,12 @@ mod tests { v.insert(FLOOR + 11 * PAGE, object(FLOOR + 12 * PAGE, 7, 2 * PAGE)); v.insert(FLOOR + 12 * PAGE, object(FLOOR + 13 * PAGE, 7, 3 * PAGE)); assert_eq!(v.len(), 3); + // A range that fills the hole between two it continues makes one of the three. + v.insert(FLOOR + 20 * PAGE, private(FLOOR + 21 * PAGE, Prot::Read)); + v.insert(FLOOR + 22 * PAGE, private(FLOOR + 23 * PAGE, Prot::Read)); + v.insert(FLOOR + 21 * PAGE, private(FLOOR + 22 * PAGE, Prot::Read)); + assert_eq!(v.find(FLOOR + 20 * PAGE).map(|(s, e)| (s, e.end)), Some((FLOOR + 20 * PAGE, FLOOR + 23 * PAGE))); + assert_eq!(v.len(), 4); } #[test] @@ -605,17 +612,28 @@ mod tests { let ((s, a), (t, b)) = (pair[0], pair[1]); assert!(!(a.end == t && a.continued_by(s, b)), "round {round}: {s:#x} and {t:#x} were left apart"); } - // The free index is exactly the gaps. - let longest_gap = { - let (mut best, mut run) = (0, 0); - for p in &map { - run = if p.is_none() { run + 1 } else { 0 }; - best = best.max(run); + // The free index is exactly the gaps: a placement is the smallest + // gap that holds it at any alignment, the lowest of equals. + let mut gaps: Vec<(u64, u64)> = Vec::new(); + for (p, held) in map.iter().enumerate() { + match (held, gaps.last_mut()) { + (Some(_), _) => {} + (None, Some((s, len))) if *s + *len == p as u64 => *len += 1, + (None, _) => gaps.push((p as u64, 1)), } - best - }; - assert!(longest_gap == 0 || v.place(longest_gap * PAGE, PAGE).is_some(), "round {round}"); - assert_eq!(v.place((longest_gap + 1) * PAGE, PAGE), None, "round {round}"); + } + let longest = gaps.iter().map(|&(_, len)| len).max().unwrap_or(0); + for (n, align) in [(1, 1), (2, 1), (3, 4), (8, 1), (5, 16), (longest, 1), (longest + 1, 1)] { + if n == 0 { + continue; + } + let want = gaps + .iter() + .filter(|&&(_, len)| len >= n + align - 1) + .min_by_key(|&&(s, len)| (len, s)) + .map(|&(s, _)| (FLOOR + s * PAGE).next_multiple_of(align * PAGE)); + assert_eq!(v.place(n * PAGE, align * PAGE), want, "round {round}: {n} pages aligned to {align}"); + } } } From 2b3d4429c70bc8b8244b431810cde83003780bd1 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 20:39:12 +0200 Subject: [PATCH 2/3] Review r1 of #1256: a range is charged for one free index, and the record says where map/unmap's time went `range_heap` still charged each range for the by-start free map this PR deletes, and its comment named two indexes and two removal lists. It now charges a range's entry, its gap's place by length, and its share of the one list a removal gathers, at up to three times its length while that list doubles. The mixed-load test compares the free index with the page map's gaps exactly. #1223's paragraph putting the T14's `mem.map_unmap_4k` in the inventory moves here, with this PR's host split. Co-Authored-By: Claude Opus 5.5 --- ...memory-hot-paths-are-slower-than-2-mib-pages.md | 14 ++++++++++++++ kernel/pure/vm.rs | 3 +++ kernel/src/mm/space.rs | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md index d3964ab5c12..4a2918caec8 100644 --- a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md +++ b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md @@ -124,6 +124,20 @@ The faults and `mem.anon.total` are now ahead of Linux on both benches. Behind: `issues/many-osbench-rows-on-the-amd-bench-are-bimodal-per-boot.md` says what its modes are. +**Where `mem.map_unmap_4k` goes, 2026-11-11** ([evidence](https://github.com/ToyOSOrg/evidence/tree/cf6a882c04a646a8998e3bf0ad66e9f699b8e8aa/memperf/diagnosis)): +stage probes at 31eb5416b, one boot a bench. Of the T14's 2,070 ns the +inventory's insert took 735 and its removal 720 (`kernel/pure/vm.rs`; 515 +and 504 on the AMD bench), and the two calls' entry, dispatch, lock and exit +about 300. No table is built or unlinked and nothing is flushed: best fit +places the page in a span that already has tables. `mem.munmap_part.4k`'s +removal took 665 of its 1,398 ns. The inventory kept its free ranges twice, +by start and by length, rewrote both on every change, and listed a removal's +starts before taking them; it now keeps them once, by length +([evidence](https://github.com/ToyOSOrg/evidence/tree/fffbc92ad7894c5c3c3867be65a655899a6f1e5b/memperf/inventory)). +On the host, under load, one page's insert went from 116-176 to 50-82 ns and +its removal from 148-177 to 105-145, with one allocation where there were +two. Not yet measured on metal. + Owner: the memory track. Exit: each path above measured on the T14 at a change and at its base, and against Linux; a regression is fixed or its cost accepted by the owner, or by the coordinator for an implementation tradeoff, diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index a48b1bf0d53..73bec2dca7b 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -622,6 +622,9 @@ mod tests { (None, _) => gaps.push((p as u64, 1)), } } + let by_len: BTreeSet<(u64, u64)> = + gaps.iter().map(|&(s, len)| (len * PAGE, FLOOR + s * PAGE)).collect(); + assert_eq!(v.free, by_len, "round {round}: the free index is not the gaps"); let longest = gaps.iter().map(|&(_, len)| len).max().unwrap_or(0); for (n, align) in [(1, 1), (2, 1), (3, 4), (8, 1), (5, 16), (longest, 1), (longest + 1, 1)] { if n == 0 { diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 65e662cfea7..f6672b3d659 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -206,13 +206,13 @@ pub struct AddressSpace { /// rarely charges anything. const RANGE_GRAIN: usize = 32; -/// The heap one range takes, at most: its entry, the free range beside it and -/// that one's place by length, and its place in the lists a removal gathers. +/// The heap one range takes, at most: its entry, the free range beside it by +/// length, and its place in the list a removal gathers, which holds up to three +/// times its length while it doubles. fn range_heap() -> usize { charge::map_entry::() - + charge::map_entry::() + charge::map_entry::<(u64, u64), ()>() - + 2 * core::mem::size_of::<(u64, u64, SpaceEntry)>() + + 3 * core::mem::size_of::<(u64, SpaceEntry)>() } /// What a leaf of a range with protection `prot` says, `None` for one that From d7c559547d7014d8a025e80f7bc7378a4ba1e922 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 11 Nov 2026 21:58:41 +0200 Subject: [PATCH 3/3] issues: what #1256 owes on metal With no bench booting, the record names the A/B on both benches that the single free index still owes, and what main must show in it. Co-Authored-By: Claude Opus 5.5 --- .../user-memory-hot-paths-are-slower-than-2-mib-pages.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md index 4a2918caec8..a86db74e389 100644 --- a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md +++ b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md @@ -136,7 +136,13 @@ starts before taking them; it now keeps them once, by length ([evidence](https://github.com/ToyOSOrg/evidence/tree/fffbc92ad7894c5c3c3867be65a655899a6f1e5b/memperf/inventory)). On the host, under load, one page's insert went from 116-176 to 50-82 ns and its removal from 148-177 to 105-145, with one allocation where there were -two. Not yet measured on metal. +two. + +- **Owed on metal:** an A/B on both benches of `osbench_system` and + `osbench_mem`, main from before and after the single free index. Main after + must read `mem.map_unmap_4k` and `mem.munmap_part.4k` faster, and no other + row of the two groups slower; rows that map and unmap may improve. Until + then the gain on metal is unmeasured. Owner: the memory track. Exit: each path above measured on the T14 at a change and at its base, and against Linux; a regression is fixed or its cost