The inventory keeps its gaps once, by length, and inserts and removes in place

issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md

@@ -124,6 +124,26 @@ The faults and `mem.anon.total` are now ahead of Linux on both benches. Behind:
 `issues/many-osbench-rows-on-the-amd-bench-are-bimodal-per-boot.md` says
 what its modes are.
 
+**Where `mem.map_unmap_4k` goes, 2026-11-11** ([evidence](https://github.com/ToyOSOrg/evidence/tree/cf6a882c04a646a8998e3bf0ad66e9f699b8e8aa/memperf/diagnosis)):
+stage probes at 31eb5416b, one boot a bench. Of the T14's 2,070 ns the
+inventory's insert took 735 and its removal 720 (`kernel/pure/vm.rs`; 515
+and 504 on the AMD bench), and the two calls' entry, dispatch, lock and exit
+about 300. No table is built or unlinked and nothing is flushed: best fit
+places the page in a span that already has tables. `mem.munmap_part.4k`'s
+removal took 665 of its 1,398 ns. The inventory kept its free ranges twice,
+by start and by length, rewrote both on every change, and listed a removal's
+starts before taking them; it now keeps them once, by length
+([evidence](https://github.com/ToyOSOrg/evidence/tree/fffbc92ad7894c5c3c3867be65a655899a6f1e5b/memperf/inventory)).
+On the host, under load, one page's insert went from 116-176 to 50-82 ns and
+its removal from 148-177 to 105-145, with one allocation where there were
+two.
+
+- **Owed on metal:** an A/B on both benches of `osbench_system` and
+  `osbench_mem`, main from before and after the single free index. Main after
+  must read `mem.map_unmap_4k` and `mem.munmap_part.4k` faster, and no other
+  row of the two groups slower; rows that map and unmap may improve. Until
+  then the gain on metal is unmeasured.
+
 Owner: the memory track. Exit: each path above measured on the T14 at a
 change and at its base, and against Linux; a regression is fixed or its cost
 accepted by the owner, or by the coordinator for an implementation tradeoff,

kernel/pure/vm.rs

@@ -4,8 +4,8 @@
 //!
 //! Ranges are whole 4 KiB pages inside `[floor, top)`, never overlapping.
 //! What is not an entry is free, and placement looks at nothing else: the
-//! free ranges are kept by address and by length, so finding room, cutting a
-//! range out and inserting one are logarithmic in the entries.
+//! free ranges, the gaps the entries leave, are kept by length, so finding
+//! room, cutting a range out and inserting one are logarithmic in the entries.
 //!
 //! **Every change is all-or-nothing.** A caller asks what a change would do
 //! before making it ([`Inventory::covered`], [`Inventory::is_free`],
@@ -157,19 +157,15 @@ pub struct Inventory<O, S = O> {
     floor: u64,
     top: u64,
     entries: BTreeMap<u64, Entry<O, S>>,
-    /// Free ranges by start, each to its end.
-    free: BTreeMap<u64, u64>,
-    /// The same ranges by `(length, start)`.
-    by_len: BTreeSet<(u64, u64)>,
+    /// Every gap in `[floor, top)` between entries, by `(length, start)`.
+    free: BTreeSet<(u64, u64)>,
 }
 
 impl<O: Object, S: Object> Inventory<O, S> {
     /// Nothing held in `[floor, top)`, both on pages.
     pub fn new(floor: u64, top: u64) -> Self {
         assert!(floor.is_multiple_of(PAGE) && top.is_multiple_of(PAGE) && floor < top, "vm: {floor:#x}..{top:#x}");
-        let mut inventory = Self { floor, top, entries: BTreeMap::new(), free: BTreeMap::new(), by_len: BTreeSet::new() };
-        inventory.add_free(floor, top);
-        inventory
+        Self { floor, top, entries: BTreeMap::new(), free: BTreeSet::from([(top - floor, floor)]) }
     }
 
     pub fn floor(&self) -> u64 {
@@ -244,37 +240,68 @@ impl<O: Object, S: Object> Inventory<O, S> {
     pub fn place(&self, len: u64, align: u64) -> Option<u64> {
         debug_assert!(len.is_multiple_of(PAGE) && len > 0 && align.is_power_of_two() && align >= PAGE);
         let need = len.checked_add(align - PAGE)?;
-        let &(_, start) = self.by_len.range((need, 0)..).next()?;
+        let &(_, start) = self.free.range((need, 0)..).next()?;
         Some(start.next_multiple_of(align))
     }
 
-    /// Hold `[start, entry.end)`, which is free.
+    /// Hold `[start, entry.end)`, which is free. It joins a neighbour it
+    /// continues.
     pub fn insert(&mut self, start: u64, entry: Entry<O, S>) {
-        assert!(self.holds(start, entry.end), "vm: {start:#x}..{:#x} is not pages of this space", entry.end);
-        let (free_start, free_end) = self.free.range(..=start).next_back().map(|(&s, &e)| (s, e)).expect("vm: inserted over held memory");
-        assert!(entry.end <= free_end, "vm: {start:#x}..{:#x} inserted over held memory", entry.end);
-        self.take_free(free_start, free_end);
-        if free_start < start {
-            self.add_free(free_start, start);
+        let end = entry.end;
+        assert!(self.holds(start, end), "vm: {start:#x}..{end:#x} is not pages of this space");
+        let (hi, joins_next) = match self.entries.range(start..).next() {
+            Some((&next, following)) => (next, next == end && entry.continued_by(start, following)),
+            None => (self.top, false),
+        };
+        let prev = self.entries.range_mut(..start).next_back();
+        let lo = prev.as_ref().map_or(self.floor, |(_, e)| e.end);
+        assert!(lo <= start && end <= hi, "vm: {start:#x}..{end:#x} inserted over held memory");
+        self.free.remove(&(hi - lo, lo));
+        if lo < start {
+            self.free.insert((start - lo, lo));
         }
-        if entry.end < free_end {
-            self.add_free(entry.end, free_end);
+        if end < hi {
+            self.free.insert((hi - end, end));
+        }
+        let joined = match prev {
+            Some((&s, before)) if lo == start && before.continued_by(s, &entry) => {
+                before.end = end;
+                s
+            }
+            _ => {
+                self.entries.insert(start, entry);
+                start
+            }
+        };
+        if joins_next {
+            let following = self.entries.remove(&end).expect("vm: the range just found");
+            self.entries.get_mut(&joined).expect("vm: the range just held").end = following.end;
         }
-        let end = entry.end;
-        self.entries.insert(start, entry);
-        self.join(start, end);
     }
 
     /// Take every entry out of `[start, end)`, cutting any that crosses its
     /// ends, and answer what was there, in address order.
     pub fn remove(&mut self, start: u64, end: u64) -> Vec<(u64, Entry<O, S>)> {
         self.split_at(start);
         self.split_at(end);
-        let starts: Vec<u64> = self.entries.range(start..end).map(|(&s, _)| s).collect();
-        let removed: Vec<(u64, Entry<O, S>)> =
-            starts.into_iter().map(|s| (s, self.entries.remove(&s).expect("vm: a start just listed"))).collect();
-        for (s, entry) in &removed {
-            self.release(*s, entry.end);
+        let lo = self.entries.range(..start).next_back().map_or(self.floor, |(_, e)| e.end);
+        // The gaps from `lo` up to the last entry taken join into one.
+        let mut gap = lo;
+        let mut removed = Vec::new();
+        while let Some((&s, _)) = self.entries.range(start..end).next() {
+            let entry = self.entries.remove(&s).expect("vm: a start just found");
+            if gap < s {
+                self.free.remove(&(s - gap, gap));
+            }
+            gap = entry.end;
+            removed.push((s, entry));
+        }
+        if !removed.is_empty() {
+            let hi = self.entries.range(gap..).next().map_or(self.top, |(&s, _)| s);
+            if gap < hi {
+                self.free.remove(&(hi - gap, gap));
+            }
+            self.free.insert((hi - lo, lo));
         }
         removed
     }
@@ -328,32 +355,6 @@ impl<O: Object, S: Object> Inventory<O, S> {
             }
         }
     }
-
-    /// `[start, end)` was held and is free now.
-    fn release(&mut self, start: u64, end: u64) {
-        let (mut lo, mut hi) = (start, end);
-        if let Some((&s, &e)) = self.free.range(..start).next_back() {
-            if e == start {
-                self.take_free(s, e);
-                lo = s;
-            }
-        }
-        if let Some(&e) = self.free.get(&end) {
-            self.take_free(end, e);
-            hi = e;
-        }
-        self.add_free(lo, hi);
-    }
-
-    fn add_free(&mut self, start: u64, end: u64) {
-        self.free.insert(start, end);
-        self.by_len.insert((end - start, start));
-    }
-
-    fn take_free(&mut self, start: u64, end: u64) {
-        self.free.remove(&start);
-        self.by_len.remove(&(end - start, start));
-    }
 }
 
 #[cfg(test)]
@@ -473,6 +474,12 @@ mod tests {
         v.insert(FLOOR + 11 * PAGE, object(FLOOR + 12 * PAGE, 7, 2 * PAGE));
         v.insert(FLOOR + 12 * PAGE, object(FLOOR + 13 * PAGE, 7, 3 * PAGE));
         assert_eq!(v.len(), 3);
+        // A range that fills the hole between two it continues makes one of the three.
+        v.insert(FLOOR + 20 * PAGE, private(FLOOR + 21 * PAGE, Prot::Read));
+        v.insert(FLOOR + 22 * PAGE, private(FLOOR + 23 * PAGE, Prot::Read));
+        v.insert(FLOOR + 21 * PAGE, private(FLOOR + 22 * PAGE, Prot::Read));
+        assert_eq!(v.find(FLOOR + 20 * PAGE).map(|(s, e)| (s, e.end)), Some((FLOOR + 20 * PAGE, FLOOR + 23 * PAGE)));
+        assert_eq!(v.len(), 4);
     }
 
     #[test]
@@ -605,17 +612,31 @@ mod tests {
                 let ((s, a), (t, b)) = (pair[0], pair[1]);
                 assert!(!(a.end == t && a.continued_by(s, b)), "round {round}: {s:#x} and {t:#x} were left apart");
             }
-            // The free index is exactly the gaps.
-            let longest_gap = {
-                let (mut best, mut run) = (0, 0);
-                for p in &map {
-                    run = if p.is_none() { run + 1 } else { 0 };
-                    best = best.max(run);
+            // The free index is exactly the gaps: a placement is the smallest
+            // gap that holds it at any alignment, the lowest of equals.
+            let mut gaps: Vec<(u64, u64)> = Vec::new();
+            for (p, held) in map.iter().enumerate() {
+                match (held, gaps.last_mut()) {
+                    (Some(_), _) => {}
+                    (None, Some((s, len))) if *s + *len == p as u64 => *len += 1,
+                    (None, _) => gaps.push((p as u64, 1)),
                 }
-                best
-            };
-            assert!(longest_gap == 0 || v.place(longest_gap * PAGE, PAGE).is_some(), "round {round}");
-            assert_eq!(v.place((longest_gap + 1) * PAGE, PAGE), None, "round {round}");
+            }
+            let by_len: BTreeSet<(u64, u64)> =
+                gaps.iter().map(|&(s, len)| (len * PAGE, FLOOR + s * PAGE)).collect();
+            assert_eq!(v.free, by_len, "round {round}: the free index is not the gaps");
+            let longest = gaps.iter().map(|&(_, len)| len).max().unwrap_or(0);
+            for (n, align) in [(1, 1), (2, 1), (3, 4), (8, 1), (5, 16), (longest, 1), (longest + 1, 1)] {
+                if n == 0 {
+                    continue;
+                }
+                let want = gaps
+                    .iter()
+                    .filter(|&&(_, len)| len >= n + align - 1)
+                    .min_by_key(|&&(s, len)| (len, s))
+                    .map(|&(s, _)| (FLOOR + s * PAGE).next_multiple_of(align * PAGE));
+                assert_eq!(v.place(n * PAGE, align * PAGE), want, "round {round}: {n} pages aligned to {align}");
+            }
         }
     }
 

kernel/src/mm/space.rs

@@ -206,13 +206,13 @@ pub struct AddressSpace {
 /// rarely charges anything.
 const RANGE_GRAIN: usize = 32;
 
-/// The heap one range takes, at most: its entry, the free range beside it and
-/// that one's place by length, and its place in the lists a removal gathers.
+/// The heap one range takes, at most: its entry, the free range beside it by
+/// length, and its place in the list a removal gathers, which holds up to three
+/// times its length while it doubles.
 fn range_heap() -> usize {
     charge::map_entry::<u64, SpaceEntry>()
-        + charge::map_entry::<u64, u64>()
         + charge::map_entry::<(u64, u64), ()>()
-        + 2 * core::mem::size_of::<(u64, u64, SpaceEntry)>()
+        + 3 * core::mem::size_of::<(u64, SpaceEntry)>()
 }
 
 /// What a leaf of a range with protection `prot` says, `None` for one that