From 151e90a0a6d938c2d4acd95f4de951b04b618286 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 27 Oct 2026 21:57:30 +0100 Subject: [PATCH 1/4] alloc: a large block is a mapping of its own, grown by vm_remap without a copy Growing a large allocation copied it every time. dlmalloc-rs, the heap std and libc allocate with, leaves out the C original's direct mapping of a chunk past DEFAULT_MMAP_THRESHOLD (256 KiB): every block lives in a segment, `mmap_resize` and the `remap` hook are unreachable, and a `realloc` the segment cannot extend is malloc, copy and free. On the T14 at aeba59fb0, growing a block from 1 MiB to 256 MiB by doubling took 67 ms, its last doubling 34 ms. std's and libc's allocators now map every block of 256 KiB or more on its own (`sdk/libc/src/mapped.rs`, one source for both), and grow or shrink it with a new syscall, `vm_remap`: committed private memory of one protection, resized in place where the pages after it are free, and otherwise, when the caller allows it, moved by moving its leaves, with nothing copied and the old place unmapped. Which kind a block is follows from its layout, which every call carries, so nothing is recorded; a block crossing the threshold is moved once, and one aligned past a page grows by a copy, since a move keeps only a page's alignment. A freed large block's memory goes back to the kernel at once. The tradeoff: a block of 256 KiB or more is no longer reused from the heap, so a program that frees and allocates such blocks at a high rate pays a reserve, a release and its faults each time, as it does under musl or glibc below their thresholds. osbench's `malloc_churn.1048576` measures exactly that. `mem.realloc_grow` measured nothing on Linux: the block never escaped, and LLVM removed the allocation, the write, every `realloc` and the free, leaving two clock reads per doubling (15-25 ns). The block now escapes through `black_box`. Co-Authored-By: Claude Opus 5.5 --- ...b-pages-still-copies-each-programs-text.md | 7 +- kernel/src/mm/space.rs | 91 ++++++++++++++ kernel/src/syscall/dispatch.rs | 3 +- kernel/src/syscall/vm.rs | 19 ++- sdk/abi/src/syscall.rs | 24 +++- sdk/libc/src/lib.rs | 35 +++++- sdk/libc/src/mapped.rs | 53 ++++++++ sdk/std/sys/alloc.rs | 20 +++ tests/guest/osbench/src/mem.rs | 7 +- .../rust-tests/src/bin/allocator_stress.rs | 40 ++++++ tests/guest/rust-tests/src/bin/mmap_stress.rs | 118 +++++++++++++++++- 11 files changed, 402 insertions(+), 15 deletions(-) create mode 100644 sdk/libc/src/mapped.rs diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index b713908b20d..955c941b0cd 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -77,9 +77,10 @@ live split of a user leaf. 4. **Demand-zero anonymous `mmap`.** Built, over 4 KiB pages only: a fault populates one page, unmapping a range no thread touched sends no IPI, and `munmap` and a `FIXED` `mmap` take any run of pages, as POSIX does. - `vm_reserve`, `vm_protect`, `vm_discard`, `vm_decommit` and `vm_release` - are the calls, behind libc's `mmap`, `mprotect`, `madvise` and `munmap` - and std's allocator, which trims, and its thread stacks, each reserved + `vm_reserve`, `vm_protect`, `vm_discard`, `vm_decommit`, `vm_remap` and + `vm_release` are the calls, behind libc's `mmap`, `mprotect`, `madvise` and + `munmap`, std's and libc's allocators, which trim and grow a large block + by `vm_remap` in place or by moving its pages, and std's thread stacks, each reserved at the size asked with a guard page below. A commitment is promised its frames when it is made, and every other allocation is refused rather than take a promised frame (`kernel::frames`), so a commitment past what memory diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 2d6c069e69e..2605e67d175 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -592,6 +592,97 @@ impl AddressSpace { Ok(()) } + /// Resize `[start, start + old)`, committed private memory of one + /// protection that is not executable, to `new` bytes, whole pages both + /// (`toyos_abi::syscall::vm_remap`): a shrink releases the tail, a growth + /// takes the free pages after it, or else, if `may_move`, its leaves move + /// to a new place and nothing is copied. Promised and all or nothing, as + /// [`reserve`](Self::reserve); answers where the range is now. + pub fn remap(&mut self, start: UserAddr, old: u64, new: u64, may_move: bool) -> Result { + let (start, end) = self.held(start, old)?; + let (_, entry) = self.inventory.find(start).expect("space: a page `held` answered for"); + let prot = entry.prot; + // Neighbours alike join, so one entry is the whole of one kind and protection. + if !matches!(entry.kind, Kind::Private { committed: true }) || prot == Prot::ReadExec || entry.end < end { + return Err(SyscallError::InvalidArgument); + } + if new == 0 || !new.is_multiple_of(PAGE_SIZE) || new > self.inventory.top() - self.inventory.floor() { + return Err(SyscallError::InvalidArgument); + } + if new <= old { + if new < old { + self.room(start + new, end, 0)?; + self.release(UserAddr::new(start + new), old - new); + } + return Ok(UserAddr::new(start)); + } + let in_place = start.checked_add(new).is_some_and(|grown| self.inventory.holds(end, grown) && self.inventory.is_free(end, grown)); + let to = if in_place { + self.room(end, start + new, 1)?; + start + } else if may_move { + self.room(start, end, 1)?; + self.inventory.place(new, PAGE_SIZE).ok_or(SyscallError::ResourceExhausted)? + } else { + return Err(SyscallError::ResourceExhausted); + }; + let added = (new - old) / PAGE_SIZE; + if !pmm::promise(added) { + return Err(SyscallError::ResourceExhausted); + } + let built = if in_place { self.build(end, start + new) } else { self.build(to, to + new) }; + built.inspect_err(|_| pmm::unpromise(added))?; + let kind = Kind::Private { committed: true }; + if in_place { + self.inventory.insert(end, Entry { end: start + new, prot, kind }); + } else { + let present = self.move_leaves(start, end, to); + self.inventory.remove(start, end); + self.inventory.insert(to, Entry { end: to + new, prot, kind }); + let inventory = &self.inventory; + let unlinked = self.tables.unlink(start, end, |s, e| inventory.backed(s, e)); + if present || !unlinked.is_empty() { + self.shoot_down(start, end, &unlinked); + } + unlinked.free(); + self.usage.taken.fetch_add(1, Relaxed); + } + self.usage.made.fetch_add(1, Relaxed); + self.settle(); + Ok(UserAddr::new(to)) + } + + /// Move every leaf of `[from, end)` to the same place in the range from + /// `to`, whose tables are built and whose leaves are empty, leaving the + /// old ones empty: their frames stay this space's. Answers whether one + /// was present, which some CPU may still cache. + fn move_leaves(&mut self, from: u64, end: u64, to: u64) -> bool { + // A held `LeafTable` borrows all the tables, so leaves are taken out + // of the source a batch at a time and then written at the destination. + const BATCH: usize = 64; + let mut present = false; + let mut va = from; + while va < end { + let mut batch = [(0, Leaf::Empty); BATCH]; + let mut n = 0; + let mut leaves = self.tables.leaf_table(va).unwrap_or_else(|| panic!("space: no tables under {va:#x}")); + while va < end && leaves.covers(va) && n < BATCH { + let leaf = leaves.get(va); + if leaf != Leaf::Empty { + leaves.set(va, Leaf::Empty); + batch[n] = (va, leaf); + n += 1; + } + va += PAGE_SIZE; + } + for &(at, leaf) in &batch[..n] { + present |= matches!(leaf, Leaf::Mapped(_)); + self.tables.set(at - from + to, leaf); + } + } + present + } + /// `[start, start + len)` as pages every one of which a range holds. fn held(&self, start: UserAddr, len: u64) -> Result<(u64, u64), SyscallError> { let end = pages(start.raw(), len).ok_or(SyscallError::InvalidArgument)?; diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 28ac025fe00..21a1f8b59a8 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -51,7 +51,7 @@ use super::proc::{ sys_job_create, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, }; use super::vm::{ - shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, + shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, sys_vm_remap, sys_vm_protect, sys_vm_release, sys_vm_reserve, }; @@ -364,6 +364,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_VM_PROTECT => sys_vm_protect(a1, a2, a3), SYS_VM_DISCARD => sys_vm_discard(a1, a2), SYS_VM_DECOMMIT => sys_vm_decommit(a1, a2), + SYS_VM_REMAP => sys_vm_remap(a1, a2, a3, a4), SYS_PREAD => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; with_object(RawHandle(a1 as u32), Rights::READ, |o| ops::pread(o, a4, &mut buf)) diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index 95258dfb6a4..00aa749247b 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -1,5 +1,5 @@ -//! The caller's own address space: `vm_*` reserve, commit, protect, discard -//! and take away private memory, `dlopen` maps a library image, `SYS_TLS_ALLOC_BLOCK` +//! The caller's own address space: `vm_*` reserve, commit, protect, discard, +//! resize and take away private memory, `dlopen` maps a library image, `SYS_TLS_ALLOC_BLOCK` //! maps a thread's per-module TLS block, `SYS_QUERY_MODULES` reports what is //! loaded. Exhausting address space is an error return, never an `.expect`. //! @@ -89,6 +89,21 @@ pub(super) fn sys_vm_decommit(addr: u64, len: u64) -> u64 { answer(range(addr, len).and_then(|(start, len)| process::current_address_space().lock().decommit(start, len))) } +/// Resize committed private memory in place, or move it when `may_move` is 1. +pub(super) fn sys_vm_remap(addr: u64, old: u64, new: u64, may_move: u64) -> u64 { + let remapped = (|| { + let may_move = match may_move { + 0 => false, + 1 => true, + _ => return Err(SyscallError::InvalidArgument), + }; + let (start, old) = range(addr, old)?; + let new = whole_pages(new).ok_or(SyscallError::InvalidArgument)?; + process::current_address_space().lock().remap(start, old, new, may_move) + })(); + remapped.map_or_else(SyscallError::to_u64, UserAddr::raw) +} + /// The first `len` bytes of the shared memory object `handle` names, as a /// program image; `Err` is the syscall's answer. pub(super) fn shared_image(handle: u64, len: u64) -> Result { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index dcb934f5cfb..1fc67c0ace6 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -321,6 +321,8 @@ pub const SYS_VM_DISCARD: u64 = 141; pub const SYS_VM_DECOMMIT: u64 = 142; /// A memory object over part of another's pages. See [`shm_sub`]. pub const SYS_SHM_SUB: u64 = 143; +/// Resize a run of committed private memory, in place or moved. See [`vm_remap`]. +pub const SYS_VM_REMAP: u64 = 144; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -328,13 +330,13 @@ pub const SYS_SHM_SUB: u64 = 143; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 145; +pub const SYSCALL_PROFILE_BINS: usize = 146; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_SHM_SUB < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_VM_REMAP < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -2435,6 +2437,24 @@ pub unsafe fn vm_decommit(addr: *mut u8, len: usize) -> Result<(), SyscallError> check_unit(syscall(SYS_VM_DECOMMIT, addr as u64, len as u64, 0, 0)) } +/// Resize `[addr, addr + old_len)`, committed private memory of one +/// protection that is not executable, to `new_len` bytes, both rounded up to +/// whole pages, as Linux's `mremap` does: its pages keep their contents and a +/// grown tail reads as zeros. It grows in place where the pages after it are +/// free; otherwise, if `may_move`, its pages move to a new place, unmapped at +/// the old, with nothing copied. A shrink gives back the tail in place. +/// Answers where the range is now. Refused, it is left as it was: +/// `BadAddress` for a page no range holds, `InvalidArgument` for any other +/// memory or a zero length, and `ResourceExhausted` where it has no room. +/// +/// # Safety +/// Nothing reaches the old pages after this answers a new place, nor past +/// `new_len` after a shrink. +pub unsafe fn vm_remap(addr: *mut u8, old_len: usize, new_len: usize, may_move: bool) -> Result<*mut u8, SyscallError> { + check(syscall(SYS_VM_REMAP, addr as u64, old_len as u64, new_len as u64, may_move as u64)) + .map(|at| core::ptr::with_exposed_provenance_mut(at as usize)) +} + /// Non-blocking read. Returns bytes read, or `Err(WouldBlock)` if no data available. pub fn read_nonblock(handle: RawHandle, buf: &mut [u8]) -> Result { check(syscall(SYS_READ_NONBLOCK, handle.0 as u64, buf.as_mut_ptr() as u64, buf.len() as u64, 0)).map(|n| n as usize) diff --git a/sdk/libc/src/lib.rs b/sdk/libc/src/lib.rs index be1e48bc27c..7c3260acefc 100644 --- a/sdk/libc/src/lib.rs +++ b/sdk/libc/src/lib.rs @@ -48,6 +48,9 @@ mod uname; mod utf8; mod wchar; +#[cfg(not(feature = "std-runtime"))] +mod mapped; + // C runtime: the entry `arch::_start` calls, panic handler, and global allocator. // Only for pure C programs (no Rust std). When linked into a Rust program // with std, std provides these. @@ -193,15 +196,45 @@ mod runtime { #[global_allocator] pub(crate) static ALLOCATOR: LibcAllocator = LibcAllocator; + use super::mapped; + unsafe impl core::alloc::GlobalAlloc for LibcAllocator { unsafe fn alloc(&self, layout: core::alloc::Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } unsafe { DLMALLOC.lock().malloc(layout.size(), layout.align()) } } + unsafe fn alloc_zeroed(&self, layout: core::alloc::Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } + unsafe { DLMALLOC.lock().calloc(layout.size(), layout.align()) } + } unsafe fn dealloc(&self, ptr: *mut u8, layout: core::alloc::Layout) { + if mapped::holds(layout) { + return unsafe { mapped::dealloc(ptr, layout) }; + } unsafe { DLMALLOC.lock().free(ptr, layout.size(), layout.align()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: core::alloc::Layout, new_size: usize) -> *mut u8 { - unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) } + if mapped::resizes(layout, new_size) { + return unsafe { mapped::resize(ptr, layout, new_size) }; + } + // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. + let new = unsafe { core::alloc::Layout::from_size_align_unchecked(new_size, layout.align()) }; + if !mapped::holds(layout) && !mapped::holds(new) { + return unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) }; + } + // Between the heap and a mapping of its own: moved, as the default does. + let moved = unsafe { self.alloc(new) }; + if !moved.is_null() { + unsafe { + core::ptr::copy_nonoverlapping(ptr, moved, layout.size().min(new_size)); + self.dealloc(ptr, layout); + } + } + moved } } } diff --git a/sdk/libc/src/mapped.rs b/sdk/libc/src/mapped.rs new file mode 100644 index 00000000000..34323b8dae6 --- /dev/null +++ b/sdk/libc/src/mapped.rs @@ -0,0 +1,53 @@ +//! Blocks of [`THRESHOLD`] bytes or more, each a mapping of its own beside +//! the heap that holds every smaller one, as dlmalloc's C original makes them +//! and its Rust port does not: such a block grows by `vm_remap`, in place or +//! by moving its pages, never by copying them, and its memory goes back at its +//! free. One source for std's allocator (`sdk/std/sys/alloc.rs`) and libc's. +//! +//! Which a block is follows from its layout alone, which every call about it +//! carries, so nothing is recorded. + +use core::alloc::Layout; +use core::ptr; + +use toyos_abi::syscall::{self, VmAt, VmProt, PAGE_SIZE}; + +/// dlmalloc's `DEFAULT_MMAP_THRESHOLD`. +const THRESHOLD: usize = 256 * 1024; + +/// Whether a block of `layout` is a mapping of its own. +pub(crate) fn holds(layout: Layout) -> bool { + layout.size() >= THRESHOLD +} + +/// Whether [`resize`] can take a mapped block of `layout` to `new_size`: a +/// move keeps only a page's alignment. +pub(crate) fn resizes(layout: Layout, new_size: usize) -> bool { + holds(layout) && new_size >= THRESHOLD && layout.align() <= PAGE_SIZE +} + +/// A mapping for `layout`, of zeros, or null where none is granted. +pub(crate) fn alloc(layout: Layout) -> *mut u8 { + let at = VmAt::Anywhere { align: layout.align().max(PAGE_SIZE) }; + // SAFETY: placed anywhere, so it replaces nothing. + unsafe { syscall::vm_reserve(layout.size(), VmProt::ReadWrite, at) }.unwrap_or(ptr::null_mut()) +} + +/// # Safety +/// `block` is a mapping [`alloc`] or [`resize`] answered for `layout`, which +/// nothing reaches after. +pub(crate) unsafe fn dealloc(block: *mut u8, layout: Layout) { + // Refused only at the space's bound on ranges, and then the block stays. + let _ = unsafe { syscall::vm_release(block, layout.size()) }; +} + +/// The mapping `block` of `layout` resized to `new_size` bytes, its contents +/// kept, where [`resizes`] says it can be; null, and the block as it was, +/// where the kernel refuses. +/// +/// # Safety +/// As [`dealloc`], once this answers a block. +pub(crate) unsafe fn resize(block: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + debug_assert!(resizes(layout, new_size)); + unsafe { syscall::vm_remap(block, layout.size(), new_size, true) }.unwrap_or(ptr::null_mut()) +} diff --git a/sdk/std/sys/alloc.rs b/sdk/std/sys/alloc.rs index cc8fd1d6a0b..63e793da6b3 100644 --- a/sdk/std/sys/alloc.rs +++ b/sdk/std/sys/alloc.rs @@ -6,6 +6,9 @@ use crate::alloc::Layout; use crate::ptr; use crate::sys::sync::Mutex; +#[path = "../../libc/src/mapped.rs"] +mod mapped; + /// dlmalloc's memory: the process's own private pages, committed whole and /// populated as they are touched, and given back in part when it trims. struct ToyOsAllocator; @@ -72,24 +75,41 @@ impl Drop for DropLock { #[inline] pub unsafe fn alloc(layout: Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.malloc(layout.size(), layout.align()) } } #[inline] pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.calloc(layout.size(), layout.align()) } } #[inline] pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { + if mapped::holds(layout) { + return unsafe { mapped::dealloc(ptr, layout) }; + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.free(ptr, layout.size(), layout.align()) } } #[inline] pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if mapped::resizes(layout, new_size) { + return unsafe { mapped::resize(ptr, layout, new_size) }; + } + // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. + let new = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + if mapped::holds(layout) || mapped::holds(new) { + return unsafe { super::realloc_fallback(ptr, layout, new_size) }; + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.realloc(ptr, layout.size(), layout.align(), new_size) } } diff --git a/tests/guest/osbench/src/mem.rs b/tests/guest/osbench/src/mem.rs index 3fc9bd177f1..b9d0712bb69 100644 --- a/tests/guest/osbench/src/mem.rs +++ b/tests/guest/osbench/src/mem.rs @@ -188,13 +188,14 @@ fn churn(size: usize, threads: usize) { /// A block grown by `realloc` from 1 MiB to 256 MiB, doubling, its first /// mebibyte written: ns for the whole growth and for its last doubling, five /// rounds after one. An allocator that can move the pages answers without -/// copying; one that cannot copies each time. +/// copying; one that cannot copies each time. The block escapes through +/// `black_box`, or the compiler drops every call. fn realloc_grow() { let round = || -> (u64, u64) { let mut size = MIB; let layout = Layout::from_size_align(size, 16).expect("a layout"); // SAFETY: a non-zero layout. - let mut ptr = unsafe { alloc(layout) }; + let mut ptr = std::hint::black_box(unsafe { alloc(layout) }); assert!(!ptr.is_null()); // SAFETY: inside the block. unsafe { std::ptr::write_bytes(ptr, 0x33, MIB) }; @@ -202,7 +203,7 @@ fn realloc_grow() { while size < 256 * MIB { let t = Instant::now(); // SAFETY: `ptr` holds `size` bytes allocated with this alignment. - ptr = unsafe { realloc(ptr, Layout::from_size_align(size, 16).expect("a layout"), size * 2) }; + ptr = std::hint::black_box(unsafe { realloc(ptr, Layout::from_size_align(size, 16).expect("a layout"), size * 2) }); last = ns_since(t); total += last; assert!(!ptr.is_null(), "realloc to {} bytes was refused", size * 2); diff --git a/tests/guest/rust-tests/src/bin/allocator_stress.rs b/tests/guest/rust-tests/src/bin/allocator_stress.rs index ec6049cdc68..7390e224cd1 100644 --- a/tests/guest/rust-tests/src/bin/allocator_stress.rs +++ b/tests/guest/rust-tests/src/bin/allocator_stress.rs @@ -13,6 +13,7 @@ fn main() { test_fragmentation_resistance(); test_memory_stats(); test_oom_graceful(); + test_large_blocks(); println!("all allocator stress tests passed"); } @@ -225,6 +226,45 @@ fn test_grow_and_shrink() { println!(" grow and shrink: ok"); } +/// Blocks past the allocator's mapping threshold, a mapping each: a vector +/// grown by doubling from the heap into one and on, and shrunk back into the +/// heap, keeps every byte; a zeroed one reads zeros where a freed one of its +/// size was written; and one aligned past a page keeps its alignment when it +/// grows. +fn test_large_blocks() { + let byte = |i: usize| (i % 251) as u8; + let mut v: Vec = Vec::new(); + for len in (10..=26).map(|shift| 1usize << shift) { + v.reserve_exact(len - v.len()); + v.extend((v.len()..len).map(byte)); + assert!(v.iter().enumerate().all(|(i, &b)| b == byte(i)), "grown to {len} bytes, a byte changed"); + } + v.truncate(100 * 1024); + v.shrink_to_fit(); + assert!(v.iter().enumerate().all(|(i, &b)| b == byte(i)), "shrunk into the heap, a byte changed"); + drop(v); + + let written = vec![0x5Au8; 8 << 20]; + drop(written); + let zeroed = vec![0u8; 8 << 20]; + assert!(zeroed.iter().all(|&b| b == 0), "a zeroed large block held a byte"); + drop(zeroed); + + let layout = std::alloc::Layout::from_size_align(4 << 20, 2 << 20).unwrap(); + // SAFETY: a non-zero layout; the block is written inside it, grown with + // it and freed with the grown layout. + unsafe { + let p = std::alloc::alloc(layout); + assert!(!p.is_null() && p as usize % (2 << 20) == 0, "{p:p} is not 2 MiB aligned"); + p.write_bytes(0x3C, 4 << 20); + let q = std::alloc::realloc(p, layout, 8 << 20); + assert!(!q.is_null() && q as usize % (2 << 20) == 0, "grown, {q:p} is not 2 MiB aligned"); + assert!((0..4 << 20).step_by(4096).all(|i| *q.add(i) == 0x3C), "grown, an aligned block lost a byte"); + std::alloc::dealloc(q, std::alloc::Layout::from_size_align(8 << 20, 2 << 20).unwrap()); + } + println!(" large blocks: ok"); +} + /// Test that the allocator handles fragmentation gracefully. /// Allocate alternating sizes, free every other one, then try to /// allocate in the freed gaps. diff --git a/tests/guest/rust-tests/src/bin/mmap_stress.rs b/tests/guest/rust-tests/src/bin/mmap_stress.rs index d3b2d6f7a5c..d8fac187da8 100644 --- a/tests/guest/rust-tests/src/bin/mmap_stress.rs +++ b/tests/guest/rust-tests/src/bin/mmap_stress.rs @@ -3,7 +3,9 @@ //! range, across two, or a hole — and a fixed reserve replaces whatever the //! pages it names held; `vm_protect` commits a reservation and changes //! protection in place, contents kept; `vm_discard` keeps the pages and reads -//! back zeros; `vm_decommit` makes them a reservation again. A page +//! back zeros; `vm_decommit` makes them a reservation again; `vm_remap` +//! grows, moves and shrinks committed memory, contents kept, and loses no +//! frame or promise when it moves. A page //! populated with its neighbours is populated as its own range says. The //! ranges one space holds are bounded, and a commitment past what memory //! holds is refused when it is asked for. @@ -39,7 +41,7 @@ fn set(p: *mut u8, page: usize, value: u8) { } /// The pages a child makes and touches, none of which allows the touch. -const FAULTING: [&str; 4] = ["reserved", "protected-none", "decommitted", "read-only-beside-a-run"]; +const FAULTING: [&str; 5] = ["reserved", "protected-none", "decommitted", "read-only-beside-a-run", "moved-from"]; fn main() { if let Some(case) = std::env::args().nth(1) { @@ -49,10 +51,12 @@ fn main() { protection(); discard_and_decommit(); ranges_are_bounded(); + resize(); + remap_keeps_memory(); for case in FAULTING { faults(case); } - println!("vm_reserve, vm_protect, vm_discard, vm_decommit and vm_release do what each says"); + println!("vm_reserve, vm_protect, vm_discard, vm_decommit, vm_remap and vm_release do what each says"); } fn placement_and_release() { @@ -208,6 +212,107 @@ fn ranges_are_bounded() { println!(" PASS: a space holds at most {MAX_RANGES} ranges, and gives each back at the bound"); } +fn remap(p: *mut u8, old: usize, new: usize, may_move: bool) -> Result<*mut u8, SyscallError> { + // SAFETY: every range remapped here is this test's own, reached through + // raw pointers only, and through the answer alone once it moves. + unsafe { syscall::vm_remap(p, old * PAGE, new * PAGE, may_move) } +} + +/// `pages` committed pages followed by a read-only one of the same +/// reservation, its own range, so the first `pages` cannot grow in place. +fn blocked(pages: usize) -> *mut u8 { + let p = reserve((pages + 1) * PAGE, VmProt::ReadWrite); + unsafe { syscall::vm_protect(p.add(pages * PAGE), PAGE, VmProt::Read) }.expect("protect the blocking page"); + p +} + +fn resize() { + // Grown into the free pages after it: the same address, the bytes kept, + // and a tail of zeros that takes writes. + let p = reserve(8 * PAGE, VmProt::ReadWrite); + unsafe { syscall::vm_release(p.add(4 * PAGE), 4 * PAGE) }.expect("free the last four pages"); + (0..4).for_each(|page| set(p, page, 0x50 + page as u8)); + assert_eq!(remap(p, 4, 8, false), Ok(p), "growth into free pages"); + assert_eq!((0..8).map(|page| byte(p, page)).collect::>(), [0x50, 0x51, 0x52, 0x53, 0, 0, 0, 0], "grown in place"); + set(p, 7, 0x57); + assert_eq!(byte(p, 7), 0x57, "the grown tail is not writable"); + + // Shrunk: the tail goes and the rest stays. + assert_eq!(remap(p, 8, 2, false), Ok(p), "a shrink"); + assert_eq!((byte(p, 0), byte(p, 1)), (0x50, 0x51), "a shrink took a kept page's byte"); + assert_eq!(unsafe { syscall::vm_protect(p, 3 * PAGE, VmProt::ReadWrite) }, Err(SyscallError::BadAddress), "a shrink kept its tail"); + unsafe { syscall::vm_release(p, 2 * PAGE) }.expect("release"); + + // A neighbour in the way refuses a growth that may not move, and moves + // one that may: every page's byte at the new place, nothing at the old, + // the neighbour untouched. 768 pages cross tables of leaves and batches. + const PAGES: usize = 768; + let p = blocked(PAGES); + (0..PAGES).step_by(3).for_each(|page| set(p, page, page as u8 | 1)); + assert_eq!(remap(p, PAGES, 2 * PAGES, false), Err(SyscallError::ResourceExhausted), "growth into a held page"); + assert_eq!(byte(p, 3), 3 | 1, "a refused growth touched the range"); + let q = remap(p, PAGES, 2 * PAGES, true).expect("a growth that may move"); + assert_ne!(q, p, "growth past a held page stayed in place"); + for page in 0..2 * PAGES { + let want = if page < PAGES && page % 3 == 0 { page as u8 | 1 } else { 0 }; + assert_eq!(byte(q, page), want, "page {page} of the moved range"); + } + set(q, 1, 0x61); + assert_eq!(byte(q, 1), 0x61, "a moved page is not writable"); + assert_eq!(unsafe { syscall::vm_protect(p, PAGE, VmProt::ReadWrite) }, Err(SyscallError::BadAddress), "the old place is still held"); + assert_eq!(unsafe { syscall::vm_protect(p.add(PAGES * PAGE), PAGE, VmProt::Read) }, Ok(()), "the neighbour went"); + unsafe { syscall::vm_release(p.add(PAGES * PAGE), PAGE) }.expect("release the neighbour"); + unsafe { syscall::vm_release(q, 2 * PAGES * PAGE) }.expect("release the moved range"); + + // What remap refuses: any memory but committed private pages of one + // protection, a page no range holds, and a length of nothing. + let none = reserve(2 * PAGE, VmProt::None); + assert_eq!(remap(none, 2, 4, true), Err(SyscallError::InvalidArgument), "a reservation"); + unsafe { syscall::vm_release(none, 2 * PAGE) }.expect("release"); + let mixed = blocked(1); + assert_eq!(remap(mixed, 2, 4, true), Err(SyscallError::InvalidArgument), "two protections"); + assert_eq!(remap(mixed, 1, 0, true), Err(SyscallError::InvalidArgument), "no bytes"); + assert_eq!(remap(unsafe { mixed.add(1) }, 1, 2, true), Err(SyscallError::InvalidArgument), "off a page"); + unsafe { syscall::vm_release(mixed, 2 * PAGE) }.expect("release"); + assert_eq!(remap(mixed, 1, 2, true), Err(SyscallError::BadAddress), "a page no range holds"); + let clock = toyos_abi::clock::CLOCK_PAGE as *mut u8; + assert_eq!(remap(clock, 1, 2, true), Err(SyscallError::InvalidArgument), "a shared object's page"); + // `page` asserts the clock page's magic. + let _ = toyos_abi::clock::page(); + println!(" PASS: remap grows in place or moves the pages, shrinks in place, and refuses all but committed private memory"); +} + +/// Bytes the machine has in use or promised. +fn used() -> u64 { + let mut header = [0u8; syscall::SYSINFO_HEADER_SIZE]; + assert_eq!(toyos::system::sysinfo(&mut header), header.len(), "the sysinfo header"); + syscall::SysinfoHeader::decode(&header).memory_used +} + +/// A written 16 MiB moved 32 times, each time grown by the page that becomes +/// the next move's neighbour, gives every frame and promise back: the +/// machine's memory in use after is what it was before, give or take what the +/// rest of it did meanwhile, where one move that kept its range's promise +/// would cost 16 MiB. +fn remap_keeps_memory() { + const PAGES: usize = 4096; + let before = used(); + let mut p = blocked(PAGES); + (0..PAGES).for_each(|page| set(p, page, page as u8)); + for round in 0..32 { + let q = remap(p, PAGES, PAGES + 1, true).unwrap_or_else(|e| panic!("move {round}: {e:?}")); + assert_ne!(q, p, "move {round} stayed in place"); + unsafe { syscall::vm_release(p.add(PAGES * PAGE), PAGE) }.expect("release the old neighbour"); + unsafe { syscall::vm_protect(q.add(PAGES * PAGE), PAGE, VmProt::Read) }.expect("make the grown page the neighbour"); + p = q; + } + assert!((0..PAGES).all(|page| byte(p, page) == page as u8), "a page lost its byte over the moves"); + unsafe { syscall::vm_release(p, (PAGES + 1) * PAGE) }.expect("release the moved range"); + let after = used(); + assert!(after < before + 8 * (1 << 20), "moves kept memory: {before} bytes in use before, {after} after"); + println!(" PASS: a range moved 32 times gives back every frame and promise"); +} + /// A child that makes the page `case` names and touches it must die of it. fn faults(case: &str) { let out = Command::new("/system/bin/test_rs_mmap_stress") @@ -241,6 +346,13 @@ fn touch(case: &str) { println!("SURVIVED, wrote a read-only page"); return; } + "moved-from" => { + // Written, then moved past its neighbour: the old place is gone. + let page = blocked(1); + set(page, 0, 0x77); + assert_ne!(remap(page, 1, 2, true).expect("move"), page, "a growth past a held page stayed"); + page + } "protected-none" | "decommitted" => { let page = reserve(PAGE, VmProt::ReadWrite); set(page, 0, 0x77); From 204243e7f3eba9316f6fabf4d0d0a4b0d6af2b3b Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 27 Oct 2026 22:21:26 +0100 Subject: [PATCH 2/4] corpus: libc's realloc and calloc across the mapping threshold A pure C program allocates through libc's own allocator, whose routing between dlmalloc's heap and a mapping of its own is written beside std's rather than shared with it: a block grown from a kibibyte to 64 MiB keeps its bytes as it crosses into a mapping and grows on, and when it is shrunk back into the heap, and a large calloc reads zeros where a freed block of its size was written. Co-Authored-By: Claude Opus 5.5 --- tests/corpus/tinycc/215_large_realloc.c | 60 ++++++++++++++++++++ tests/corpus/tinycc/215_large_realloc.expect | 3 + tests/corpus/tinycc/LICENSE | 4 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/corpus/tinycc/215_large_realloc.c create mode 100644 tests/corpus/tinycc/215_large_realloc.expect diff --git a/tests/corpus/tinycc/215_large_realloc.c b/tests/corpus/tinycc/215_large_realloc.c new file mode 100644 index 00000000000..da1a5d50993 --- /dev/null +++ b/tests/corpus/tinycc/215_large_realloc.c @@ -0,0 +1,60 @@ +/* C's allocator past the size at which a block is a mapping of its own: a + block grown by realloc from a kibibyte to 64 MiB, doubling, keeps every + byte as it crosses from the heap into a mapping and grows on, and again + when it is shrunk back into the heap; calloc's large block reads zeros + where a freed one of its size was written. */ +#include +#include +#include + +static void judge(const char *what, int is) { + printf("%s: %s\n", what, is ? "yes" : "NO"); +} + +static unsigned char byte(size_t i) { + return (unsigned char)(i % 251); +} + +static int holds(const unsigned char *p, size_t len) { + for (size_t i = 0; i < len; i++) + if (p[i] != byte(i)) + return 0; + return 1; +} + +int main(void) { + size_t len = 1024; + unsigned char *p = malloc(len); + for (size_t i = 0; i < len; i++) + p[i] = byte(i); + int kept = 1; + while (len < (64u << 20)) { + unsigned char *q = realloc(p, len * 2); + if (!q) { + kept = 0; + break; + } + p = q; + kept &= holds(p, len); + for (size_t i = len; i < len * 2; i++) + p[i] = byte(i); + len *= 2; + } + judge("a block grown to 64 MiB keeps its bytes", kept && holds(p, len)); + + unsigned char *small = realloc(p, 1024); + judge("shrunk back to a kibibyte, it keeps its bytes", small && holds(small, 1024)); + free(small); + + size_t big = 8u << 20; + unsigned char *written = malloc(big); + memset(written, 0x5a, big); + free(written); + unsigned char *zeroed = calloc(big, 1); + int zeros = zeroed != NULL; + for (size_t i = 0; zeros && i < big; i++) + zeros = zeroed[i] == 0; + judge("a large calloc reads zeros", zeros); + free(zeroed); + return 0; +} diff --git a/tests/corpus/tinycc/215_large_realloc.expect b/tests/corpus/tinycc/215_large_realloc.expect new file mode 100644 index 00000000000..464d1d65eb3 --- /dev/null +++ b/tests/corpus/tinycc/215_large_realloc.expect @@ -0,0 +1,3 @@ +a block grown to 64 MiB keeps its bytes: yes +shrunk back to a kibibyte, it keeps its bytes: yes +a large calloc reads zeros: yes diff --git a/tests/corpus/tinycc/LICENSE b/tests/corpus/tinycc/LICENSE index 31428f8414f..b1aec487f73 100644 --- a/tests/corpus/tinycc/LICENSE +++ b/tests/corpus/tinycc/LICENSE @@ -21,9 +21,9 @@ against tinycc upstream at 64552b3faa39ee7948a9ea21bfcc11045b90c70d (repo.or.cz/tinycc.git, 2026-08-05), allowing for the `-` → `_` renames this project made to some filenames. - tinycc/ 338 files beside this one: 239 byte-identical to tinycc's `tests/tests2`, + tinycc/ 340 files beside this one: 239 byte-identical to tinycc's `tests/tests2`, 17 tinycc files this project modified, - 82 not upstream at all — 81 cases written here, plus + 84 not upstream at all — 83 cases written here, plus `fred.txt`, which is output `40_stdio.c` writes when it runs and which was committed by accident. From dcb6c7855d6f2def655c543bcd1465b07b85bb5d Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 27 Oct 2026 23:34:11 +0100 Subject: [PATCH 3/4] review r1: one home for large-block routing, a real in-place caller, the leak recorded - `mapped::realloc` now owns the whole decision for std and libc. A block under the threshold resizes in the heap, a mapping stays a mapping by `vm_remap`, and any other block moves by a copy. libc's hand-written crossing arm, and its incomplete comment, are gone. - A mapping aligned past a page now tries `vm_remap` in place before it copies. That is `may_move = false`'s production caller, and it saves the copy whenever the pages after the block are free. - A refused release of a large block is not refused only at the range bound. `room` also refuses when the range charge cannot grow under low memory. The leak is recorded in `issues/a-refused-release-of-a-large-block-leaks-it.md`. - `issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md` no longer says realloc copies, and voids the Linux reading that never allocated. - Comments: `vm_remap`'s doc drops the `mremap` comparison, `mapped.rs`'s header drops its rationale, and `build/sysroot.rs` names both libc files std includes. Correction to 151e90a0a's message: a program pays a reserve and release per large block "as it does under musl or glibc above their thresholds", not below. Co-Authored-By: Claude Opus 5.5 --- build/sysroot.rs | 10 +-- ...fused-release-of-a-large-block-leaks-it.md | 33 ++++++++++ ...y-hot-paths-are-slower-than-2-mib-pages.md | 14 +++-- sdk/abi/src/syscall.rs | 8 +-- sdk/libc/src/lib.rs | 19 +----- sdk/libc/src/mapped.rs | 63 +++++++++++++------ sdk/std/sys/alloc.rs | 15 ++--- 7 files changed, 101 insertions(+), 61 deletions(-) create mode 100644 issues/a-refused-release-of-a-large-block-leaks-it.md diff --git a/build/sysroot.rs b/build/sysroot.rs index ee3d762c0ab..81a583a54fb 100644 --- a/build/sysroot.rs +++ b/build/sysroot.rs @@ -25,11 +25,11 @@ //! the primary's fork repository at the commit this tree pins and never moved ([`fork_checkout`]). //! `library/std` names `toyos-abi` and `toyos` by version, and every bootstrap //! run is handed this worktree's by one `[patch]` ([`std_patch`]); each file of -//! its ToyOS backend, `sdk/std` and the C allocator `sdk/libc/src/malloc.rs` -//! it includes, it names by a `#[path]` up out of the fork. So -//! each checkout's std compiles against its own worktree's ABI and backend with -//! nothing rewritten, and the fork names no path to either crate, so moving one -//! breaks no fork commit pinned before the move. The build is bootstrap's +//! its ToyOS backend, `sdk/std` and the allocator files of `sdk/libc/src` it +//! includes (`malloc.rs`, `mapped.rs`), it names by a `#[path]` up out of the +//! fork. So each checkout's std compiles against its own worktree's ABI and +//! backend with nothing rewritten, and the fork names no path to either crate, +//! so moving one breaks no fork commit pinned before the move. The build is bootstrap's //! stage-0 local rebuild: the compiler the checkout names (`build/compiler.rs`) //! compiles the checkout's `library/` for the guest targets into //! `/build/toyos-std/`. diff --git a/issues/a-refused-release-of-a-large-block-leaks-it.md b/issues/a-refused-release-of-a-large-block-leaks-it.md new file mode 100644 index 00000000000..5ad07428789 --- /dev/null +++ b/issues/a-refused-release-of-a-large-block-leaks-it.md @@ -0,0 +1,33 @@ +--- +status: open +kind: defect +opened: 2026-10-27 +--- + +# A refused release of a large block leaks it + +std's and libc's allocators free a block of 256 KiB or more with `vm_release` +(`sdk/libc/src/mapped.rs`), and nothing can be done with a refusal. The block +stays mapped, committed and unreachable, so its frames and its promise are lost +until the process ends. + +The release is refused in two ways: +- At `MAX_RANGES`. +- When the space's range charge cannot grow (`AddressSpace::room`, then + `Charge::resize`, then `pmm::charge`). `pmm::charge` refuses exactly when + memory is low. + +Freeing a block from the middle of one inventory entry cuts it, because +adjacent large blocks and heap segments that continue each other join. `room` +then counts the two cuts. When that count crosses a `RANGE_GRAIN` above what is +already charged, the release needs a new charge. + +dlmalloc keeps a segment whose release is refused and reuses it. A large block +has no such fallback. + +Owner: the memory track. Exit: no refusal of a large block's release loses its +memory. One way is for a release, which leaves a space at most one range +larger, to need no new charge, because every change that adds ranges charges +for two more than it leaves. A release at `MAX_RANGES` must also give the +memory back, for example by freeing the frames (`vm_discard`) and returning +the promise. diff --git a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md index eb67d4d30b6..c363a28f498 100644 --- a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md +++ b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md @@ -70,7 +70,6 @@ each), what is still slower than the base: | `mem.first_touch.4k` ns/page | 887; 508 with runs | 11 (2 MiB pages; Linux 927) | | `mem.map_touch_unmap.4k` ns/page | 909; 532 with runs | 207 | | `mem.anon.total` ns/MiB | 228,677; 132,877 with runs | 51,612 (Linux 258,177) | -| `mem.realloc_grow.1m_256m` ms | 116; 67 with runs | 30 (Linux 131 ns, by `mremap`) | | `fs.tmp.seq_read.1048576` MB/s | 6,060 | 9,425 | | `fs.tmp.seq_write.1048576` MB/s | 4,297 | 5,889 | | `fs.tmp.readdir.1000` ms | 1.20 | 0.74 | @@ -79,11 +78,16 @@ each), what is still slower than the base: The copies (`fs.tmp`, `ipc`, `pipe`) point at the user-copy path above. "With runs" is `a75abd3f6` against `b48c7dd9d` on the T14, one run each; -`6d826e313` gave 500, 521, 132,434 and 67 ms for the first four rows. +`6d826e313` gave 500, 521 and 132,434 for the first three rows. -`mem.realloc_grow` still copies: std's dlmalloc 0.2.14 never maps a block on -its own, so its `remap` hook is never called, and moving pages instead needs -large blocks mapped alone, which `mem.malloc_churn.1048576` would pay for. +`mem.realloc_grow` no longer copies. std and libc map a block of 256 KiB or +more on its own (`sdk/libc/src/mapped.rs`) and grow it by `vm_remap`, which +moves pages. On the T14 at 204243e7f, `1m_256m` took 45.9 us, its last +doubling 10.0 us. Linux, with osbench's block made to escape, took 86.5 and +20.0 us. Its earlier 131 ns came from a build that never allocated. The owner +accepted on 2026-10-27 that `mem.malloc_churn.1048576` loses heap reuse for +this (T14: 29,505 to 194 kops/s; Linux 215) until ToyOS's own allocator +brings it back. Owner: the memory track. Exit: each path above measured on the T14 at a change and at its base, and against Linux; a regression is fixed or its cost diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 1fc67c0ace6..ea96be40f9d 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2439,10 +2439,10 @@ pub unsafe fn vm_decommit(addr: *mut u8, len: usize) -> Result<(), SyscallError> /// Resize `[addr, addr + old_len)`, committed private memory of one /// protection that is not executable, to `new_len` bytes, both rounded up to -/// whole pages, as Linux's `mremap` does: its pages keep their contents and a -/// grown tail reads as zeros. It grows in place where the pages after it are -/// free; otherwise, if `may_move`, its pages move to a new place, unmapped at -/// the old, with nothing copied. A shrink gives back the tail in place. +/// whole pages: its pages keep their contents and a grown tail reads as +/// zeros. It grows in place where the pages after it are free; otherwise, if +/// `may_move`, its pages move to a new place, unmapped at the old, with +/// nothing copied. A shrink gives back the tail in place. /// Answers where the range is now. Refused, it is left as it was: /// `BadAddress` for a page no range holds, `InvalidArgument` for any other /// memory or a zero length, and `ResourceExhausted` where it has no room. diff --git a/sdk/libc/src/lib.rs b/sdk/libc/src/lib.rs index 7c3260acefc..4dbc59296a6 100644 --- a/sdk/libc/src/lib.rs +++ b/sdk/libc/src/lib.rs @@ -218,23 +218,8 @@ mod runtime { unsafe { DLMALLOC.lock().free(ptr, layout.size(), layout.align()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: core::alloc::Layout, new_size: usize) -> *mut u8 { - if mapped::resizes(layout, new_size) { - return unsafe { mapped::resize(ptr, layout, new_size) }; - } - // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. - let new = unsafe { core::alloc::Layout::from_size_align_unchecked(new_size, layout.align()) }; - if !mapped::holds(layout) && !mapped::holds(new) { - return unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) }; - } - // Between the heap and a mapping of its own: moved, as the default does. - let moved = unsafe { self.alloc(new) }; - if !moved.is_null() { - unsafe { - core::ptr::copy_nonoverlapping(ptr, moved, layout.size().min(new_size)); - self.dealloc(ptr, layout); - } - } - moved + let small = || unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) }; + unsafe { mapped::realloc(self, ptr, layout, new_size, small) } } } } diff --git a/sdk/libc/src/mapped.rs b/sdk/libc/src/mapped.rs index 34323b8dae6..00c809110fa 100644 --- a/sdk/libc/src/mapped.rs +++ b/sdk/libc/src/mapped.rs @@ -1,13 +1,13 @@ //! Blocks of [`THRESHOLD`] bytes or more, each a mapping of its own beside -//! the heap that holds every smaller one, as dlmalloc's C original makes them -//! and its Rust port does not: such a block grows by `vm_remap`, in place or -//! by moving its pages, never by copying them, and its memory goes back at its -//! free. One source for std's allocator (`sdk/std/sys/alloc.rs`) and libc's. +//! the heap that holds every smaller one: such a block grows by `vm_remap`, +//! in place or by moving its pages, never by copying them, and its memory +//! goes back at its free. One source for std's allocator +//! (`sdk/std/sys/alloc.rs`) and libc's. //! //! Which a block is follows from its layout alone, which every call about it //! carries, so nothing is recorded. -use core::alloc::Layout; +use core::alloc::{GlobalAlloc, Layout}; use core::ptr; use toyos_abi::syscall::{self, VmAt, VmProt, PAGE_SIZE}; @@ -20,12 +20,6 @@ pub(crate) fn holds(layout: Layout) -> bool { layout.size() >= THRESHOLD } -/// Whether [`resize`] can take a mapped block of `layout` to `new_size`: a -/// move keeps only a page's alignment. -pub(crate) fn resizes(layout: Layout, new_size: usize) -> bool { - holds(layout) && new_size >= THRESHOLD && layout.align() <= PAGE_SIZE -} - /// A mapping for `layout`, of zeros, or null where none is granted. pub(crate) fn alloc(layout: Layout) -> *mut u8 { let at = VmAt::Anywhere { align: layout.align().max(PAGE_SIZE) }; @@ -34,20 +28,49 @@ pub(crate) fn alloc(layout: Layout) -> *mut u8 { } /// # Safety -/// `block` is a mapping [`alloc`] or [`resize`] answered for `layout`, which +/// `block` is a mapping [`alloc`] or [`realloc`] answered for `layout`, which /// nothing reaches after. pub(crate) unsafe fn dealloc(block: *mut u8, layout: Layout) { - // Refused only at the space's bound on ranges, and then the block stays. + // A refusal leaves the block mapped and unreachable + // (`issues/a-refused-release-of-a-large-block-leaks-it.md`). let _ = unsafe { syscall::vm_release(block, layout.size()) }; } -/// The mapping `block` of `layout` resized to `new_size` bytes, its contents -/// kept, where [`resizes`] says it can be; null, and the block as it was, -/// where the kernel refuses. +/// `GlobalAlloc::realloc` for `heap`, whose blocks this module routes: +/// `small` resizes a block that stays under [`THRESHOLD`], a mapping stays a +/// mapping by `vm_remap`, and any other block moves by a copy. A mapping +/// aligned past a page may only grow in place, since a move keeps only a +/// page's alignment. /// /// # Safety -/// As [`dealloc`], once this answers a block. -pub(crate) unsafe fn resize(block: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - debug_assert!(resizes(layout, new_size)); - unsafe { syscall::vm_remap(block, layout.size(), new_size, true) }.unwrap_or(ptr::null_mut()) +/// As `GlobalAlloc::realloc`. +pub(crate) unsafe fn realloc( + heap: &impl GlobalAlloc, + block: *mut u8, + layout: Layout, + new_size: usize, + small: impl FnOnce() -> *mut u8, +) -> *mut u8 { + // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. + let new = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + match (holds(layout), holds(new)) { + (false, false) => return small(), + (true, true) => { + let may_move = layout.align() <= PAGE_SIZE; + match unsafe { syscall::vm_remap(block, layout.size(), new_size, may_move) } { + Ok(at) => return at, + Err(_) if may_move => return ptr::null_mut(), + Err(_) => {} + } + } + _ => {} + } + let moved = unsafe { heap.alloc(new) }; + if !moved.is_null() { + unsafe { + ptr::copy_nonoverlapping(block, moved, layout.size().min(new_size)); + heap.dealloc(block, layout); + } + } + moved } diff --git a/sdk/std/sys/alloc.rs b/sdk/std/sys/alloc.rs index 63e793da6b3..1f2b2525601 100644 --- a/sdk/std/sys/alloc.rs +++ b/sdk/std/sys/alloc.rs @@ -102,14 +102,9 @@ pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { #[inline] pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if mapped::resizes(layout, new_size) { - return unsafe { mapped::resize(ptr, layout, new_size) }; - } - // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. - let new = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - if mapped::holds(layout) || mapped::holds(new) { - return unsafe { super::realloc_fallback(ptr, layout, new_size) }; - } - let _lock = lock(); - unsafe { (*DLMALLOC.get()).0.realloc(ptr, layout.size(), layout.align(), new_size) } + let small = || { + let _lock = lock(); + unsafe { (*DLMALLOC.get()).0.realloc(ptr, layout.size(), layout.align(), new_size) } + }; + unsafe { mapped::realloc(&crate::alloc::System, ptr, layout, new_size, small) } } From a9377c9daeaf6fa1ffedc45938d8aadc0b2da391 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 28 Oct 2026 00:00:57 +0100 Subject: [PATCH 4/4] review r2: mapped.rs says when a large block is copied A mapping aligned past a page that cannot grow in place is copied; the header claimed none ever is. Co-Authored-By: Claude Opus 5.5 --- sdk/libc/src/mapped.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/libc/src/mapped.rs b/sdk/libc/src/mapped.rs index 00c809110fa..53d79795af8 100644 --- a/sdk/libc/src/mapped.rs +++ b/sdk/libc/src/mapped.rs @@ -1,8 +1,8 @@ //! Blocks of [`THRESHOLD`] bytes or more, each a mapping of its own beside //! the heap that holds every smaller one: such a block grows by `vm_remap`, -//! in place or by moving its pages, never by copying them, and its memory -//! goes back at its free. One source for std's allocator -//! (`sdk/std/sys/alloc.rs`) and libc's. +//! in place or by moving its pages, and is copied only when it is aligned past +//! a page and cannot grow in place; its memory goes back at its free. One +//! source for std's allocator (`sdk/std/sys/alloc.rs`) and libc's. //! //! Which a block is follows from its layout alone, which every call about it //! carries, so nothing is recorded.