diff --git a/build/sysroot.rs b/build/sysroot.rs index ee3d762c0ab..81a583a54fb 100644 --- a/build/sysroot.rs +++ b/build/sysroot.rs @@ -25,11 +25,11 @@ //! the primary's fork repository at the commit this tree pins and never moved ([`fork_checkout`]). //! `library/std` names `toyos-abi` and `toyos` by version, and every bootstrap //! run is handed this worktree's by one `[patch]` ([`std_patch`]); each file of -//! its ToyOS backend, `sdk/std` and the C allocator `sdk/libc/src/malloc.rs` -//! it includes, it names by a `#[path]` up out of the fork. So -//! each checkout's std compiles against its own worktree's ABI and backend with -//! nothing rewritten, and the fork names no path to either crate, so moving one -//! breaks no fork commit pinned before the move. The build is bootstrap's +//! its ToyOS backend, `sdk/std` and the allocator files of `sdk/libc/src` it +//! includes (`malloc.rs`, `mapped.rs`), it names by a `#[path]` up out of the +//! fork. So each checkout's std compiles against its own worktree's ABI and +//! backend with nothing rewritten, and the fork names no path to either crate, +//! so moving one breaks no fork commit pinned before the move. The build is bootstrap's //! stage-0 local rebuild: the compiler the checkout names (`build/compiler.rs`) //! compiles the checkout's `library/` for the guest targets into //! `/build/toyos-std/`. diff --git a/issues/a-refused-release-of-a-large-block-leaks-it.md b/issues/a-refused-release-of-a-large-block-leaks-it.md new file mode 100644 index 00000000000..5ad07428789 --- /dev/null +++ b/issues/a-refused-release-of-a-large-block-leaks-it.md @@ -0,0 +1,33 @@ +--- +status: open +kind: defect +opened: 2026-10-27 +--- + +# A refused release of a large block leaks it + +std's and libc's allocators free a block of 256 KiB or more with `vm_release` +(`sdk/libc/src/mapped.rs`), and nothing can be done with a refusal. The block +stays mapped, committed and unreachable, so its frames and its promise are lost +until the process ends. + +The release is refused in two ways: +- At `MAX_RANGES`. +- When the space's range charge cannot grow (`AddressSpace::room`, then + `Charge::resize`, then `pmm::charge`). `pmm::charge` refuses exactly when + memory is low. + +Freeing a block from the middle of one inventory entry cuts it, because +adjacent large blocks and heap segments that continue each other join. `room` +then counts the two cuts. When that count crosses a `RANGE_GRAIN` above what is +already charged, the release needs a new charge. + +dlmalloc keeps a segment whose release is refused and reuses it. A large block +has no such fallback. + +Owner: the memory track. Exit: no refusal of a large block's release loses its +memory. One way is for a release, which leaves a space at most one range +larger, to need no new charge, because every change that adds ranges charges +for two more than it leaves. A release at `MAX_RANGES` must also give the +memory back, for example by freeing the frames (`vm_discard`) and returning +the promise. diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index 6587e29ea02..ca28344fb0d 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -77,9 +77,10 @@ live split of a user leaf. 4. **Demand-zero anonymous `mmap`.** Built, over 4 KiB pages only: a fault populates one page, unmapping a range no thread touched sends no IPI, and `munmap` and a `FIXED` `mmap` take any run of pages, as POSIX does. - `vm_reserve`, `vm_protect`, `vm_discard`, `vm_decommit` and `vm_release` - are the calls, behind libc's `mmap`, `mprotect`, `madvise` and `munmap` - and std's allocator, which trims, and its thread stacks, each reserved + `vm_reserve`, `vm_protect`, `vm_discard`, `vm_decommit`, `vm_remap` and + `vm_release` are the calls, behind libc's `mmap`, `mprotect`, `madvise` and + `munmap`, std's and libc's allocators, which trim and grow a large block + by `vm_remap` in place or by moving its pages, and std's thread stacks, each reserved at the size asked with a guard page below. A commitment is promised its frames when it is made, and every other allocation is refused rather than take a promised frame (`kernel::frames`), so a commitment past what memory diff --git a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md index eb67d4d30b6..c363a28f498 100644 --- a/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md +++ b/issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md @@ -70,7 +70,6 @@ each), what is still slower than the base: | `mem.first_touch.4k` ns/page | 887; 508 with runs | 11 (2 MiB pages; Linux 927) | | `mem.map_touch_unmap.4k` ns/page | 909; 532 with runs | 207 | | `mem.anon.total` ns/MiB | 228,677; 132,877 with runs | 51,612 (Linux 258,177) | -| `mem.realloc_grow.1m_256m` ms | 116; 67 with runs | 30 (Linux 131 ns, by `mremap`) | | `fs.tmp.seq_read.1048576` MB/s | 6,060 | 9,425 | | `fs.tmp.seq_write.1048576` MB/s | 4,297 | 5,889 | | `fs.tmp.readdir.1000` ms | 1.20 | 0.74 | @@ -79,11 +78,16 @@ each), what is still slower than the base: The copies (`fs.tmp`, `ipc`, `pipe`) point at the user-copy path above. "With runs" is `a75abd3f6` against `b48c7dd9d` on the T14, one run each; -`6d826e313` gave 500, 521, 132,434 and 67 ms for the first four rows. +`6d826e313` gave 500, 521 and 132,434 for the first three rows. -`mem.realloc_grow` still copies: std's dlmalloc 0.2.14 never maps a block on -its own, so its `remap` hook is never called, and moving pages instead needs -large blocks mapped alone, which `mem.malloc_churn.1048576` would pay for. +`mem.realloc_grow` no longer copies. std and libc map a block of 256 KiB or +more on its own (`sdk/libc/src/mapped.rs`) and grow it by `vm_remap`, which +moves pages. On the T14 at 204243e7f, `1m_256m` took 45.9 us, its last +doubling 10.0 us. Linux, with osbench's block made to escape, took 86.5 and +20.0 us. Its earlier 131 ns came from a build that never allocated. The owner +accepted on 2026-10-27 that `mem.malloc_churn.1048576` loses heap reuse for +this (T14: 29,505 to 194 kops/s; Linux 215) until ToyOS's own allocator +brings it back. Owner: the memory track. Exit: each path above measured on the T14 at a change and at its base, and against Linux; a regression is fixed or its cost diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 3cc57b27c45..519b10f4cf6 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -610,6 +610,97 @@ impl AddressSpace { Ok(()) } + /// Resize `[start, start + old)`, committed private memory of one + /// protection that is not executable, to `new` bytes, whole pages both + /// (`toyos_abi::syscall::vm_remap`): a shrink releases the tail, a growth + /// takes the free pages after it, or else, if `may_move`, its leaves move + /// to a new place and nothing is copied. Promised and all or nothing, as + /// [`reserve`](Self::reserve); answers where the range is now. + pub fn remap(&mut self, start: UserAddr, old: u64, new: u64, may_move: bool) -> Result { + let (start, end) = self.held(start, old)?; + let (_, entry) = self.inventory.find(start).expect("space: a page `held` answered for"); + let prot = entry.prot; + // Neighbours alike join, so one entry is the whole of one kind and protection. + if !matches!(entry.kind, Kind::Private { committed: true }) || prot == Prot::ReadExec || entry.end < end { + return Err(SyscallError::InvalidArgument); + } + if new == 0 || !new.is_multiple_of(PAGE_SIZE) || new > self.inventory.top() - self.inventory.floor() { + return Err(SyscallError::InvalidArgument); + } + if new <= old { + if new < old { + self.room(start + new, end, 0)?; + self.release(UserAddr::new(start + new), old - new); + } + return Ok(UserAddr::new(start)); + } + let in_place = start.checked_add(new).is_some_and(|grown| self.inventory.holds(end, grown) && self.inventory.is_free(end, grown)); + let to = if in_place { + self.room(end, start + new, 1)?; + start + } else if may_move { + self.room(start, end, 1)?; + self.inventory.place(new, PAGE_SIZE).ok_or(SyscallError::ResourceExhausted)? + } else { + return Err(SyscallError::ResourceExhausted); + }; + let added = (new - old) / PAGE_SIZE; + if !pmm::promise(&self.job, added) { + return Err(SyscallError::ResourceExhausted); + } + let built = if in_place { self.build(end, start + new) } else { self.build(to, to + new) }; + built.inspect_err(|_| pmm::unpromise(&self.job, added))?; + let kind = Kind::Private { committed: true }; + if in_place { + self.inventory.insert(end, Entry { end: start + new, prot, kind }); + } else { + let present = self.move_leaves(start, end, to); + self.inventory.remove(start, end); + self.inventory.insert(to, Entry { end: to + new, prot, kind }); + let inventory = &self.inventory; + let unlinked = self.tables.unlink(start, end, |s, e| inventory.backed(s, e)); + if present || !unlinked.is_empty() { + self.shoot_down(start, end, &unlinked); + } + self.tables.free(unlinked); + self.usage.taken.fetch_add(1, Relaxed); + } + self.usage.made.fetch_add(1, Relaxed); + self.settle(); + Ok(UserAddr::new(to)) + } + + /// Move every leaf of `[from, end)` to the same place in the range from + /// `to`, whose tables are built and whose leaves are empty, leaving the + /// old ones empty: their frames stay this space's. Answers whether one + /// was present, which some CPU may still cache. + fn move_leaves(&mut self, from: u64, end: u64, to: u64) -> bool { + // A held `LeafTable` borrows all the tables, so leaves are taken out + // of the source a batch at a time and then written at the destination. + const BATCH: usize = 64; + let mut present = false; + let mut va = from; + while va < end { + let mut batch = [(0, Leaf::Empty); BATCH]; + let mut n = 0; + let mut leaves = self.tables.leaf_table(va).unwrap_or_else(|| panic!("space: no tables under {va:#x}")); + while va < end && leaves.covers(va) && n < BATCH { + let leaf = leaves.get(va); + if leaf != Leaf::Empty { + leaves.set(va, Leaf::Empty); + batch[n] = (va, leaf); + n += 1; + } + va += PAGE_SIZE; + } + for &(at, leaf) in &batch[..n] { + present |= matches!(leaf, Leaf::Mapped(_)); + self.tables.set(at - from + to, leaf); + } + } + present + } + /// `[start, start + len)` as pages every one of which a range holds. fn held(&self, start: UserAddr, len: u64) -> Result<(u64, u64), SyscallError> { let end = pages(start.raw(), len).ok_or(SyscallError::InvalidArgument)?; diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 078810ab35f..b64719b1229 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -51,7 +51,7 @@ use super::proc::{ sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_join, sys_thread_spawn, }; use super::vm::{ - shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, + shared_image, sys_dlopen, sys_dlsym, sys_query_modules, sys_tls_alloc_block, sys_vm_decommit, sys_vm_discard, sys_vm_remap, sys_vm_protect, sys_vm_release, sys_vm_reserve, }; @@ -364,6 +364,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_VM_PROTECT => sys_vm_protect(a1, a2, a3), SYS_VM_DISCARD => sys_vm_discard(a1, a2), SYS_VM_DECOMMIT => sys_vm_decommit(a1, a2), + SYS_VM_REMAP => sys_vm_remap(a1, a2, a3, a4), SYS_PREAD => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; with_object(RawHandle(a1 as u32), Rights::READ, |o| ops::pread(o, a4, &mut buf)) diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index 9eaacd01ab9..30365cfc39f 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -1,5 +1,5 @@ -//! The caller's own address space: `vm_*` reserve, commit, protect, discard -//! and take away private memory, `dlopen` maps a library image, `SYS_TLS_ALLOC_BLOCK` +//! The caller's own address space: `vm_*` reserve, commit, protect, discard, +//! resize and take away private memory, `dlopen` maps a library image, `SYS_TLS_ALLOC_BLOCK` //! maps a thread's per-module TLS block, `SYS_QUERY_MODULES` reports what is //! loaded. Exhausting address space is an error return, never an `.expect`. //! @@ -89,6 +89,21 @@ pub(super) fn sys_vm_decommit(addr: u64, len: u64) -> u64 { answer(range(addr, len).and_then(|(start, len)| process::current_address_space().lock().decommit(start, len))) } +/// Resize committed private memory in place, or move it when `may_move` is 1. +pub(super) fn sys_vm_remap(addr: u64, old: u64, new: u64, may_move: u64) -> u64 { + let remapped = (|| { + let may_move = match may_move { + 0 => false, + 1 => true, + _ => return Err(SyscallError::InvalidArgument), + }; + let (start, old) = range(addr, old)?; + let new = whole_pages(new).ok_or(SyscallError::InvalidArgument)?; + process::current_address_space().lock().remap(start, old, new, may_move) + })(); + remapped.map_or_else(SyscallError::to_u64, UserAddr::raw) +} + /// The first `len` bytes of the shared memory object `handle` names, as a /// program image; `Err` is the syscall's answer. pub(super) fn shared_image(handle: u64, len: u64) -> Result { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 3ee7518eb80..bc68e7ed4c4 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -333,6 +333,8 @@ pub const SYS_SHM_SUB: u64 = 143; pub const SYS_JOB_FLOOR: u64 = 144; /// What a job's account holds. See [`account_info`]. pub const SYS_ACCOUNT_INFO: u64 = 145; +/// Resize a run of committed private memory, in place or moved. See [`vm_remap`]. +pub const SYS_VM_REMAP: u64 = 146; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -340,13 +342,13 @@ pub const SYS_ACCOUNT_INFO: u64 = 145; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 147; +pub const SYSCALL_PROFILE_BINS: usize = 148; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_ACCOUNT_INFO < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_VM_REMAP < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -2487,6 +2489,24 @@ pub unsafe fn vm_decommit(addr: *mut u8, len: usize) -> Result<(), SyscallError> check_unit(syscall(SYS_VM_DECOMMIT, addr as u64, len as u64, 0, 0)) } +/// Resize `[addr, addr + old_len)`, committed private memory of one +/// protection that is not executable, to `new_len` bytes, both rounded up to +/// whole pages: its pages keep their contents and a grown tail reads as +/// zeros. It grows in place where the pages after it are free; otherwise, if +/// `may_move`, its pages move to a new place, unmapped at the old, with +/// nothing copied. A shrink gives back the tail in place. +/// Answers where the range is now. Refused, it is left as it was: +/// `BadAddress` for a page no range holds, `InvalidArgument` for any other +/// memory or a zero length, and `ResourceExhausted` where it has no room. +/// +/// # Safety +/// Nothing reaches the old pages after this answers a new place, nor past +/// `new_len` after a shrink. +pub unsafe fn vm_remap(addr: *mut u8, old_len: usize, new_len: usize, may_move: bool) -> Result<*mut u8, SyscallError> { + check(syscall(SYS_VM_REMAP, addr as u64, old_len as u64, new_len as u64, may_move as u64)) + .map(|at| core::ptr::with_exposed_provenance_mut(at as usize)) +} + /// Non-blocking read. Returns bytes read, or `Err(WouldBlock)` if no data available. pub fn read_nonblock(handle: RawHandle, buf: &mut [u8]) -> Result { check(syscall(SYS_READ_NONBLOCK, handle.0 as u64, buf.as_mut_ptr() as u64, buf.len() as u64, 0)).map(|n| n as usize) diff --git a/sdk/libc/src/lib.rs b/sdk/libc/src/lib.rs index be1e48bc27c..4dbc59296a6 100644 --- a/sdk/libc/src/lib.rs +++ b/sdk/libc/src/lib.rs @@ -48,6 +48,9 @@ mod uname; mod utf8; mod wchar; +#[cfg(not(feature = "std-runtime"))] +mod mapped; + // C runtime: the entry `arch::_start` calls, panic handler, and global allocator. // Only for pure C programs (no Rust std). When linked into a Rust program // with std, std provides these. @@ -193,15 +196,30 @@ mod runtime { #[global_allocator] pub(crate) static ALLOCATOR: LibcAllocator = LibcAllocator; + use super::mapped; + unsafe impl core::alloc::GlobalAlloc for LibcAllocator { unsafe fn alloc(&self, layout: core::alloc::Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } unsafe { DLMALLOC.lock().malloc(layout.size(), layout.align()) } } + unsafe fn alloc_zeroed(&self, layout: core::alloc::Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } + unsafe { DLMALLOC.lock().calloc(layout.size(), layout.align()) } + } unsafe fn dealloc(&self, ptr: *mut u8, layout: core::alloc::Layout) { + if mapped::holds(layout) { + return unsafe { mapped::dealloc(ptr, layout) }; + } unsafe { DLMALLOC.lock().free(ptr, layout.size(), layout.align()) } } unsafe fn realloc(&self, ptr: *mut u8, layout: core::alloc::Layout, new_size: usize) -> *mut u8 { - unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) } + let small = || unsafe { DLMALLOC.lock().realloc(ptr, layout.size(), layout.align(), new_size) }; + unsafe { mapped::realloc(self, ptr, layout, new_size, small) } } } } diff --git a/sdk/libc/src/mapped.rs b/sdk/libc/src/mapped.rs new file mode 100644 index 00000000000..53d79795af8 --- /dev/null +++ b/sdk/libc/src/mapped.rs @@ -0,0 +1,76 @@ +//! Blocks of [`THRESHOLD`] bytes or more, each a mapping of its own beside +//! the heap that holds every smaller one: such a block grows by `vm_remap`, +//! in place or by moving its pages, and is copied only when it is aligned past +//! a page and cannot grow in place; its memory goes back at its free. One +//! source for std's allocator (`sdk/std/sys/alloc.rs`) and libc's. +//! +//! Which a block is follows from its layout alone, which every call about it +//! carries, so nothing is recorded. + +use core::alloc::{GlobalAlloc, Layout}; +use core::ptr; + +use toyos_abi::syscall::{self, VmAt, VmProt, PAGE_SIZE}; + +/// dlmalloc's `DEFAULT_MMAP_THRESHOLD`. +const THRESHOLD: usize = 256 * 1024; + +/// Whether a block of `layout` is a mapping of its own. +pub(crate) fn holds(layout: Layout) -> bool { + layout.size() >= THRESHOLD +} + +/// A mapping for `layout`, of zeros, or null where none is granted. +pub(crate) fn alloc(layout: Layout) -> *mut u8 { + let at = VmAt::Anywhere { align: layout.align().max(PAGE_SIZE) }; + // SAFETY: placed anywhere, so it replaces nothing. + unsafe { syscall::vm_reserve(layout.size(), VmProt::ReadWrite, at) }.unwrap_or(ptr::null_mut()) +} + +/// # Safety +/// `block` is a mapping [`alloc`] or [`realloc`] answered for `layout`, which +/// nothing reaches after. +pub(crate) unsafe fn dealloc(block: *mut u8, layout: Layout) { + // A refusal leaves the block mapped and unreachable + // (`issues/a-refused-release-of-a-large-block-leaks-it.md`). + let _ = unsafe { syscall::vm_release(block, layout.size()) }; +} + +/// `GlobalAlloc::realloc` for `heap`, whose blocks this module routes: +/// `small` resizes a block that stays under [`THRESHOLD`], a mapping stays a +/// mapping by `vm_remap`, and any other block moves by a copy. A mapping +/// aligned past a page may only grow in place, since a move keeps only a +/// page's alignment. +/// +/// # Safety +/// As `GlobalAlloc::realloc`. +pub(crate) unsafe fn realloc( + heap: &impl GlobalAlloc, + block: *mut u8, + layout: Layout, + new_size: usize, + small: impl FnOnce() -> *mut u8, +) -> *mut u8 { + // SAFETY: `GlobalAlloc::realloc` requires `new_size` at this alignment to be a layout. + let new = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + match (holds(layout), holds(new)) { + (false, false) => return small(), + (true, true) => { + let may_move = layout.align() <= PAGE_SIZE; + match unsafe { syscall::vm_remap(block, layout.size(), new_size, may_move) } { + Ok(at) => return at, + Err(_) if may_move => return ptr::null_mut(), + Err(_) => {} + } + } + _ => {} + } + let moved = unsafe { heap.alloc(new) }; + if !moved.is_null() { + unsafe { + ptr::copy_nonoverlapping(block, moved, layout.size().min(new_size)); + heap.dealloc(block, layout); + } + } + moved +} diff --git a/sdk/std/sys/alloc.rs b/sdk/std/sys/alloc.rs index cc8fd1d6a0b..1f2b2525601 100644 --- a/sdk/std/sys/alloc.rs +++ b/sdk/std/sys/alloc.rs @@ -6,6 +6,9 @@ use crate::alloc::Layout; use crate::ptr; use crate::sys::sync::Mutex; +#[path = "../../libc/src/mapped.rs"] +mod mapped; + /// dlmalloc's memory: the process's own private pages, committed whole and /// populated as they are touched, and given back in part when it trims. struct ToyOsAllocator; @@ -72,24 +75,36 @@ impl Drop for DropLock { #[inline] pub unsafe fn alloc(layout: Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.malloc(layout.size(), layout.align()) } } #[inline] pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { + if mapped::holds(layout) { + return mapped::alloc(layout); + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.calloc(layout.size(), layout.align()) } } #[inline] pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { + if mapped::holds(layout) { + return unsafe { mapped::dealloc(ptr, layout) }; + } let _lock = lock(); unsafe { (*DLMALLOC.get()).0.free(ptr, layout.size(), layout.align()) } } #[inline] pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - let _lock = lock(); - unsafe { (*DLMALLOC.get()).0.realloc(ptr, layout.size(), layout.align(), new_size) } + let small = || { + let _lock = lock(); + unsafe { (*DLMALLOC.get()).0.realloc(ptr, layout.size(), layout.align(), new_size) } + }; + unsafe { mapped::realloc(&crate::alloc::System, ptr, layout, new_size, small) } } diff --git a/tests/corpus/tinycc/215_large_realloc.c b/tests/corpus/tinycc/215_large_realloc.c new file mode 100644 index 00000000000..da1a5d50993 --- /dev/null +++ b/tests/corpus/tinycc/215_large_realloc.c @@ -0,0 +1,60 @@ +/* C's allocator past the size at which a block is a mapping of its own: a + block grown by realloc from a kibibyte to 64 MiB, doubling, keeps every + byte as it crosses from the heap into a mapping and grows on, and again + when it is shrunk back into the heap; calloc's large block reads zeros + where a freed one of its size was written. */ +#include +#include +#include + +static void judge(const char *what, int is) { + printf("%s: %s\n", what, is ? "yes" : "NO"); +} + +static unsigned char byte(size_t i) { + return (unsigned char)(i % 251); +} + +static int holds(const unsigned char *p, size_t len) { + for (size_t i = 0; i < len; i++) + if (p[i] != byte(i)) + return 0; + return 1; +} + +int main(void) { + size_t len = 1024; + unsigned char *p = malloc(len); + for (size_t i = 0; i < len; i++) + p[i] = byte(i); + int kept = 1; + while (len < (64u << 20)) { + unsigned char *q = realloc(p, len * 2); + if (!q) { + kept = 0; + break; + } + p = q; + kept &= holds(p, len); + for (size_t i = len; i < len * 2; i++) + p[i] = byte(i); + len *= 2; + } + judge("a block grown to 64 MiB keeps its bytes", kept && holds(p, len)); + + unsigned char *small = realloc(p, 1024); + judge("shrunk back to a kibibyte, it keeps its bytes", small && holds(small, 1024)); + free(small); + + size_t big = 8u << 20; + unsigned char *written = malloc(big); + memset(written, 0x5a, big); + free(written); + unsigned char *zeroed = calloc(big, 1); + int zeros = zeroed != NULL; + for (size_t i = 0; zeros && i < big; i++) + zeros = zeroed[i] == 0; + judge("a large calloc reads zeros", zeros); + free(zeroed); + return 0; +} diff --git a/tests/corpus/tinycc/215_large_realloc.expect b/tests/corpus/tinycc/215_large_realloc.expect new file mode 100644 index 00000000000..464d1d65eb3 --- /dev/null +++ b/tests/corpus/tinycc/215_large_realloc.expect @@ -0,0 +1,3 @@ +a block grown to 64 MiB keeps its bytes: yes +shrunk back to a kibibyte, it keeps its bytes: yes +a large calloc reads zeros: yes diff --git a/tests/corpus/tinycc/LICENSE b/tests/corpus/tinycc/LICENSE index 31428f8414f..b1aec487f73 100644 --- a/tests/corpus/tinycc/LICENSE +++ b/tests/corpus/tinycc/LICENSE @@ -21,9 +21,9 @@ against tinycc upstream at 64552b3faa39ee7948a9ea21bfcc11045b90c70d (repo.or.cz/tinycc.git, 2026-08-05), allowing for the `-` → `_` renames this project made to some filenames. - tinycc/ 338 files beside this one: 239 byte-identical to tinycc's `tests/tests2`, + tinycc/ 340 files beside this one: 239 byte-identical to tinycc's `tests/tests2`, 17 tinycc files this project modified, - 82 not upstream at all — 81 cases written here, plus + 84 not upstream at all — 83 cases written here, plus `fred.txt`, which is output `40_stdio.c` writes when it runs and which was committed by accident. diff --git a/tests/guest/osbench/src/mem.rs b/tests/guest/osbench/src/mem.rs index 3fc9bd177f1..b9d0712bb69 100644 --- a/tests/guest/osbench/src/mem.rs +++ b/tests/guest/osbench/src/mem.rs @@ -188,13 +188,14 @@ fn churn(size: usize, threads: usize) { /// A block grown by `realloc` from 1 MiB to 256 MiB, doubling, its first /// mebibyte written: ns for the whole growth and for its last doubling, five /// rounds after one. An allocator that can move the pages answers without -/// copying; one that cannot copies each time. +/// copying; one that cannot copies each time. The block escapes through +/// `black_box`, or the compiler drops every call. fn realloc_grow() { let round = || -> (u64, u64) { let mut size = MIB; let layout = Layout::from_size_align(size, 16).expect("a layout"); // SAFETY: a non-zero layout. - let mut ptr = unsafe { alloc(layout) }; + let mut ptr = std::hint::black_box(unsafe { alloc(layout) }); assert!(!ptr.is_null()); // SAFETY: inside the block. unsafe { std::ptr::write_bytes(ptr, 0x33, MIB) }; @@ -202,7 +203,7 @@ fn realloc_grow() { while size < 256 * MIB { let t = Instant::now(); // SAFETY: `ptr` holds `size` bytes allocated with this alignment. - ptr = unsafe { realloc(ptr, Layout::from_size_align(size, 16).expect("a layout"), size * 2) }; + ptr = std::hint::black_box(unsafe { realloc(ptr, Layout::from_size_align(size, 16).expect("a layout"), size * 2) }); last = ns_since(t); total += last; assert!(!ptr.is_null(), "realloc to {} bytes was refused", size * 2); diff --git a/tests/guest/rust-tests/src/bin/allocator_stress.rs b/tests/guest/rust-tests/src/bin/allocator_stress.rs index ec6049cdc68..7390e224cd1 100644 --- a/tests/guest/rust-tests/src/bin/allocator_stress.rs +++ b/tests/guest/rust-tests/src/bin/allocator_stress.rs @@ -13,6 +13,7 @@ fn main() { test_fragmentation_resistance(); test_memory_stats(); test_oom_graceful(); + test_large_blocks(); println!("all allocator stress tests passed"); } @@ -225,6 +226,45 @@ fn test_grow_and_shrink() { println!(" grow and shrink: ok"); } +/// Blocks past the allocator's mapping threshold, a mapping each: a vector +/// grown by doubling from the heap into one and on, and shrunk back into the +/// heap, keeps every byte; a zeroed one reads zeros where a freed one of its +/// size was written; and one aligned past a page keeps its alignment when it +/// grows. +fn test_large_blocks() { + let byte = |i: usize| (i % 251) as u8; + let mut v: Vec = Vec::new(); + for len in (10..=26).map(|shift| 1usize << shift) { + v.reserve_exact(len - v.len()); + v.extend((v.len()..len).map(byte)); + assert!(v.iter().enumerate().all(|(i, &b)| b == byte(i)), "grown to {len} bytes, a byte changed"); + } + v.truncate(100 * 1024); + v.shrink_to_fit(); + assert!(v.iter().enumerate().all(|(i, &b)| b == byte(i)), "shrunk into the heap, a byte changed"); + drop(v); + + let written = vec![0x5Au8; 8 << 20]; + drop(written); + let zeroed = vec![0u8; 8 << 20]; + assert!(zeroed.iter().all(|&b| b == 0), "a zeroed large block held a byte"); + drop(zeroed); + + let layout = std::alloc::Layout::from_size_align(4 << 20, 2 << 20).unwrap(); + // SAFETY: a non-zero layout; the block is written inside it, grown with + // it and freed with the grown layout. + unsafe { + let p = std::alloc::alloc(layout); + assert!(!p.is_null() && p as usize % (2 << 20) == 0, "{p:p} is not 2 MiB aligned"); + p.write_bytes(0x3C, 4 << 20); + let q = std::alloc::realloc(p, layout, 8 << 20); + assert!(!q.is_null() && q as usize % (2 << 20) == 0, "grown, {q:p} is not 2 MiB aligned"); + assert!((0..4 << 20).step_by(4096).all(|i| *q.add(i) == 0x3C), "grown, an aligned block lost a byte"); + std::alloc::dealloc(q, std::alloc::Layout::from_size_align(8 << 20, 2 << 20).unwrap()); + } + println!(" large blocks: ok"); +} + /// Test that the allocator handles fragmentation gracefully. /// Allocate alternating sizes, free every other one, then try to /// allocate in the freed gaps. diff --git a/tests/guest/rust-tests/src/bin/mmap_stress.rs b/tests/guest/rust-tests/src/bin/mmap_stress.rs index d3b2d6f7a5c..d8fac187da8 100644 --- a/tests/guest/rust-tests/src/bin/mmap_stress.rs +++ b/tests/guest/rust-tests/src/bin/mmap_stress.rs @@ -3,7 +3,9 @@ //! range, across two, or a hole — and a fixed reserve replaces whatever the //! pages it names held; `vm_protect` commits a reservation and changes //! protection in place, contents kept; `vm_discard` keeps the pages and reads -//! back zeros; `vm_decommit` makes them a reservation again. A page +//! back zeros; `vm_decommit` makes them a reservation again; `vm_remap` +//! grows, moves and shrinks committed memory, contents kept, and loses no +//! frame or promise when it moves. A page //! populated with its neighbours is populated as its own range says. The //! ranges one space holds are bounded, and a commitment past what memory //! holds is refused when it is asked for. @@ -39,7 +41,7 @@ fn set(p: *mut u8, page: usize, value: u8) { } /// The pages a child makes and touches, none of which allows the touch. -const FAULTING: [&str; 4] = ["reserved", "protected-none", "decommitted", "read-only-beside-a-run"]; +const FAULTING: [&str; 5] = ["reserved", "protected-none", "decommitted", "read-only-beside-a-run", "moved-from"]; fn main() { if let Some(case) = std::env::args().nth(1) { @@ -49,10 +51,12 @@ fn main() { protection(); discard_and_decommit(); ranges_are_bounded(); + resize(); + remap_keeps_memory(); for case in FAULTING { faults(case); } - println!("vm_reserve, vm_protect, vm_discard, vm_decommit and vm_release do what each says"); + println!("vm_reserve, vm_protect, vm_discard, vm_decommit, vm_remap and vm_release do what each says"); } fn placement_and_release() { @@ -208,6 +212,107 @@ fn ranges_are_bounded() { println!(" PASS: a space holds at most {MAX_RANGES} ranges, and gives each back at the bound"); } +fn remap(p: *mut u8, old: usize, new: usize, may_move: bool) -> Result<*mut u8, SyscallError> { + // SAFETY: every range remapped here is this test's own, reached through + // raw pointers only, and through the answer alone once it moves. + unsafe { syscall::vm_remap(p, old * PAGE, new * PAGE, may_move) } +} + +/// `pages` committed pages followed by a read-only one of the same +/// reservation, its own range, so the first `pages` cannot grow in place. +fn blocked(pages: usize) -> *mut u8 { + let p = reserve((pages + 1) * PAGE, VmProt::ReadWrite); + unsafe { syscall::vm_protect(p.add(pages * PAGE), PAGE, VmProt::Read) }.expect("protect the blocking page"); + p +} + +fn resize() { + // Grown into the free pages after it: the same address, the bytes kept, + // and a tail of zeros that takes writes. + let p = reserve(8 * PAGE, VmProt::ReadWrite); + unsafe { syscall::vm_release(p.add(4 * PAGE), 4 * PAGE) }.expect("free the last four pages"); + (0..4).for_each(|page| set(p, page, 0x50 + page as u8)); + assert_eq!(remap(p, 4, 8, false), Ok(p), "growth into free pages"); + assert_eq!((0..8).map(|page| byte(p, page)).collect::>(), [0x50, 0x51, 0x52, 0x53, 0, 0, 0, 0], "grown in place"); + set(p, 7, 0x57); + assert_eq!(byte(p, 7), 0x57, "the grown tail is not writable"); + + // Shrunk: the tail goes and the rest stays. + assert_eq!(remap(p, 8, 2, false), Ok(p), "a shrink"); + assert_eq!((byte(p, 0), byte(p, 1)), (0x50, 0x51), "a shrink took a kept page's byte"); + assert_eq!(unsafe { syscall::vm_protect(p, 3 * PAGE, VmProt::ReadWrite) }, Err(SyscallError::BadAddress), "a shrink kept its tail"); + unsafe { syscall::vm_release(p, 2 * PAGE) }.expect("release"); + + // A neighbour in the way refuses a growth that may not move, and moves + // one that may: every page's byte at the new place, nothing at the old, + // the neighbour untouched. 768 pages cross tables of leaves and batches. + const PAGES: usize = 768; + let p = blocked(PAGES); + (0..PAGES).step_by(3).for_each(|page| set(p, page, page as u8 | 1)); + assert_eq!(remap(p, PAGES, 2 * PAGES, false), Err(SyscallError::ResourceExhausted), "growth into a held page"); + assert_eq!(byte(p, 3), 3 | 1, "a refused growth touched the range"); + let q = remap(p, PAGES, 2 * PAGES, true).expect("a growth that may move"); + assert_ne!(q, p, "growth past a held page stayed in place"); + for page in 0..2 * PAGES { + let want = if page < PAGES && page % 3 == 0 { page as u8 | 1 } else { 0 }; + assert_eq!(byte(q, page), want, "page {page} of the moved range"); + } + set(q, 1, 0x61); + assert_eq!(byte(q, 1), 0x61, "a moved page is not writable"); + assert_eq!(unsafe { syscall::vm_protect(p, PAGE, VmProt::ReadWrite) }, Err(SyscallError::BadAddress), "the old place is still held"); + assert_eq!(unsafe { syscall::vm_protect(p.add(PAGES * PAGE), PAGE, VmProt::Read) }, Ok(()), "the neighbour went"); + unsafe { syscall::vm_release(p.add(PAGES * PAGE), PAGE) }.expect("release the neighbour"); + unsafe { syscall::vm_release(q, 2 * PAGES * PAGE) }.expect("release the moved range"); + + // What remap refuses: any memory but committed private pages of one + // protection, a page no range holds, and a length of nothing. + let none = reserve(2 * PAGE, VmProt::None); + assert_eq!(remap(none, 2, 4, true), Err(SyscallError::InvalidArgument), "a reservation"); + unsafe { syscall::vm_release(none, 2 * PAGE) }.expect("release"); + let mixed = blocked(1); + assert_eq!(remap(mixed, 2, 4, true), Err(SyscallError::InvalidArgument), "two protections"); + assert_eq!(remap(mixed, 1, 0, true), Err(SyscallError::InvalidArgument), "no bytes"); + assert_eq!(remap(unsafe { mixed.add(1) }, 1, 2, true), Err(SyscallError::InvalidArgument), "off a page"); + unsafe { syscall::vm_release(mixed, 2 * PAGE) }.expect("release"); + assert_eq!(remap(mixed, 1, 2, true), Err(SyscallError::BadAddress), "a page no range holds"); + let clock = toyos_abi::clock::CLOCK_PAGE as *mut u8; + assert_eq!(remap(clock, 1, 2, true), Err(SyscallError::InvalidArgument), "a shared object's page"); + // `page` asserts the clock page's magic. + let _ = toyos_abi::clock::page(); + println!(" PASS: remap grows in place or moves the pages, shrinks in place, and refuses all but committed private memory"); +} + +/// Bytes the machine has in use or promised. +fn used() -> u64 { + let mut header = [0u8; syscall::SYSINFO_HEADER_SIZE]; + assert_eq!(toyos::system::sysinfo(&mut header), header.len(), "the sysinfo header"); + syscall::SysinfoHeader::decode(&header).memory_used +} + +/// A written 16 MiB moved 32 times, each time grown by the page that becomes +/// the next move's neighbour, gives every frame and promise back: the +/// machine's memory in use after is what it was before, give or take what the +/// rest of it did meanwhile, where one move that kept its range's promise +/// would cost 16 MiB. +fn remap_keeps_memory() { + const PAGES: usize = 4096; + let before = used(); + let mut p = blocked(PAGES); + (0..PAGES).for_each(|page| set(p, page, page as u8)); + for round in 0..32 { + let q = remap(p, PAGES, PAGES + 1, true).unwrap_or_else(|e| panic!("move {round}: {e:?}")); + assert_ne!(q, p, "move {round} stayed in place"); + unsafe { syscall::vm_release(p.add(PAGES * PAGE), PAGE) }.expect("release the old neighbour"); + unsafe { syscall::vm_protect(q.add(PAGES * PAGE), PAGE, VmProt::Read) }.expect("make the grown page the neighbour"); + p = q; + } + assert!((0..PAGES).all(|page| byte(p, page) == page as u8), "a page lost its byte over the moves"); + unsafe { syscall::vm_release(p, (PAGES + 1) * PAGE) }.expect("release the moved range"); + let after = used(); + assert!(after < before + 8 * (1 << 20), "moves kept memory: {before} bytes in use before, {after} after"); + println!(" PASS: a range moved 32 times gives back every frame and promise"); +} + /// A child that makes the page `case` names and touches it must die of it. fn faults(case: &str) { let out = Command::new("/system/bin/test_rs_mmap_stress") @@ -241,6 +346,13 @@ fn touch(case: &str) { println!("SURVIVED, wrote a read-only page"); return; } + "moved-from" => { + // Written, then moved past its neighbour: the old place is gone. + let page = blocked(1); + set(page, 0, 0x77); + assert_ne!(remap(page, 1, 2, true).expect("move"), page, "a growth past a held page stayed"); + page + } "protected-none" | "decommitted" => { let page = reserve(PAGE, VmProt::ReadWrite); set(page, 0, 0x77);